chore: add extensible chopsticks setup (#740)

To spin up Chopsticks-based networks using actual data.

It's based on top of #714.

For more info about how to use this new utility, check the
`chopsticks/README.md` file.

Author: ntn-x2

.gitignore

@@ -17,10 +17,12 @@
 # ignore vscode settings
 .vscode
 
-**/__pycache__
-
 **/node_modules
+**/.yarn/*
+!**/.yarn/releases
 
 runtimes/spiritnet/src/xcm_tests/e2e/out
 
-*.db.sqlite*
+chopsticks/**/db.sqlite*
+chopsticks/**/extra.yaml
+chopsticks/**/.tmp.yaml

chopsticks/.nvmrc

@@ -0,0 +1 @@
+v20.11.0

chopsticks/.yarn/releases/yarn-4.5.0.cjs

@@ -0,0 +1,33 @@
+# KILT Chopsticks utilities
+
+This project contains setup code to spawn [Chopsticks](https://github.com/AcalaNetwork/chopsticks)-based deployments for both Spiritnet and Peregrine runtimes.
+
+## How to spawn
+
+First, `cd` into the `chipsticks` folder and run first `nvm use` to configure the right Node version, and then `yarn` to install any Node dependencies.
+
+To spin up a deployment using the actual state currently on Spiritnet or Peregrine, run either `yarn peregrine:spawn` or `yarn spiritnet:spawn`.
+
+### Add customizations
+
+By default, the config for each chain specifies only the endpoint to fetch the state from and the path to store the db folder.
+
+Any additional information, such as storage or WASM overrides, or port number specifications, can be specified using the same YAML format in each folder's `extra.yaml` file.
+These files are not tracked by git and are specific for each user's machine.
+If a chain does not include an `extra.yaml` file, the default config specified in the chain's `config.yaml` file will be used.
+
+A list of example configurations is given in the [examples](./examples/) folder.
+
+So, if for example the sudo key for the Peregrine network must be overridden, follow these steps:
+
+1. Create the file `runtimes/peregrine/kilt/extra.yaml`.
+2. Copy the content of [`storage.example.yaml`](./examples/storage.example.yaml) into `runtimes/peregrine/kilt/extra.yaml`.
+3. From within the `chopsticks` directory run `yarn peregrine:spawn`.
+
+This process can be applied to every chain folder, also for multiple within the same environment, e.g., the sudo key for PILT and the sudo key for Paseo can be overridden by having two `extra.yaml` files within each folder.
+
+## How it works
+
+When spawning a network, a temporary file called `.tmp.yaml` will be generated within each chain's folder.
+**THIS DOES NOT HAVE TO BE MANUALLY EDITED**, and it will be automatically cleaned up whenever the network spawning process is stopped.
+If, for any reason, this should not be the case, you can run `yarn peregrine:cleanup` or `yarn spiritnet:cleanup` to clean up these temporary files.

chopsticks/README.md

@@ -0,0 +1,5 @@
+port: 50002
+wasm-override: ../../../target/debug/wbuild/spiritnet-runtime/spiritnet_runtime.wasm
+import-storage:
+    Sudo:
+        Key: 15jSz35ugoWTc61xHPoxEkHte4o7UanKCk1gx1dizA8yuNs8

chopsticks/examples/complete.example.yaml

@@ -0,0 +1 @@
+port: 50002

chopsticks/examples/port.example.yaml

@@ -0,0 +1,3 @@
+import-storage:
+    Sudo:
+        Key: 15jSz35ugoWTc61xHPoxEkHte4o7UanKCk1gx1dizA8yuNs8

chopsticks/examples/storage.example.yaml

@@ -0,0 +1 @@
+wasm-override: ../../../target/debug/wbuild/spiritnet-runtime/spiritnet_runtime.wasm

chopsticks/examples/wasm.example.yaml

@@ -0,0 +1,28 @@
+{
+  "name": "@kiltprotocol/chopsticks-networks",
+  "license": "MIT",
+  "devDependencies": {
+    "@acala-network/chopsticks": "^0.15.0",
+    "@alexlafroscia/yaml-merge": "^4.1.0"
+  },
+  "scripts": {
+    "internal:peregrine:relay-setup": "yarn yaml-merge runtimes/peregrine/relay/config.yaml runtimes/peregrine/relay/extra.yaml > runtimes/peregrine/relay/.tmp.yaml",
+    "internal:peregrine:relay-teardown": "rm runtimes/peregrine/relay/.tmp.yaml",
+    "internal:peregrine:kilt-setup": "yarn yaml-merge runtimes/peregrine/kilt/config.yaml runtimes/peregrine/kilt/extra.yaml > runtimes/peregrine/kilt/.tmp.yaml",
+    "internal:peregrine:kilt-teardown": "rm runtimes/peregrine/kilt/.tmp.yaml",
+    "internal:peregrine:ah-setup": "yarn yaml-merge runtimes/peregrine/ah/config.yaml runtimes/peregrine/ah/extra.yaml > runtimes/peregrine/ah/.tmp.yaml",
+    "internal:peregrine:ah-teardown": "rm runtimes/peregrine/ah/.tmp.yaml",
+    "internal:spiritnet:relay-setup": "yarn yaml-merge runtimes/spiritnet/relay/config.yaml runtimes/spiritnet/relay/extra.yaml > runtimes/spiritnet/relay/.tmp.yaml",
+    "internal:spiritnet:relay-teardown": "rm runtimes/spiritnet/relay/.tmp.yaml",
+    "internal:spiritnet:kilt-setup": "yarn yaml-merge runtimes/spiritnet/kilt/config.yaml runtimes/spiritnet/kilt/extra.yaml > runtimes/spiritnet/kilt/.tmp.yaml",
+    "internal:spiritnet:kilt-teardown": "rm runtimes/spiritnet/kilt/.tmp.yaml",
+    "internal:spiritnet:ah-setup": "yarn yaml-merge runtimes/spiritnet/ah/config.yaml runtimes/spiritnet/ah/extra.yaml > runtimes/spiritnet/ah/.tmp.yaml",
+    "internal:spiritnet:ah-teardown": "rm runtimes/spiritnet/ah/.tmp.yaml",
+    "pre:peregrine:spawn": "yarn internal:peregrine:relay-setup && yarn internal:peregrine:kilt-setup && yarn internal:peregrine:ah-setup",
+    "peregrine:spawn": "bash -c 'trap \"echo \"\"Cleaning up temporary resources. Please wait a few seconds...\"\" && yarn peregrine:cleanup\" SIGINT; yarn pre:peregrine:spawn && yarn chopsticks xcm -r runtimes/peregrine/relay/.tmp.yaml -p runtimes/peregrine/kilt/.tmp.yaml -p runtimes/peregrine/ah/.tmp.yaml'",
+    "peregrine:cleanup": "yarn internal:peregrine:relay-teardown && yarn internal:peregrine:kilt-teardown && yarn internal:peregrine:ah-teardown",
+    "pre:spiritnet:spawn": "yarn internal:spiritnet:relay-setup && yarn internal:spiritnet:kilt-setup && yarn internal:spiritnet:ah-setup",
+    "spiritnet:spawn": "bash -c 'trap \"echo \"\"Cleaning up temporary resources. Please wait a few seconds...\"\" && yarn spiritnet:cleanup\" SIGINT; yarn pre:spiritnet:spawn && yarn chopsticks xcm -r runtimes/spiritnet/relay/.tmp.yaml -p runtimes/spiritnet/kilt/.tmp.yaml -p runtimes/spiritnet/ah/.tmp.yaml'",
+    "spiritnet:cleanup": "yarn internal:spiritnet:relay-teardown && yarn internal:spiritnet:kilt-teardown && yarn internal:spiritnet:ah-teardown"
+  }
+}

chopsticks/package.json

@@ -0,0 +1,4 @@
+endpoint:
+    - wss://sys.ibp.network/asset-hub-paseo
+    - wss://asset-hub-paseo.dotters.network
+db: db/peregrine/ah/db.sqlite

chopsticks/runtimes/peregrine/ah/config.yaml

@@ -0,0 +1,3 @@
+endpoint:
+    - wss://peregrine.kilt.io
+db: db/peregrine/kilt/db.sqlite

chopsticks/runtimes/peregrine/kilt/config.yaml

@@ -0,0 +1,4 @@
+endpoint:
+    - wss://rpc.ibp.network/paseo
+    - wss://paseo.dotters.network
+db: db/peregrine/relay/db.sqlite

chopsticks/runtimes/peregrine/relay/config.yaml

@@ -0,0 +1,4 @@
+endpoint:
+    - wss://sys.ibp.network/asset-hub-polkadot
+    - wss://asset-hub-polkadot.dotters.network
+db: db/spiritnet/ah/db.sqlite

chopsticks/runtimes/spiritnet/ah/config.yaml

@@ -0,0 +1,3 @@
+endpoint:
+    - wss://spiritnet.kilt.io
+db: db/spiritnet/kilt/db.sqlite

chopsticks/runtimes/spiritnet/kilt/config.yaml

@@ -0,0 +1,4 @@
+endpoint:
+    - wss://rpc.ibp.network/polkadot
+    - wss://polkadot.dotters.network
+db: db/spiritnet/relay/db.sqlite