Baedekker

.baedeker/README.md

@@ -0,0 +1,65 @@
+# What can baedeker do
+
+Baedeker is a tool for running live substrate based chains in miscellaneous configurations.
+
+## Testing runtime upgrades with local net
+
+### Create chain spec that contains live chain data
+
+In order to generate a chain spec (on a Linux machine):
+
+1. Have rust installed and configured to build the subtensor project 
+2. Install baedeker:
+
+```bash
+git clone https://github.com/UniqueNetwork/baedeker.git && cd baedeker
+cargo build –release
+sudo cp ./target/release/baedeker /usr/local/bin
+sudo chmod +x /usr/local/bin/baedeker
+```
+
+3. Run baedeker from project root to generate the chain spec from finney state:
+
+```bash
+sudo .baedeker/up-local.sh .baedeker/forkless-data.jsonnet --tla-str=forked_spec=subtensor --tla-str=fork_source=wss://entrypoint-finney.opentensor.ai
+```
+
+It will output the chain spec file to 
+
+```
+.baedeker/.bdk-env/specs/subtensor.json
+```
+
+and also it will create secrets for misnamed alice, bob, and charlie and write them to this folder:
+
+```
+.baedeker/.bdk-env/secrets
+```
+
+### Testing process
+
+In this process we test runtime upgrade from a chain running runtime build off commit A to commit B.
+
+1. Run baedeker to build the spec with current data (see above)
+   The baedeker generated files are .gitignored, so they will not be affected by further `git checkout` commands
+2. Checkout the commit A
+3. Build the binary for commit A
+4. Launch the local network with node binary, spec, and keys:
+
+```bash
+./scripts/localnet-baedeker.sh
+```
+Note: For launching the older versions of network that use Polkadot pre-1.0 version, add  --ws-port 9947 to localnet-baedeker.sh
+
+5. Stop the local network
+
+6. Edit localnet-baedeker.sh so that it doesn't delete existing chain data when started again
+
+7. Checkout the commit B that we want to upgrade the network to
+
+8. Build the binary
+
+9. Restart local network using edited localnet-baedeker.sh
+
+10. Do runtime upgrade
+

.baedeker/forkless-data.jsonnet

@@ -0,0 +1,41 @@
+local
+m = import 'baedeker-library/mixin/spec.libsonnet',
+rm = import 'baedeker-library/mixin/raw-spec.libsonnet',
+;
+
+function(relay_spec, forked_spec, fork_source)
+
+local relay = {
+	name: 'subtensor',
+	bin: 'bin/subtensor',
+	spec: {Raw:{
+		local modifyRaw = bdk.mixer([
+			rm.resetNetworking($),
+			rm.decodeSpec(),
+			rm.polkaLaunchPara($),
+			rm.reencodeSpec(),
+		]),
+		raw_spec: modifyRaw({
+			name: "Unused",
+			id: "%s_local" % forked_spec,
+			chainType: "Live",
+			codeSubstitutes: {},
+			genesis: {
+				raw: {
+					top: cql.chain(fork_source).latest._preloadKeys._raw,
+					childrenDefault: {},
+				},
+			},
+		}),
+	}},
+	nodes: {
+		[name]: {
+			bin: $.bin,
+			wantedKeys: 'standalone',
+		},
+		for name in ['alice', 'bob', 'charlie']
+	},
+};
+
+relay + {
+}

.baedeker/rewrites.jsonnet

@@ -0,0 +1,11 @@
+local dotenv = {
+    [std.splitLimit(line, "=", 2)[0]]: std.splitLimit(line, "=", 2)[1]
+    for line in std.split(importstr "../.env", "\n")
+    if line != ""
+    if std.member(line, "=")
+};
+
+function(prev, repoDir)
+(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({
+	'bin/subtensor':'%s/target/release/subtensor' % repoDir,
+})(prev)

.baedeker/up-local.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+BDK_DIR=$(dirname $(readlink -f "$0"))
+RUST_LOG=info baedeker --spec=docker -J$BDK_DIR/vendor/ --generator=docker_compose=$BDK_DIR/.bdk-env --generator=docker_compose_discover=$BDK_DIR/.bdk-env/discover.env --secret=file=$BDK_DIR/.bdk-env/secret  --tla-str=relay_spec=rococo-local --input-modules='lib:baedeker-library/ops/nginx.libsonnet' --input-modules='lib:baedeker-library/ops/devtools.libsonnet' --tla-str=repoDir=$(realpath $BDK_DIR/..) $@ $BDK_DIR/rewrites.jsonnet
+cd $BDK_DIR/.bdk-env
+
+# Change the network type to development
+sed -i 's#"chainType": "Live"#"chainType": "Development"#' ./specs/subtensor.json
+
+# Give Alice some balance
+sed -i 's#\"0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d\": \"0x1d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\"#\"0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d\": \"0x0000000000000000010000000000000000407a10f35a0000000000000000000000000000000000000000000000000000\"#' ./specs/subtensor.json
+
+#docker compose up -d --wait --remove-orphans

.baedeker/up.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+BDK_DIR=$(dirname $(readlink -f "$0"))
+RUST_LOG=info baedeker --spec=docker -J$BDK_DIR/vendor/ --generator=docker_compose=$BDK_DIR/.bdk-env --generator=docker_compose_discover=$BDK_DIR/.bdk-env/discover.env --secret=file=$BDK_DIR/.bdk-env/secret  --tla-str=relay_spec=rococo-local --input-modules='lib:baedeker-library/ops/nginx.libsonnet' --input-modules='lib:baedeker-library/ops/devtools.libsonnet' --tla-str=repoDir=$(realpath $BDK_DIR/..) $@ $BDK_DIR/rewrites.jsonnet
+cd $BDK_DIR/.bdk-env
+#docker compose up -d --wait --remove-orphans

.baedeker/vendor/baedeker-library/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Unique Network
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

.baedeker/vendor/baedeker-library/inputs/base.libsonnet

@@ -0,0 +1,54 @@
+local
+	genesisState = import '../util/genesisState.libsonnet',
+	{mixinAllChains, ...} = import '../util/mixin.libsonnet',
+	k = import '../mixin/keys.libsonnet',
+;
+
+function(prev, final)
+
+// :code
+local WELLKNOWN_CODE = '0x3a636f6465';
+
+local genesisMixin = {
+	// TODO: Process from wasm once native runtime free world lands.
+	specJson: cql.description('<build spec for %s>' % self.path, bdk.processSpec(self.bin, self.spec)),
+	genesisWasm: self.specJson.genesis.raw.top[WELLKNOWN_CODE],
+	genesisWasmData: cql.runtimeWasm(self.genesisWasm),
+	genesisStateVersion: self.genesisWasmData.version.state_version,
+	genesisHead: genesisState(self.specJson, self.genesisStateVersion),
+
+	ss58Format: super?.ss58Format ?? 42,
+	signatureSchema: super?.signatureSchema ?? 'Sr25519',
+	// FIXME: Try to guess from runtime metadata.
+	// If null - try to guess the schema.
+	// I.e use StashOf of pallet_staking, if staking presents in schema, and so on.
+	validatorIdAssignment: super?.validatorIdAssignment ?? 'none',
+
+	addressSeed(seed):: cql.addressSeed(self.signatureSchema, seed, self.ss58Format),
+};
+
+local mergedChains = (prev + mixinAllChains(prev, function(chain, path) genesisMixin + {
+	path: path,
+	nodes+: {
+		[nodename]+: local hostname = '%s-node-%s' % [path, nodename]; {
+			hostname: hostname,
+			wantedKeys:
+				if node?.wantedKeys == 'para' then k.paraWantedKeys($)
+				else if node?.wantedKeys == 'para-ed' then k.paraWantedKeys($, ed = true)
+				else if node?.wantedKeys == 'para-nimbus' then k.paraWantedKeys($, nimbus = true)
+				else if node?.wantedKeys == 'relay' then k.relayWantedKeys($)
+				else if node?.wantedKeys == 'standalone' then k.standaloneWantedKeys($)
+				else if std.isObject(node?.wantedKeys) then node?.wantedKeys
+				else if !('wantedKeys' in node) then {}
+				else error 'Unknown wantedKeys: %s' % node?.wantedKeys,
+		},
+		for [nodename, node] in (chain?.nodes ?? {})
+	},
+}));
+
+mergedChains + mixinAllChains(mergedChains, function(chain, path) {
+	nodes+: {
+		[nodename]+: bdk.ensureKeys(node.hostname, node.wantedKeys, chain.ss58Format),
+		for [nodename, node] in (chain?.nodes ?? {})
+	},
+})

.baedeker/vendor/baedeker-library/mixin/keys.libsonnet

@@ -0,0 +1,55 @@
+local
+needController({validatorIdAssignment, ...}) =
+	if validatorIdAssignment == 'none' || validatorIdAssignment == 'collatorSelection' then false
+	else if validatorIdAssignment == 'staking' then true
+	else error "unknown validatorIdAssignment: %s" % validatorIdAssignment,
+;
+
+{
+	relayWantedKeys(root): {
+		[if needController(root) then '_controller']: root.signatureSchema,
+		_stash: root.signatureSchema,
+
+		gran: 'Ed25519',
+		babe: 'Sr25519',
+		imon: 'Sr25519',
+		para: 'Sr25519',
+		asgn: 'Sr25519',
+		audi: 'Sr25519',
+		// rococo: beefy is required
+		beef: 'Ecdsa',
+
+		sessionKeys: {
+			grandpa: 'gran',
+			babe: 'babe',
+			im_online: 'imon',
+			authority_discovery: 'audi',
+			para_assignment: 'asgn',
+			para_validator: 'para',
+			beefy: 'beef',
+		},
+	},
+	paraWantedKeys(root, ed = false, nimbus = false): {
+		[if needController(root) then '_controller']: root.signatureSchema,
+		_stash: root.signatureSchema,
+
+		// COMPAT: asset-hub on polkadot uses ed25519 instead of sr25519 for session keys.
+		// https://github.com/paritytech/cumulus/blob/d4bb2215bb28ee05159c4c7df1b3435177b5bf4e/parachains/common/src/lib.rs#L57-L62
+		[if nimbus then 'nmbs' else 'aura']: if ed then 'Ed25519' else 'Sr25519',
+		// COMPAT: moonbeam only supports setting nimbus key in genesis, yet rand key is required.
+		[if nimbus then 'rand']: {alias: 'nmbs'},
+
+		sessionKeys: {
+			aura: 'aura',
+		},
+	},
+	standaloneWantedKeys(root): {
+		aura: 'Sr25519',
+		gran: 'Ed25519',
+
+		sessionKeys: {
+			aura: 'aura',
+			grandpa: 'gran',
+		},
+	},
+}