From d9bbfe8e589ab99fca2ccf6c903b2ac05e201eb0 Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Sat, 3 Aug 2024 01:00:34 +0800 Subject: [PATCH] refactor: initial attempt to make oura v1 compatible with pallas `0.29.0` --- .vscode/settings.json | 3 + Cargo.lock | 441 ++++++++++++++++++++++++++++++++++----- Cargo.toml | 3 +- src/mapper/babbage.rs | 8 +- src/mapper/byron.rs | 2 +- src/mapper/map.rs | 34 ++- src/model.rs | 2 +- src/pipelining.rs | 2 +- src/sources/common.rs | 67 +++--- src/sources/n2c/run.rs | 81 +++---- src/sources/n2c/setup.rs | 4 +- src/sources/n2n/run.rs | 20 +- 12 files changed, 525 insertions(+), 142 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..3b664107 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "git.ignoreLimitWarning": true +} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 92ebce5a..67a39709 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,6 +750,24 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bip39" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" +dependencies = [ + "bitcoin_hashes", + "rand_core", + "serde", + "unicode-normalization", +] + +[[package]] +name = "bitcoin_hashes" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90064b8dee6815a6470d60bad07bbbaee885c0e12d04177138fa3291a01b7bc4" + [[package]] name = "bitflags" version = "1.3.2" @@ -863,6 +881,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", + "serde", "windows-targets 0.48.5", ] @@ -892,7 +911,7 @@ dependencies = [ "bitflags 1.3.2", "clap_lex", "indexmap 1.9.3", - "strsim", + "strsim 0.10.0", "termcolor", "textwrap", ] @@ -1065,8 +1084,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -1079,21 +1108,46 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.41", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.41", +] + [[package]] name = "deranged" version = "0.3.10" @@ -1101,6 +1155,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -1135,6 +1190,15 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +[[package]] +name = "ed25519-bip32" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb588f93c0d91b2f668849fd6d030cddb0b2e31f105963be189da5acdf492a21" +dependencies = [ + "cryptoxide", +] + [[package]] name = "either" version = "1.9.0" @@ -1156,7 +1220,7 @@ dependencies = [ "rustc_version 0.2.3", "serde", "serde_json", - "serde_with", + "serde_with 1.14.0", "url", "void", ] @@ -1260,6 +1324,12 @@ dependencies = [ "flate2", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.28" @@ -1332,9 +1402,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-io" @@ -1472,7 +1542,7 @@ dependencies = [ "thiserror", "tokio", "tokio-retry", - "tonic", + "tonic 0.10.2", "tower", "tracing", ] @@ -1485,7 +1555,7 @@ checksum = "db8a478015d079296167e3f08e096dc99cffc2cb50fa203dd38aaa9dd37f8354" dependencies = [ "prost", "prost-types", - "tonic", + "tonic 0.10.2", ] [[package]] @@ -1529,9 +1599,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -1762,6 +1832,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1772,6 +1843,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -1830,6 +1902,24 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -1999,9 +2089,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.19.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7005aaf257a59ff4de471a9d5538ec868a21586534fff7f85dd97d4043a6139" +checksum = "9d15f4203d71fdf90903c2696e55426ac97a363c67b218488a73b534ce7aca10" dependencies = [ "half", "minicbor-derive", @@ -2045,6 +2135,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + [[package]] name = "murmur3" version = "0.5.2" @@ -2101,6 +2197,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -2111,11 +2213,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -2240,6 +2353,7 @@ dependencies = [ "strum", "strum_macros", "tokio", + "tokio-retry", "unicode-truncate", ] @@ -2262,27 +2376,31 @@ dependencies = [ [[package]] name = "pallas" -version = "0.18.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94ef55b690eac7ddf43a3e7ce10d4594866c34279c424aa2ce26d757789246da" +checksum = "4b55d8e8f1b4b84d85c3779ba1161ada42cad7fd9d6f73f7057fc6f5fd8a538a" dependencies = [ "pallas-addresses", "pallas-codec", + "pallas-configs", "pallas-crypto", - "pallas-miniprotocols", - "pallas-multiplexer", + "pallas-network", "pallas-primitives", "pallas-traverse", + "pallas-txbuilder", + "pallas-utxorpc", ] [[package]] name = "pallas-addresses" -version = "0.18.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db28c4050dea032d497555bc68c269ae8e691486d8ec83f02b090487da0d0be" +checksum = "d628ad58404ddd733e8fe46fe9986489b46258a2ab1bb7b1c4b8e406b91b7cff" dependencies = [ "base58", "bech32", + "crc", + "cryptoxide", "hex", "pallas-codec", "pallas-crypto", @@ -2291,63 +2409,71 @@ dependencies = [ [[package]] name = "pallas-codec" -version = "0.18.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6e03d05d42a663526d78c8b1d4f2554f09bbf4cc846e1a9e839c558bf6103c" +checksum = "da003a7360fa032b80d38b4a15573f885f412f2b3868772d49fb072197a9d5f9" dependencies = [ "hex", "minicbor", "serde", + "thiserror", ] [[package]] -name = "pallas-crypto" -version = "0.18.2" +name = "pallas-configs" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a35fc93b3613c0a628d0820f8d5d9a52709d795b59a1754a337aee0fca289dd" +checksum = "e6d5fe523f1054c98a6603d6ad773697165733896d9fba0c97aceb5ba2394f4b" dependencies = [ - "cryptoxide", + "base64 0.22.1", "hex", + "num-rational", + "pallas-addresses", "pallas-codec", - "rand_core", + "pallas-crypto", + "pallas-primitives", "serde", - "thiserror", + "serde_json", + "serde_with 3.9.0", ] [[package]] -name = "pallas-miniprotocols" -version = "0.18.2" +name = "pallas-crypto" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a4754676d92ae351ad524d98bc32d70835856ee0623a45288bb50a5ee4b161" +checksum = "c9248ed0e594bcb0f548393264519c7adea88874d8bd7cc86f894e8ba4e918c2" dependencies = [ + "cryptoxide", "hex", - "itertools", "pallas-codec", - "pallas-multiplexer", + "rand_core", + "serde", "thiserror", - "tracing", ] [[package]] -name = "pallas-multiplexer" -version = "0.18.2" +name = "pallas-network" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4b8c7f48cd3b41ce1fac3e829c723d2e76f1cee77b864446d65b2d12ee0aa3" +checksum = "d5c93a02d31d678cf0848ad2ec625f6e483b3eef290d844b55a5e954859e6206" dependencies = [ "byteorder", "hex", - "log", + "itertools 0.13.0", "pallas-codec", + "pallas-crypto", "rand", + "socket2 0.5.5", "thiserror", + "tokio", "tracing", ] [[package]] name = "pallas-primitives" -version = "0.18.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5fdf328f41971e0b1457e2377abeb09143fa50ab79f1a6a6ab5740bc94dc4b" +checksum = "c0fa55305212f7828651c8db024e1e286198c2fccb028bbb697c68990c044959" dependencies = [ "base58", "bech32", @@ -2361,15 +2487,64 @@ dependencies = [ [[package]] name = "pallas-traverse" -version = "0.18.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58c353ecb175a63422386c80301493db9fc448407bc63322534522579e22879" +checksum = "49459bd0d2ba86fd909890a81e6238eaf051952d7e38ad63195301e72e8f458e" dependencies = [ "hex", + "itertools 0.13.0", "pallas-addresses", "pallas-codec", "pallas-crypto", "pallas-primitives", + "paste", + "serde", + "thiserror", +] + +[[package]] +name = "pallas-txbuilder" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e9d19d7fc4651156e38de563e88a9e0cc0110d864f05fbf346b65f055aecab" +dependencies = [ + "hex", + "pallas-addresses", + "pallas-codec", + "pallas-crypto", + "pallas-primitives", + "pallas-traverse", + "pallas-wallet", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "pallas-utxorpc" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037f8687bf73baf702fb8fe832c5344601b0adcde196e13577a2cb4ab4f8b5f4" +dependencies = [ + "pallas-codec", + "pallas-crypto", + "pallas-primitives", + "pallas-traverse", + "utxorpc-spec", +] + +[[package]] +name = "pallas-wallet" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d1c2eeec4abc8e601ceba4cd71cb31bda7864c33a8713fc3c4bbc22c0aedb72" +dependencies = [ + "bech32", + "bip39", + "cryptoxide", + "ed25519-bip32", + "pallas-crypto", + "rand", "thiserror", ] @@ -2402,12 +2577,55 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pathdiff" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pbjson" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +dependencies = [ + "base64 0.21.5", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +dependencies = [ + "heck", + "itertools 0.11.0", + "prost", + "prost-types", +] + +[[package]] +name = "pbjson-types" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +dependencies = [ + "bytes", + "chrono", + "pbjson", + "pbjson-build", + "prost", + "prost-build", + "serde", +] + [[package]] name = "pem" version = "3.0.4" @@ -2424,6 +2642,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -2527,6 +2755,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn 2.0.41", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2596,6 +2834,27 @@ dependencies = [ "prost-derive", ] +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck", + "itertools 0.10.5", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.41", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.12.6" @@ -2603,7 +2862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.41", @@ -3082,7 +3341,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff" dependencies = [ "serde", - "serde_with_macros", + "serde_with_macros 1.5.2", +] + +[[package]] +name = "serde_with" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.1.0", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros 3.9.0", + "time", ] [[package]] @@ -3091,12 +3368,24 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn 1.0.109", ] +[[package]] +name = "serde_with_macros" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.41", +] + [[package]] name = "sha1" version = "0.6.1" @@ -3233,6 +3522,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.24.1" @@ -3369,12 +3664,13 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "num-conv", "powerfmt", "serde", "time-core", @@ -3389,10 +3685,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -3571,6 +3868,33 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.5", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -3723,6 +4047,21 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utxorpc-spec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea97da98fc3de55c146bf5c921f131d8bad3d72d49bec59254998de5a53932" +dependencies = [ + "bytes", + "futures-core", + "pbjson", + "pbjson-types", + "prost", + "serde", + "tonic 0.11.0", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 6bbf25d7..ce330781 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ authors = ["Santiago Carmuega "] [dependencies] -pallas = "0.18.2" +pallas = "0.29.0" # pallas = { git = "https://github.com/txpipe/pallas" } # pallas = { path = "../pallas/pallas" } hex = "0.4.3" @@ -30,6 +30,7 @@ strum = "0.24" strum_macros = "0.24" prometheus_exporter = { version = "0.8.5", default-features = false } unicode-truncate = "0.2.0" +tokio-retry = "0.3" # feature logs file-rotate = { version = "0.7.1", optional = true } diff --git a/src/mapper/babbage.rs b/src/mapper/babbage.rs index fda60d46..d8252de0 100644 --- a/src/mapper/babbage.rs +++ b/src/mapper/babbage.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; + use pallas::codec::utils::KeepRaw; use pallas::ledger::primitives::babbage::{ @@ -101,7 +103,11 @@ impl EventWriter { .into(); record.native_witnesses = self - .collect_native_witness_records(&witnesses.native_script)? + .collect_native_witness_records(&witnesses.native_script.as_ref().map(|x| { + x.iter() + .map(|ws| ws.deref().clone()) + .collect::>() + }))? .into(); record.plutus_witnesses = self diff --git a/src/mapper/byron.rs b/src/mapper/byron.rs index 2ed223d5..0986512b 100644 --- a/src/mapper/byron.rs +++ b/src/mapper/byron.rs @@ -168,7 +168,7 @@ impl EventWriter { hash: &Hash<32>, cbor: &[u8], ) -> Result { - let abs_slot = pallas::ledger::traverse::time::byron_epoch_slot_to_absolute( + let abs_slot = pallas::ledger::traverse::time::relative_slot_to_absolute( source.header.consensus_data.0.epoch, source.header.consensus_data.0.slot, ); diff --git a/src/mapper/map.rs b/src/mapper/map.rs index d05d20a0..7b0826c0 100644 --- a/src/mapper/map.rs +++ b/src/mapper/map.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use std::ops::Deref; +use pallas::codec::utils::Nullable; use pallas::ledger::primitives::alonzo::MintedWitnessSet; use pallas::ledger::primitives::babbage::MintedDatumOption; use pallas::ledger::traverse::{ComputeHash, OriginalHash}; @@ -68,19 +70,24 @@ fn relay_to_string(relay: &Relay) -> String { match relay { Relay::SingleHostAddr(port, ipv4, ipv6) => { let ip = match (ipv6, ipv4) { - (None, None) => "".to_string(), - (_, Some(x)) => ip_string_from_bytes(x.as_ref()), - (Some(x), _) => ip_string_from_bytes(x.as_ref()), + (Nullable::Null, Nullable::Null) => "".to_string(), + (_, Nullable::Some(x)) => ip_string_from_bytes(x.as_ref()), + (Nullable::Some(x), _) => ip_string_from_bytes(x.as_ref()), + (Nullable::Null, Nullable::Undefined) => "".to_string(), + (Nullable::Undefined, Nullable::Null) => "".to_string(), + (Nullable::Undefined, Nullable::Undefined) => "".to_string(), }; match port { - Some(port) => format!("{ip}:{port}"), - None => ip, + Nullable::Some(port) => format!("{ip}:{port}"), + Nullable::Null => ip, + Nullable::Undefined => ip, } } Relay::SingleHostName(port, host) => match port { - Some(port) => format!("{host}:{port}"), - None => host.clone(), + Nullable::Some(port) => format!("{host}:{port}"), + Nullable::Null => host.clone(), + Nullable::Undefined => host.clone(), }, Relay::MultiHostName(host) => host.clone(), } @@ -342,8 +349,11 @@ impl EventWriter { reward_account: reward_account.to_hex(), pool_owners: pool_owners.iter().map(|p| p.to_hex()).collect(), relays: relays.iter().map(relay_to_string).collect(), - pool_metadata: pool_metadata.as_ref().map(|m| m.url.clone()), - pool_metadata_hash: pool_metadata.as_ref().map(|m| m.hash.clone().to_hex()), + pool_metadata: pool_metadata.clone().map(|m| m.url.clone()).into(), + pool_metadata_hash: pool_metadata + .clone() + .map(|m| m.hash.clone().to_hex()) + .into(), }, Certificate::PoolRetirement(pool, epoch) => EventData::PoolRetirement { pool: pool.to_hex(), @@ -454,7 +464,11 @@ impl EventWriter { .into(); record.native_witnesses = self - .collect_native_witness_records(&witnesses.native_script)? + .collect_native_witness_records(&witnesses.native_script.as_ref().map(|x| { + x.iter() + .map(|ws| ws.deref().clone()) + .collect::>() + }))? .into(); record.plutus_witnesses = self diff --git a/src/model.rs b/src/model.rs index ad661bc3..105fd9f5 100644 --- a/src/model.rs +++ b/src/model.rs @@ -247,7 +247,7 @@ impl From for EventData { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct PlutusRedeemerRecord { pub purpose: String, - pub ex_units_mem: u32, + pub ex_units_mem: u64, pub ex_units_steps: u64, pub input_idx: u32, pub plutus_data: JsonValue, diff --git a/src/pipelining.rs b/src/pipelining.rs index 2ef43c6f..e7176c1e 100644 --- a/src/pipelining.rs +++ b/src/pipelining.rs @@ -1,4 +1,4 @@ -use std::thread::JoinHandle; +use tokio::task::JoinHandle; use crate::{model::Event, Error}; diff --git a/src/sources/common.rs b/src/sources/common.rs index 4687561e..feca85bc 100644 --- a/src/sources/common.rs +++ b/src/sources/common.rs @@ -5,13 +5,19 @@ use pallas::{ ledger::traverse::{probe, Era}, network::{ miniprotocols::{chainsync, Point, MAINNET_MAGIC, TESTNET_MAGIC}, - multiplexer::{bearers::Bearer, StdChannel, StdPlexer}, + multiplexer::{Bearer, Plexer}, }, }; +use tokio_retry::strategy::{jitter, ExponentialBackoff}; +use tokio_retry::Retry; + use serde::{de::Visitor, Deserializer}; use serde::{Deserialize, Serialize}; +use std::boxed::Box; +use std::error::Error as StdError; + use crate::{ mapper::EventWriter, utils::{retry, SwallowResult, Utils}, @@ -182,36 +188,43 @@ impl RetryPolicy { } } -pub fn setup_multiplexer_attempt(bearer: &BearerKind, address: &str) -> Result { +pub async fn setup_multiplexer_attempt( + bearer: &BearerKind, + address: &str, +) -> Result { match bearer { BearerKind::Tcp => { - let bearer = Bearer::connect_tcp(address)?; - Ok(StdPlexer::new(bearer)) + let bearer = Bearer::connect_tcp(address).await?; + Ok(Plexer::new(bearer)) } #[cfg(target_family = "unix")] BearerKind::Unix => { - let unix = Bearer::connect_unix(address)?; - Ok(StdPlexer::new(unix)) + let unix = Bearer::connect_unix(address).await?; + Ok(Plexer::new(unix)) } } } -pub fn setup_multiplexer( +pub async fn setup_multiplexer( bearer: &BearerKind, address: &str, retry: &Option, -) -> Result { +) -> Result { match retry { - Some(policy) => retry::retry_operation( - || setup_multiplexer_attempt(bearer, address), - &retry::Policy { - max_retries: policy.connection_max_retries, - backoff_unit: Duration::from_secs(1), - backoff_factor: 2, - max_backoff: Duration::from_secs(policy.connection_max_backoff as u64), - }, - ), - None => setup_multiplexer_attempt(bearer, address), + Some(policy) => { + let retry_strategy = + ExponentialBackoff::from_millis(1000) // backoff_unit + .factor(2) // backoff_factor + .max_delay(Duration::from_secs(policy.connection_max_backoff as u64)) // max_backoff + .map(jitter) // add jitter to delays + .take(policy.connection_max_retries.try_into().unwrap()); // max_retries + + Retry::spawn(retry_strategy, || async { + setup_multiplexer_attempt(bearer, address).await + }) + .await + } + None => setup_multiplexer_attempt(bearer, address).await, } } @@ -262,8 +275,8 @@ pub fn should_finalize( false } -pub(crate) fn intersect_starting_point( - client: &mut chainsync::Client, +pub(crate) async fn intersect_starting_point( + client: &mut chainsync::Client, intersect_arg: &Option, since_arg: &Option, utils: &Utils, @@ -277,7 +290,7 @@ where Some(cursor) => { log::info!("found persisted cursor, will use as starting point"); let desired = cursor.try_into()?; - let (point, _) = client.find_intersect(vec![desired])?; + let (point, _) = client.find_intersect(vec![desired]).await?; Ok(point) } @@ -286,14 +299,14 @@ where log::info!("found 'fallbacks' intersect argument, will use as starting point"); let options: Result, _> = x.iter().map(|x| x.clone().try_into()).collect(); - let (point, _) = client.find_intersect(options?)?; + let (point, _) = client.find_intersect(options?).await?; Ok(point) } Some(IntersectArg::Origin) => { log::info!("found 'origin' intersect argument, will use as starting point"); - let point = client.intersect_origin()?; + let point = client.intersect_origin().await?; Ok(Some(point)) } @@ -301,14 +314,14 @@ where log::info!("found 'point' intersect argument, will use as starting point"); let options = vec![x.clone().try_into()?]; - let (point, _) = client.find_intersect(options)?; + let (point, _) = client.find_intersect(options).await?; Ok(point) } Some(IntersectArg::Tip) => { log::info!("found 'tip' intersect argument, will use as starting point"); - let point = client.intersect_tip()?; + let point = client.intersect_tip().await?; Ok(Some(point)) } @@ -318,14 +331,14 @@ where log::warn!("`since` value is deprecated, please use `intersect`"); let options = vec![x.clone().try_into()?]; - let (point, _) = client.find_intersect(options)?; + let (point, _) = client.find_intersect(options).await?; Ok(point) } None => { log::info!("no starting point specified, will use tip of chain"); - let point = client.intersect_tip()?; + let point = client.intersect_tip().await?; Ok(Some(point)) } diff --git a/src/sources/n2c/run.rs b/src/sources/n2c/run.rs index a6aef999..fa5ee975 100644 --- a/src/sources/n2c/run.rs +++ b/src/sources/n2c/run.rs @@ -4,9 +4,10 @@ use pallas::{ ledger::traverse::MultiEraBlock, network::{ miniprotocols::{chainsync, handshake, Point, MAINNET_MAGIC}, - multiplexer::StdChannel, + multiplexer::AgentChannel, }, }; +use tokio_retry::{strategy::{jitter, ExponentialBackoff}, Retry}; use crate::{ mapper::EventWriter, @@ -121,10 +122,10 @@ impl ChainObserver { Ok(()) } - fn on_next_message( + async fn on_next_message( &mut self, msg: chainsync::NextResponse, - client: &mut chainsync::N2CClient, + client: &mut chainsync::N2CClient, ) -> Result { match msg { chainsync::NextResponse::RollForward(c, t) => match self.on_roll_forward(c, &t) { @@ -138,16 +139,17 @@ impl ChainObserver { chainsync::NextResponse::Await => { let next = client .recv_while_must_reply() + .await .map_err(|x| AttemptError::Recoverable(x.into()))?; - self.on_next_message(next, client) + Box::pin(self.on_next_message(next, client)).await } } } } -fn observe_forever( - mut client: chainsync::N2CClient, +async fn observe_forever( + mut client: chainsync::N2CClient, event_writer: EventWriter, min_depth: usize, finalize_config: Option, @@ -162,8 +164,8 @@ fn observe_forever( }; loop { - match client.request_next() { - Ok(next) => match observer.on_next_message(next, &mut client) { + match client.request_next().await { + Ok(next) => match observer.on_next_message(next, &mut client).await { Ok(Continuation::Proceed) => (), Ok(Continuation::DropOut) => break Ok(()), Err(err) => break Err(err), @@ -179,11 +181,11 @@ enum AttemptError { Other(Error), } -fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { +async fn do_handshake(channel: AgentChannel, magic: u64) -> Result<(), AttemptError> { let mut client = handshake::N2CClient::new(channel); let versions = handshake::n2c::VersionTable::v1_and_above(magic); - match client.handshake(versions) { + match client.handshake(versions).await { Ok(confirmation) => match confirmation { handshake::Confirmation::Accepted(_, _) => Ok(()), _ => Err(AttemptError::Other( @@ -194,7 +196,7 @@ fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { } } -fn do_chainsync_attempt( +async fn do_chainsync_attempt( config: &super::Config, utils: Arc, output_tx: &StageSender, @@ -205,15 +207,15 @@ fn do_chainsync_attempt( }; let mut plexer = setup_multiplexer(&config.address.0, &config.address.1, &config.retry_policy) + .await .map_err(AttemptError::Recoverable)?; - let hs_channel = plexer.use_channel(0); - let cs_channel = plexer.use_channel(5); + let hs_channel = plexer.subscribe_server(0); + let cs_channel = plexer.subscribe_client(5); - plexer.muxer.spawn(); - plexer.demuxer.spawn(); + plexer.spawn(); - do_handshake(hs_channel, magic)?; + do_handshake(hs_channel, magic).await?; let mut client = chainsync::N2CClient::new(cs_channel); @@ -224,6 +226,7 @@ fn do_chainsync_attempt( &config.since, &utils, ) + .await .map_err(AttemptError::Recoverable)?; if intersection.is_none() { @@ -236,18 +239,36 @@ fn do_chainsync_attempt( let writer = EventWriter::new(output_tx.clone(), utils, config.mapper.clone()); - observe_forever(client, writer, config.min_depth, config.finalize.clone())?; + observe_forever(client, writer, config.min_depth, config.finalize.clone()).await?; Ok(()) } -pub fn do_chainsync( +pub async fn do_chainsync( config: &super::Config, utils: Arc, output_tx: StageSender, ) -> Result<(), Error> { - retry::retry_operation( - || match do_chainsync_attempt(config, utils.clone(), &output_tx) { + let retry_strategy = ExponentialBackoff::from_millis(1000) // backoff_unit in millis + .factor(2) // backoff_factor + .max_delay( + config + .retry_policy + .as_ref() + .map(|x| Duration::from_secs(x.chainsync_max_backoff as u64)) + .unwrap_or_else(|| Duration::from_secs(60)), + ) // max_backoff + .map(jitter) // add jitter to delays + .take( + config + .retry_policy + .as_ref() + .map(|x| x.chainsync_max_retries) + .unwrap_or(50).try_into().unwrap(), + ); // max_retries + + Retry::spawn(retry_strategy, || async { + match do_chainsync_attempt(config, utils.clone(), &output_tx).await { Ok(()) => Ok(()), Err(AttemptError::Other(msg)) => { log::error!("N2C error: {}", msg); @@ -255,21 +276,7 @@ pub fn do_chainsync( Ok(()) } Err(AttemptError::Recoverable(err)) => Err(err), - }, - &retry::Policy { - max_retries: config - .retry_policy - .as_ref() - .map(|x| x.chainsync_max_retries) - .unwrap_or(50), - backoff_unit: Duration::from_secs(1), - backoff_factor: 2, - max_backoff: config - .retry_policy - .as_ref() - .map(|x| x.chainsync_max_backoff as u64) - .map(Duration::from_secs) - .unwrap_or_else(|| Duration::from_secs(60)), - }, - ) + } + }) + .await } diff --git a/src/sources/n2c/setup.rs b/src/sources/n2c/setup.rs index c1bc5c39..6cfaf395 100644 --- a/src/sources/n2c/setup.rs +++ b/src/sources/n2c/setup.rs @@ -51,11 +51,11 @@ impl SourceProvider for WithUtils { let config = self.inner.clone(); let utils = self.utils.clone(); - let handle = std::thread::spawn(move || { + let handle = tokio::spawn(async move { do_chainsync(&config, utils, output_tx) + .await .expect("chainsync process fails after max retries") }); - Ok((handle, output_rx)) } } diff --git a/src/sources/n2n/run.rs b/src/sources/n2n/run.rs index f2b83267..5c450ebc 100644 --- a/src/sources/n2n/run.rs +++ b/src/sources/n2n/run.rs @@ -2,7 +2,7 @@ use std::{fmt::Debug, ops::Deref, sync::Arc, time::Duration}; use pallas::network::{ miniprotocols::{blockfetch, chainsync, handshake, Point, MAINNET_MAGIC}, - multiplexer::StdChannel, + multiplexer::AgentChannel, }; use std::sync::mpsc::{Receiver, SyncSender}; @@ -113,7 +113,7 @@ impl ChainObserver { fn on_next_message( &mut self, msg: chainsync::NextResponse, - client: &mut chainsync::N2NClient, + client: &mut chainsync::N2NClient, ) -> Result { match msg { chainsync::NextResponse::RollForward(c, t) => match self.on_roll_forward(c, &t) { @@ -136,7 +136,7 @@ impl ChainObserver { } pub(crate) fn fetch_blocks_forever( - mut client: blockfetch::Client, + mut client: blockfetch::Client, event_writer: EventWriter, input: Receiver, ) -> Result<(), Error> { @@ -152,7 +152,7 @@ pub(crate) fn fetch_blocks_forever( } fn observe_headers_forever( - mut client: chainsync::N2NClient, + mut client: chainsync::N2NClient, event_writer: EventWriter, block_requests: SyncSender, min_depth: usize, @@ -185,11 +185,11 @@ enum AttemptError { Other(Error), } -fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { +async fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { let mut client = handshake::N2NClient::new(channel); - let versions = handshake::n2n::VersionTable::v4_and_above(magic); + let versions = handshake::n2n::VersionTable::v11_and_above(magic); - match client.handshake(versions) { + match client.handshake(versions).await { Ok(confirmation) => match confirmation { handshake::Confirmation::Accepted(_, _) => Ok(()), _ => Err(AttemptError::Other( @@ -200,7 +200,7 @@ fn do_handshake(channel: StdChannel, magic: u64) -> Result<(), AttemptError> { } } -fn do_chainsync_attempt( +async fn do_chainsync_attempt( config: &super::Config, utils: Arc, output_tx: &StageSender, @@ -220,7 +220,7 @@ fn do_chainsync_attempt( plexer.muxer.spawn(); plexer.demuxer.spawn(); - do_handshake(hs_channel, magic)?; + do_handshake(hs_channel, magic).await?; let mut cs_client = chainsync::N2NClient::new(cs_channel); @@ -265,7 +265,7 @@ fn do_chainsync_attempt( Ok(()) } -pub fn do_chainsync( +pub async fn do_chainsync( config: &super::Config, utils: Arc, output_tx: StageSender,