From 264ada2fa7c6d056fc9381a6c57da38db578ca1e Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Thu, 20 Jun 2024 13:48:41 +0800 Subject: [PATCH 1/6] initial implementation for mithril support --- Cargo.lock | 1134 ++++++++++++++++++++++++++--- Cargo.toml | 8 +- examples/dolos_source/daemon.toml | 2 +- examples/mithril/daemon.toml | 12 + src/sources/mithril.rs | 256 +++++++ src/sources/mod.rs | 18 + 6 files changed, 1333 insertions(+), 97 deletions(-) create mode 100644 examples/mithril/daemon.toml create mode 100644 src/sources/mithril.rs diff --git a/Cargo.lock b/Cargo.lock index 329d726a..318e3570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -322,6 +322,17 @@ dependencies = [ "reactor-trait", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -415,7 +426,7 @@ dependencies = [ "fastrand 2.0.1", "hex", "http 0.2.11", - "hyper", + "hyper 0.14.28", "ring 0.17.8", "time", "tokio", @@ -452,7 +463,7 @@ dependencies = [ "bytes", "fastrand 2.0.1", "http 0.2.11", - "http-body", + "http-body 0.4.6", "percent-encoding", "pin-project-lite", "tracing", @@ -502,7 +513,7 @@ dependencies = [ "aws-types", "bytes", "http 0.2.11", - "http-body", + "http-body 0.4.6", "once_cell", "percent-encoding", "regex-lite", @@ -652,7 +663,7 @@ dependencies = [ "crc32fast", "hex", "http 0.2.11", - "http-body", + "http-body 0.4.6", "md-5", "pin-project-lite", "sha1 0.10.6", @@ -684,7 +695,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.11", - "http-body", + "http-body 0.4.6", "once_cell", "percent-encoding", "pin-project-lite", @@ -723,10 +734,10 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand 2.0.1", - "h2", + "h2 0.3.24", "http 0.2.11", - "http-body", - "hyper", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -764,7 +775,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.11", - "http-body", + "http-body 0.4.6", "itoa 1.0.10", "num-integer", "pin-project-lite", @@ -812,8 +823,8 @@ dependencies = [ "bytes", "futures-util", "http 0.2.11", - "http-body", - "hyper", + "http-body 0.4.6", + "hyper 0.14.28", "itoa 1.0.10", "matchit", "memchr", @@ -822,7 +833,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower", "tower-layer", "tower-service", @@ -838,13 +849,19 @@ dependencies = [ "bytes", "futures-util", "http 0.2.11", - "http-body", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", "tower-service", ] +[[package]] +name = "az" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" + [[package]] name = "backtrace" version = "0.3.69" @@ -860,6 +877,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base16ct" version = "0.1.1" @@ -918,6 +944,22 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bech32" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" + +[[package]] +name = "binary-layout" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5845e3504cf59b9588fff324710f27ee519515b8a8a9f1207042da9a9e64f819" +dependencies = [ + "doc-comment", + "paste", +] + [[package]] name = "bincode" version = "1.3.3" @@ -934,7 +976,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" dependencies = [ "bitcoin_hashes", - "rand_core", + "rand_core 0.6.4", "serde", "unicode-normalization", ] @@ -960,6 +1002,26 @@ dependencies = [ "serde", ] +[[package]] +name = "blake2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +dependencies = [ + "crypto-mac", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -994,6 +1056,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "blst" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62dc83a094a71d43eeadd254b1ec2d24cb6a0bb6cadce00df51f0db594711a32" +dependencies = [ + "cc", + "glob", + "threadpool", + "zeroize", +] + [[package]] name = "bumpalo" version = "3.15.3" @@ -1120,7 +1194,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da6bc11b07529f16944307272d5bd9b22530bc7d05751717c9d416586cedab49" dependencies = [ - "heck", + "heck 0.4.1", "indexmap 1.9.3", "log", "proc-macro2", @@ -1155,15 +1229,44 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", - "winapi", + "wasm-bindgen", + "windows-targets 0.52.4", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half 2.4.1", ] [[package]] @@ -1176,6 +1279,15 @@ dependencies = [ "inout", ] +[[package]] +name = "ckb-merkle-mountain-range" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d15193decfa1e0b151ce19e42d118db048459a27720fb3de7d3103c30adccb12" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "clap" version = "4.5.1" @@ -1204,7 +1316,7 @@ version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.52", @@ -1539,6 +1651,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -1546,7 +1664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -1557,7 +1675,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1571,12 +1689,49 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "cryptoxide" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "382ce8820a5bb815055d3553a610e8cb542b2d767bbacea99038afda96cd760d" +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version 0.4.0", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "darling" version = "0.13.4" @@ -1696,6 +1851,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.7" @@ -1785,6 +1949,17 @@ dependencies = [ "signature 1.6.4", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8 0.10.2", + "serde", + "signature 2.2.0", +] + [[package]] name = "ed25519-bip32" version = "0.4.1" @@ -1794,6 +1969,21 @@ dependencies = [ "cryptoxide", ] +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.10.0" @@ -1814,7 +2004,7 @@ dependencies = [ "dyn-clone", "lazy_static", "percent-encoding", - "reqwest", + "reqwest 0.11.24", "rustc_version 0.2.3", "serde", "serde_json", @@ -1832,12 +2022,12 @@ dependencies = [ "base16ct", "crypto-bigint 0.4.9", "der 0.6.1", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", "pkcs8 0.9.0", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1890,6 +2080,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-serde" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" +dependencies = [ + "serde", + "typeid", +] + [[package]] name = "errno" version = "0.3.8" @@ -2071,10 +2271,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "file-rotate" version = "0.7.5" @@ -2085,12 +2291,36 @@ dependencies = [ "flate2", ] +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "windows-sys 0.52.0", +] + [[package]] name = "finl_unicode" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +[[package]] +name = "fixed" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc715d38bea7b5bf487fcd79bcf8c209f0b58014f3018a7a19c2b855f472048" +dependencies = [ + "az", + "bytemuck", + "half 2.4.1", + "typenum", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -2127,6 +2357,7 @@ checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ "futures-core", "futures-sink", + "nanorand", "spin 0.9.8", ] @@ -2374,8 +2605,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2395,6 +2628,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "gmp-mpfr-sys" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0205cd82059bc63b63cf516d714352a30c44f2c74da9961dfda2617ae6b5918" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "google-cloud-auth" version = "0.11.0" @@ -2407,7 +2650,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "thiserror", @@ -2462,7 +2705,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96e4ad0802d3f416f62e7ce01ac1460898ee0efc98f8b45cd4aab7611607012f" dependencies = [ - "reqwest", + "reqwest 0.11.24", "thiserror", "tokio", ] @@ -2501,7 +2744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2524,12 +2767,41 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.0.0", + "indexmap 2.2.5", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if 1.0.0", + "crunchy", +] + [[package]] name = "handlebars" version = "5.1.0" @@ -2587,6 +2859,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -2623,7 +2901,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -2668,6 +2946,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.8.0" @@ -2696,9 +2997,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.24", "http 0.2.11", - "http-body", + "http-body 0.4.6", "httparse", "httpdate", "itoa 1.0.10", @@ -2710,6 +3011,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "itoa 1.0.10", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.23.2" @@ -2717,7 +3038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http 0.2.11", - "hyper", + "hyper 0.14.28", "log", "rustls 0.20.9", "rustls-native-certs 0.6.3", @@ -2733,7 +3054,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.11", - "hyper", + "hyper 0.14.28", "log", "rustls 0.21.10", "rustls-native-certs 0.6.3", @@ -2741,13 +3062,30 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.0.0", + "hyper 1.3.1", + "hyper-util", + "rustls 0.23.7", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2760,12 +3098,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2865,6 +3239,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "inventory" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" + [[package]] name = "io-extras" version = "0.18.1" @@ -2909,6 +3289,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "itertools" version = "0.10.5" @@ -3010,6 +3396,20 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "kes-summed-ed25519" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb27d940b150ff718b407f94ff443915c4bbf938cc813b0164c827b170738fc" +dependencies = [ + "blake2 0.9.2", + "ed25519-dalek", + "rand_core 0.5.1", + "serde", + "serde_with 2.3.3", + "zeroize", +] + [[package]] name = "lapin" version = "2.3.1" @@ -3175,7 +3575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if 1.0.0", - "digest", + "digest 0.10.7", ] [[package]] @@ -3224,6 +3624,37 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "miette" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if 1.0.0", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "mime" version = "0.3.17" @@ -3246,7 +3677,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d15f4203d71fdf90903c2696e55426ac97a363c67b218488a73b534ce7aca10" dependencies = [ - "half", + "half 1.8.3", "minicbor-derive", ] @@ -3288,6 +3719,110 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mithril-build-script" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dec2e508663d762d6eed9d387678b551757ef689a3300b0f0118061dd6acdba" +dependencies = [ + "semver 1.0.22", + "serde_json", + "serde_yaml", +] + +[[package]] +name = "mithril-client" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb5936ebaae5752e2331e66eaa1a87d871ae62f5f35814bc8e5b6e718a6f3e3" +dependencies = [ + "anyhow", + "async-recursion", + "async-trait", + "chrono", + "flate2", + "flume 0.11.0", + "futures", + "getrandom", + "mithril-common", + "reqwest 0.12.5", + "semver 1.0.22", + "serde", + "serde_json", + "slog", + "strum 0.26.2", + "tar", + "thiserror", + "tokio", + "uuid", + "zstd 0.13.1", +] + +[[package]] +name = "mithril-common" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3ec5c2bfc4b6f287d6ea3061a58c91e701b4abc5b45a92a4182241a64a3e47" +dependencies = [ + "anyhow", + "async-trait", + "bech32 0.11.0", + "blake2 0.10.6", + "chrono", + "ciborium", + "ckb-merkle-mountain-range", + "digest 0.10.7", + "ed25519-dalek", + "fixed", + "hex", + "kes-summed-ed25519", + "mithril-build-script", + "mithril-stm", + "nom", + "pallas-addresses", + "pallas-codec", + "pallas-hardano", + "pallas-network", + "pallas-primitives", + "pallas-traverse", + "rand_chacha", + "rand_core 0.6.4", + "rayon", + "semver 1.0.22", + "serde", + "serde_bytes", + "serde_json", + "serde_with 3.8.1", + "serde_yaml", + "sha2", + "slog", + "strum 0.26.2", + "thiserror", + "tokio", + "typetag", + "walkdir", + "wasm-bindgen", +] + +[[package]] +name = "mithril-stm" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26d467cd609dff0d2b6253ff021c0d620fa605c21d7d0e5890270f1576d6356" +dependencies = [ + "blake2 0.10.6", + "blst", + "digest 0.10.7", + "num-bigint", + "num-rational", + "num-traits", + "rand_core 0.6.4", + "rayon", + "rug", + "serde", + "thiserror", +] + [[package]] name = "multimap" version = "0.8.3" @@ -3300,6 +3835,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -3482,6 +4026,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.64" @@ -3547,7 +4097,7 @@ dependencies = [ "aws-sdk-s3", "aws-sdk-sqs", "aws-types", - "bech32", + "bech32 0.9.1", "clap", "config", "crossterm", @@ -3564,22 +4114,25 @@ dependencies = [ "handlebars", "hex", "indicatif", + "itertools 0.12.1", "jsonwebtoken", "kafka", "lapin", "merge", + "miette", + "mithril-client", "murmur3", "net2", "openssl", "pallas", "r2d2_redis", "regex", - "reqwest", + "reqwest 0.11.24", "serde", "serde_json", "sqlx", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.25.3", "thiserror", "tokio", "tonic 0.11.0", @@ -3601,6 +4154,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caff54706df99d2a78a5a4e3455ff45448d81ef1bb63c22cd14052ca0e993a3f" + [[package]] name = "p12" version = "0.6.3" @@ -3639,6 +4198,7 @@ dependencies = [ "pallas-codec", "pallas-configs", "pallas-crypto", + "pallas-hardano", "pallas-network", "pallas-primitives", "pallas-traverse", @@ -3653,7 +4213,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48a8c469b0328e61398c6ff65dce717acf664a5e1f15a788b7d8f0efc10f3153" dependencies = [ "base58", - "bech32", + "bech32 0.9.1", "crc", "cryptoxide", "hex", @@ -3701,11 +4261,25 @@ dependencies = [ "cryptoxide", "hex", "pallas-codec", - "rand_core", + "rand_core 0.6.4", "serde", "thiserror", ] +[[package]] +name = "pallas-hardano" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b2fad14a720b4f1248de270d46a2467ca431e447cc0c09b7b44e96ad88bd5a" +dependencies = [ + "binary-layout", + "pallas-network", + "pallas-traverse", + "tap", + "thiserror", + "tracing", +] + [[package]] name = "pallas-network" version = "0.27.0" @@ -3731,7 +4305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0476fed841261c33a0fcba321bd69961c3d392eba50821dc0e64808439c71aa" dependencies = [ "base58", - "bech32", + "bech32 0.9.1", "hex", "log", "pallas-codec", @@ -3793,7 +4367,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b326ca34ee0d4ffff0199375be943b0cc5a78c4eb6c86d2a0bdd15611fd657eb" dependencies = [ - "bech32", + "bech32 0.9.1", "bip39", "cryptoxide", "ed25519-bip32", @@ -3859,7 +4433,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" dependencies = [ - "heck", + "heck 0.4.1", "itertools 0.11.0", "prost 0.12.3", "prost-types 0.12.3", @@ -4171,7 +4745,7 @@ dependencies = [ "base64 0.13.1", "env_logger 0.9.3", "http 0.2.11", - "hyper", + "hyper 0.14.28", "hyper-rustls 0.23.2", "log", "num", @@ -4207,8 +4781,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", - "heck", - "itertools 0.10.5", + "heck 0.4.1", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -4242,7 +4816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.52", @@ -4313,7 +4887,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4323,9 +4897,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" + [[package]] name = "rand_core" version = "0.6.4" @@ -4481,35 +5061,79 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.24", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-tls 0.5.0", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ - "async-compression", - "base64 0.21.7", + "base64 0.22.0", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http 0.2.11", - "http-body", - "hyper", - "hyper-tls", + "h2 0.4.5", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-rustls 0.27.2", + "hyper-tls 0.6.0", + "hyper-util", "ipnet", "js-sys", "log", "mime", - "mime_guess", "native-tls", "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 1.0.4", + "rustls-pemfile 2.1.2", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "system-configuration", "tokio", "tokio-native-tls", @@ -4518,8 +5142,9 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -4592,19 +5217,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" dependencies = [ "const-oid", - "digest", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", "pkcs8 0.10.2", - "rand_core", + "rand_core 0.6.4", "signature 2.2.0", "spki 0.7.3", "subtle", "zeroize", ] +[[package]] +name = "rug" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8df4099c6fa90a1a7f5ddc0c7fba50991080fa2084d5a78808a5a3cab406bb9" +dependencies = [ + "az", + "gmp-mpfr-sys", + "libc", + "libm", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -4702,6 +5339,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls" +version = "0.23.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebbbdb961df0ad3f2652da8f3fdc4b36122f568f968f45ad3316f26c025c677b" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "subtle", + "zeroize", +] + [[package]] name = "rustls-connector" version = "0.18.5" @@ -4807,6 +5457,15 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.23" @@ -4908,6 +5567,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.197" @@ -4921,9 +5589,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa 1.0.10", "ryu", @@ -4961,6 +5629,22 @@ dependencies = [ "serde_with_macros 1.5.2", ] +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap 1.9.3", + "serde", + "serde_json", + "serde_with_macros 2.3.3", + "time", +] + [[package]] name = "serde_with" version = "3.8.1" @@ -4991,6 +5675,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling 0.20.9", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "serde_with_macros" version = "3.8.1" @@ -5003,6 +5699,19 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.2.5", + "itoa 1.0.10", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.6.1" @@ -5020,7 +5729,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -5037,7 +5746,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -5094,8 +5803,8 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest", - "rand_core", + "digest 0.10.7", + "rand_core 0.6.4", ] [[package]] @@ -5104,8 +5813,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", - "rand_core", + "digest 0.10.7", + "rand_core 0.6.4", ] [[package]] @@ -5135,12 +5844,24 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + [[package]] name = "smallvec" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "snap" version = "1.1.1" @@ -5294,7 +6015,7 @@ checksum = "0a4ee1e104e00dedb6aa5ffdd1343107b0a4702e862a84320ee7cc74782d96fc" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.4.1", "hex", "once_cell", "proc-macro2", @@ -5324,7 +6045,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -5456,13 +6177,35 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +[[package]] +name = "strum" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum_macros" version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ - "heck", + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.52", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", "proc-macro2", "quote", "rustversion", @@ -5475,6 +6218,27 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "supports-color" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9829b314621dfc575df4e409e79f9d6a66a3bd707ab73f23cb4aa3a854ac854f" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0a1e5168041f5f3ff68ff7d95dcb9c8749df29f6e7e89ada40dd4c9de404ee" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "1.0.109" @@ -5503,6 +6267,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "system-configuration" version = "0.5.1" @@ -5540,6 +6310,23 @@ dependencies = [ "winx", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.14" @@ -5579,6 +6366,27 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix 0.38.31", + "windows-sys 0.48.0", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.57" @@ -5609,6 +6417,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.3.34" @@ -5657,9 +6474,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -5686,9 +6503,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -5748,6 +6565,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.7", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -5841,10 +6669,10 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "h2", + "h2 0.3.24", "http 0.2.11", - "http-body", - "hyper", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -5871,10 +6699,10 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", + "h2 0.3.24", "http 0.2.11", - "http-body", - "hyper", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -6002,12 +6830,42 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typeid" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" + [[package]] name = "typenum" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "typetag" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661d18414ec032a49ece2d56eee03636e43c4e8d577047ab334c0ba892e29aaf" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "ucd-trie" version = "0.1.6" @@ -6035,6 +6893,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.22" @@ -6077,6 +6941,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.7.1" @@ -6189,6 +7059,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -6331,6 +7211,19 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.118.2" @@ -6427,7 +7320,7 @@ dependencies = [ "sha2", "toml 0.5.11", "windows-sys 0.52.0", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -6693,7 +7586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81b149b61bd1402bcd5d456c616302812f8bebd65c56f720cefd86ab6cf5c8d8" dependencies = [ "anyhow", - "heck", + "heck 0.4.1", "indexmap 2.2.5", "wit-parser", ] @@ -6813,7 +7706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea274a806c3eeef5008d32881a999065591c646f0f889ca07fd1223f54378e8b" dependencies = [ "anyhow", - "heck", + "heck 0.4.1", "proc-macro2", "quote", "shellexpand", @@ -7049,6 +7942,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "winx" version = "0.36.3" @@ -7088,6 +7991,17 @@ dependencies = [ "wast 35.0.2", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys 0.4.13", + "rustix 0.38.31", +] + [[package]] name = "xmlparser" version = "0.13.6" @@ -7134,6 +8048,20 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] [[package]] name = "zstd" @@ -7141,7 +8069,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe 7.1.0", ] [[package]] @@ -7154,11 +8091,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index f433c42e..47b75288 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,11 +18,12 @@ gcp = ["google-cloud-pubsub", "google-cloud-googleapis", "google-cloud-default", rabbitmq = ["lapin"] redis = ["r2d2_redis"] u5c = ["tonic"] +mithril = ["mithril-client"] # elasticsearch = auto feature flag # kafka = auto feature flag [dependencies] -pallas = "0.27.0" +pallas = { version = "0.27.0", features = ["hardano"] } # pallas = { path = "../pallas/pallas" } # pallas = { git = "https://github.com/txpipe/pallas" } @@ -49,7 +50,7 @@ strum = "0.24" strum_macros = "0.25" unicode-truncate = "0.2.0" thiserror = "1.0.39" -indicatif = "0.17.3" +indicatif = "0.17.8" tracing = "0.1.37" tracing-subscriber = "0.3.17" anyhow = "1.0.77" @@ -77,3 +78,6 @@ aws-sdk-s3 = { version = "^1.1", optional = true } aws-sdk-sqs = { version = "^1.1", optional = true } aws-sdk-lambda = { version = "^1.1", optional = true } extism = { version = "1.2.0", optional = true } +mithril-client = { version = "^0.8", optional = true, features = ["fs"] } +miette = { version = "7.2.0", features = ["fancy"] } +itertools = "0.12.1" \ No newline at end of file diff --git a/examples/dolos_source/daemon.toml b/examples/dolos_source/daemon.toml index 14988558..35f1a4d0 100644 --- a/examples/dolos_source/daemon.toml +++ b/examples/dolos_source/daemon.toml @@ -1,5 +1,5 @@ [source] -type = "UtxoRPC" +type = "U5C" url = "https://50051-romantic-calmness-b55bqg.us1.demeter.run" [intersect] diff --git a/examples/mithril/daemon.toml b/examples/mithril/daemon.toml new file mode 100644 index 00000000..ee07347b --- /dev/null +++ b/examples/mithril/daemon.toml @@ -0,0 +1,12 @@ +[source] +type = "Mithril" +aggregator = "https://aggregator.pre-release-preview.api.mithril.network/aggregator" +genesis_key = "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d" +snapshot_download_dir = "./snapshot" +skip_validation = false + +[intersect] +type = "Origin" + +[sink] +type = "Stdout" diff --git a/src/sources/mithril.rs b/src/sources/mithril.rs new file mode 100644 index 00000000..bc8e56a2 --- /dev/null +++ b/src/sources/mithril.rs @@ -0,0 +1,256 @@ +use miette::{Context as _, IntoDiagnostic as _}; +use pallas::{ledger::traverse::MultiEraBlock, network::miniprotocols::Point::{self, *}}; +use serde::Deserialize; +use gasket::framework::*; +use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; +use tracing::warn; +use std::{path::Path, sync::Arc}; +use itertools::Itertools; + +use crate::framework::*; + +struct Feedback { + _multi: MultiProgress, + download_pb: ProgressBar, + validate_pb: ProgressBar, +} + +impl Feedback { + fn indeterminate_progress_bar(owner: &mut MultiProgress) -> ProgressBar { + let pb = ProgressBar::new_spinner(); + + pb.set_style( + ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] {msg}").unwrap(), + ); + + owner.add(pb) + } + + fn bytes_progress_bar(owner: &mut MultiProgress) -> ProgressBar { + let pb = ProgressBar::new_spinner(); + + pb.set_style( + ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} (eta: {eta}) {msg}", + ) + .unwrap() + .progress_chars("#>-"), + ); + + owner.add(pb) + } +} + +impl Default for Feedback { + fn default() -> Self { + let mut multi = MultiProgress::new(); + + Self { + download_pb: Self::bytes_progress_bar(&mut multi), + validate_pb: Self::indeterminate_progress_bar(&mut multi), + _multi: multi, + } + } +} + +#[async_trait::async_trait] +impl mithril_client::feedback::FeedbackReceiver for Feedback { + async fn handle_event(&self, event: mithril_client::feedback::MithrilEvent) { + match event { + mithril_client::feedback::MithrilEvent::SnapshotDownloadStarted { .. } => { + self.download_pb.set_message("snapshot download started") + } + mithril_client::feedback::MithrilEvent::SnapshotDownloadProgress { + downloaded_bytes, + size, + .. + } => { + self.download_pb.set_length(size); + self.download_pb.set_position(downloaded_bytes); + self.download_pb.set_message("downloading Mithril snapshot"); + } + mithril_client::feedback::MithrilEvent::SnapshotDownloadCompleted { .. } => { + self.download_pb.set_message("snapshot download completed"); + } + mithril_client::feedback::MithrilEvent::CertificateChainValidationStarted { + .. + } => { + self.validate_pb + .set_message("certificate chain validation started"); + } + mithril_client::feedback::MithrilEvent::CertificateValidated { + certificate_hash: hash, + .. + } => { + self.validate_pb + .set_message(format!("validating cert: {hash}")); + } + mithril_client::feedback::MithrilEvent::CertificateChainValidated { .. } => { + self.validate_pb.set_message("certificate chain validated"); + } + } + } +} + +async fn fetch_snapshot( + config: &Config, + feedback: Arc, +) -> MithrilResult<()> { + let client = ClientBuilder::aggregator(&config.aggregator, &config.genesis_key) + .add_feedback_receiver(feedback) + .build()?; + + let snapshots = client.snapshot().list().await?; + + let last_digest = snapshots + .first() + .ok_or(MithrilError::msg("no snapshot available"))? + .digest + .as_ref(); + + let snapshot = client + .snapshot() + .get(last_digest) + .await? + .ok_or(MithrilError::msg("no snapshot available"))?; + + let target_directory = Path::new(&config.snapshot_download_dir); + + client + .snapshot() + .download_unpack(&snapshot, target_directory) + .await?; + + if let Err(e) = client.snapshot().add_statistics(&snapshot).await { + warn!("failed incrementing snapshot download statistics: {:?}", e); + } + + let certificate = if config.skip_validation { + client + .certificate() + .get(&snapshot.certificate_hash) + .await? + .ok_or(MithrilError::msg("certificate for snapshot not found"))? + } else { + client + .certificate() + .verify_chain(&snapshot.certificate_hash) + .await? + }; + + let message = MessageBuilder::new() + .compute_snapshot_message(&certificate, target_directory) + .await?; + + assert!(certificate.match_message(&message)); + + Ok(()) +} + +#[derive(Stage)] +#[stage( + name = "source", + unit = "()", + worker = "Worker" +)] +pub struct Stage { + config: Config, + + pub output: SourceOutputPort, +} + +pub struct Worker { + config: Config, +} + +impl Worker { +} + +#[async_trait::async_trait(?Send)] +impl gasket::framework::Worker for Worker { + async fn bootstrap(stage: &Stage) -> Result { + let feedback = Arc::new(Feedback::default()); + let target_directory = Path::new(&stage.config.snapshot_download_dir); + + if !target_directory.exists() { + std::fs::create_dir_all(target_directory) + .map_err(|err| miette::miette!(err.to_string())) + .context(format!( + "Failed to create directory: {}", + target_directory.display() + )) + .map_err(|_| WorkerError::Panic)?; + } + + fetch_snapshot(&stage.config, feedback.clone()) + .await + .map_err(|err| miette::miette!(err.to_string())) + .context("fetching and validating mithril snapshot") + .map_err(|_| WorkerError::Panic)?; + + Ok(Self { + config: stage.config.clone(), + }) + } + + async fn schedule(&mut self, _stage: &mut Stage) -> Result, WorkerError> { + Ok(WorkSchedule::Unit(())) + } + + async fn execute(&mut self, _unit: &(), stage: &mut Stage) -> Result<(), WorkerError> { + + let immutable_path = Path::new(&self.config.snapshot_download_dir).join("immutable"); + + let iter = pallas::storage::hardano::immutable::read_blocks(&immutable_path) + .into_diagnostic() + .context("reading immutable db") + .map_err(|_| WorkerError::Panic)?; + + for chunk in iter.chunks(100).into_iter() { + let bodies: Vec<_> = chunk + .try_collect() + .into_diagnostic() + .context("reading block data") + .map_err(|_| WorkerError::Panic)?; + + let blocks: Vec<(Point, Vec)> = bodies + .iter() + .map(|b| { + let blockd = MultiEraBlock::decode(b) + .into_diagnostic() + .context("decoding block cbor") + .unwrap(); + (Specific(blockd.slot(), blockd.hash().to_vec()), b.clone()) + }) + .collect(); + + for (point, block) in blocks { + let event = ChainEvent::Apply(point, Record::CborBlock(block)); + stage.output.send(event.into()).await.or_panic()?; + } + } + + Ok(()) + } +} + +#[derive(Deserialize, Clone)] +pub struct Config { + aggregator: String, + genesis_key: String, + snapshot_download_dir: String, + skip_validation: bool, +} + + +impl Config { + pub fn bootstrapper(self, _ctx: &Context) -> Result { + let stage = Stage { + config: self, + output: Default::default(), + }; + + Ok(stage) + } +} \ No newline at end of file diff --git a/src/sources/mod.rs b/src/sources/mod.rs index a66a1834..87cb3950 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -15,6 +15,9 @@ pub mod u5c; #[cfg(feature = "aws")] pub mod s3; +#[cfg(feature = "mithril")] +pub mod mithril; + pub enum Bootstrapper { N2N(n2n::Stage), @@ -26,6 +29,9 @@ pub enum Bootstrapper { #[cfg(feature = "aws")] S3(s3::Stage), + + #[cfg(feature = "mithril")] + Mithril(mithril::Stage), } impl Bootstrapper { @@ -41,6 +47,9 @@ impl Bootstrapper { #[cfg(feature = "aws")] Bootstrapper::S3(p) => &mut p.output, + + #[cfg(feature = "mithril")] + Bootstrapper::Mithril(p) => &mut p.output, } } @@ -56,6 +65,9 @@ impl Bootstrapper { #[cfg(feature = "aws")] Bootstrapper::S3(x) => gasket::runtime::spawn_stage(x, policy), + + #[cfg(feature = "mithril")] + Bootstrapper::Mithril(x) => gasket::runtime::spawn_stage(x, policy), } } } @@ -73,6 +85,9 @@ pub enum Config { #[cfg(feature = "aws")] S3(s3::Config), + + #[cfg(feature = "mithril")] + Mithril(mithril::Config), } impl Config { @@ -88,6 +103,9 @@ impl Config { #[cfg(feature = "aws")] Config::S3(c) => Ok(Bootstrapper::S3(c.bootstrapper(ctx)?)), + + #[cfg(feature = "mithril")] + Config::Mithril(c) => Ok(Bootstrapper::Mithril(c.bootstrapper(ctx)?)), } } } From dbcb6186b0bf5deecca6adeedf3b24e853682eab Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Thu, 20 Jun 2024 13:50:42 +0800 Subject: [PATCH 2/6] formatting fixes --- src/sources/mithril.rs | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/sources/mithril.rs b/src/sources/mithril.rs index bc8e56a2..85219918 100644 --- a/src/sources/mithril.rs +++ b/src/sources/mithril.rs @@ -1,12 +1,15 @@ -use miette::{Context as _, IntoDiagnostic as _}; -use pallas::{ledger::traverse::MultiEraBlock, network::miniprotocols::Point::{self, *}}; -use serde::Deserialize; use gasket::framework::*; -use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use tracing::warn; -use std::{path::Path, sync::Arc}; use itertools::Itertools; +use miette::{Context as _, IntoDiagnostic as _}; +use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; +use pallas::{ + ledger::traverse::MultiEraBlock, + network::miniprotocols::Point::{self, *}, +}; +use serde::Deserialize; +use std::{path::Path, sync::Arc}; +use tracing::warn; use crate::framework::*; @@ -93,10 +96,7 @@ impl mithril_client::feedback::FeedbackReceiver for Feedback { } } -async fn fetch_snapshot( - config: &Config, - feedback: Arc, -) -> MithrilResult<()> { +async fn fetch_snapshot(config: &Config, feedback: Arc) -> MithrilResult<()> { let client = ClientBuilder::aggregator(&config.aggregator, &config.genesis_key) .add_feedback_receiver(feedback) .build()?; @@ -149,11 +149,7 @@ async fn fetch_snapshot( } #[derive(Stage)] -#[stage( - name = "source", - unit = "()", - worker = "Worker" -)] +#[stage(name = "source", unit = "()", worker = "Worker")] pub struct Stage { config: Config, @@ -164,8 +160,7 @@ pub struct Worker { config: Config, } -impl Worker { -} +impl Worker {} #[async_trait::async_trait(?Send)] impl gasket::framework::Worker for Worker { @@ -182,7 +177,7 @@ impl gasket::framework::Worker for Worker { )) .map_err(|_| WorkerError::Panic)?; } - + fetch_snapshot(&stage.config, feedback.clone()) .await .map_err(|err| miette::miette!(err.to_string())) @@ -199,13 +194,12 @@ impl gasket::framework::Worker for Worker { } async fn execute(&mut self, _unit: &(), stage: &mut Stage) -> Result<(), WorkerError> { - let immutable_path = Path::new(&self.config.snapshot_download_dir).join("immutable"); let iter = pallas::storage::hardano::immutable::read_blocks(&immutable_path) - .into_diagnostic() - .context("reading immutable db") - .map_err(|_| WorkerError::Panic)?; + .into_diagnostic() + .context("reading immutable db") + .map_err(|_| WorkerError::Panic)?; for chunk in iter.chunks(100).into_iter() { let bodies: Vec<_> = chunk @@ -243,7 +237,6 @@ pub struct Config { skip_validation: bool, } - impl Config { pub fn bootstrapper(self, _ctx: &Context) -> Result { let stage = Stage { @@ -253,4 +246,4 @@ impl Config { Ok(stage) } -} \ No newline at end of file +} From 1df667642ca733796c1736d8a020db60f89c2460 Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Tue, 25 Jun 2024 21:21:48 +0800 Subject: [PATCH 3/6] - removed indicatif progress bar - prevent loop back into slot 0 after syncing --- src/sources/mithril.rs | 119 +++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 41 deletions(-) diff --git a/src/sources/mithril.rs b/src/sources/mithril.rs index 85219918..c4f94f33 100644 --- a/src/sources/mithril.rs +++ b/src/sources/mithril.rs @@ -1,5 +1,4 @@ use gasket::framework::*; -use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use itertools::Itertools; use miette::{Context as _, IntoDiagnostic as _}; use mithril_client::{ClientBuilder, MessageBuilder, MithrilError, MithrilResult}; @@ -8,51 +7,85 @@ use pallas::{ network::miniprotocols::Point::{self, *}, }; use serde::Deserialize; -use std::{path::Path, sync::Arc}; -use tracing::warn; +use std::{ + path::Path, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; +use tracing::{info, warn}; use crate::framework::*; struct Feedback { - _multi: MultiProgress, - download_pb: ProgressBar, - validate_pb: ProgressBar, + progress_logger: Arc>, } impl Feedback { - fn indeterminate_progress_bar(owner: &mut MultiProgress) -> ProgressBar { - let pb = ProgressBar::new_spinner(); - - pb.set_style( - ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] {msg}").unwrap(), - ); - - owner.add(pb) + fn new(log_interval: Duration) -> Self { + Self { + progress_logger: Arc::new(Mutex::new(ProgressLogger::new(log_interval))), + } } - fn bytes_progress_bar(owner: &mut MultiProgress) -> ProgressBar { - let pb = ProgressBar::new_spinner(); - - pb.set_style( - ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} (eta: {eta}) {msg}", - ) - .unwrap() - .progress_chars("#>-"), - ); - - owner.add(pb) + fn log_progress(&self, downloaded_bytes: u64, size: u64) { + if let Ok(mut logger) = self.progress_logger.lock() { + logger.log(downloaded_bytes, size); + } } } impl Default for Feedback { fn default() -> Self { - let mut multi = MultiProgress::new(); + Self::new(Duration::from_secs(10)) + } +} + +struct ProgressLogger { + last_log_time: Option, + log_interval: Duration, + initial_logged: bool, +} +impl ProgressLogger { + fn new(log_interval: Duration) -> Self { Self { - download_pb: Self::bytes_progress_bar(&mut multi), - validate_pb: Self::indeterminate_progress_bar(&mut multi), - _multi: multi, + last_log_time: None, + log_interval, + initial_logged: false, + } + } + + fn log(&mut self, downloaded_bytes: u64, size: u64) { + let now = Instant::now(); + let percentage = (downloaded_bytes as f64 / size as f64 * 100.0).round() as u64; + + if !self.initial_logged { + info!( + "Initial snapshot download progress: {}% ({}/{} bytes)", + percentage, downloaded_bytes, size + ); + self.initial_logged = true; + self.last_log_time = Some(now); + return; + } + + if downloaded_bytes == size { + info!( + "Snapshot download complete: 100% ({}/{} bytes)", + downloaded_bytes, size + ); + self.last_log_time = Some(now); + return; + } + + if let Some(last_time) = self.last_log_time { + if now.duration_since(last_time) >= self.log_interval { + info!( + "Snapshot download progress: {}% ({}/{} bytes)", + percentage, downloaded_bytes, size + ); + self.last_log_time = Some(now); + } } } } @@ -62,35 +95,31 @@ impl mithril_client::feedback::FeedbackReceiver for Feedback { async fn handle_event(&self, event: mithril_client::feedback::MithrilEvent) { match event { mithril_client::feedback::MithrilEvent::SnapshotDownloadStarted { .. } => { - self.download_pb.set_message("snapshot download started") + info!("snapshot download started"); } mithril_client::feedback::MithrilEvent::SnapshotDownloadProgress { downloaded_bytes, size, .. } => { - self.download_pb.set_length(size); - self.download_pb.set_position(downloaded_bytes); - self.download_pb.set_message("downloading Mithril snapshot"); + self.log_progress(downloaded_bytes, size); } mithril_client::feedback::MithrilEvent::SnapshotDownloadCompleted { .. } => { - self.download_pb.set_message("snapshot download completed"); + info!("snapshot download completed"); } mithril_client::feedback::MithrilEvent::CertificateChainValidationStarted { .. } => { - self.validate_pb - .set_message("certificate chain validation started"); + info!("certificate chain validation started"); } mithril_client::feedback::MithrilEvent::CertificateValidated { certificate_hash: hash, .. } => { - self.validate_pb - .set_message(format!("validating cert: {hash}")); + info!("certificate validated: {hash}"); } mithril_client::feedback::MithrilEvent::CertificateChainValidated { .. } => { - self.validate_pb.set_message("certificate chain validated"); + info!("certificate chain validation completed"); } } } @@ -158,6 +187,7 @@ pub struct Stage { pub struct Worker { config: Config, + is_bootstrapped: bool, } impl Worker {} @@ -186,11 +216,16 @@ impl gasket::framework::Worker for Worker { Ok(Self { config: stage.config.clone(), + is_bootstrapped: false, }) } async fn schedule(&mut self, _stage: &mut Stage) -> Result, WorkerError> { - Ok(WorkSchedule::Unit(())) + if self.is_bootstrapped { + return Ok(WorkSchedule::Done); + } else { + Ok(WorkSchedule::Unit(())) + } } async fn execute(&mut self, _unit: &(), stage: &mut Stage) -> Result<(), WorkerError> { @@ -225,6 +260,8 @@ impl gasket::framework::Worker for Worker { } } + self.is_bootstrapped = true; + Ok(()) } } From 6c6bda69e8b8e1c909d9548d42002a97bd3eae3a Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Mon, 8 Jul 2024 12:45:21 +0800 Subject: [PATCH 4/6] attempt to read from valid intersection according to config --- examples/mithril/daemon.toml | 6 ++- src/sources/mithril.rs | 79 ++++++++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/examples/mithril/daemon.toml b/examples/mithril/daemon.toml index ee07347b..24796ab5 100644 --- a/examples/mithril/daemon.toml +++ b/examples/mithril/daemon.toml @@ -6,7 +6,11 @@ snapshot_download_dir = "./snapshot" skip_validation = false [intersect] -type = "Origin" +type = "Point" +value = [ + 46243673, + "75af0dbb19e37a5efa0e1d5109efffb4ffddae04d45008e88e63951fb3337fb6", +] [sink] type = "Stdout" diff --git a/src/sources/mithril.rs b/src/sources/mithril.rs index c4f94f33..96623662 100644 --- a/src/sources/mithril.rs +++ b/src/sources/mithril.rs @@ -177,11 +177,56 @@ async fn fetch_snapshot(config: &Config, feedback: Arc) -> MithrilResu Ok(()) } +fn get_starting_points( + dir: &Path, + config: &IntersectConfig, +) -> Result, Box> { + match config { + IntersectConfig::Tip => pallas::storage::hardano::immutable::get_tip(dir)? + .map_or(Ok(vec![Point::Origin]), |point| Ok(vec![point])), + IntersectConfig::Origin => Ok(vec![Point::Origin]), + IntersectConfig::Point(slot, hash) => { + let hash_bytes = hex::decode(hash)?; + Ok(vec![Point::Specific(*slot, hash_bytes)]) + } + IntersectConfig::Breadcrumbs(points) => points + .iter() + .map(|(slot, hash)| { + let hash_bytes = hex::decode(hash)?; + Ok(Point::Specific(*slot, hash_bytes)) + }) + .collect(), + } +} + +fn read_blocks_with_config( + immutable_path: &Path, + config: &IntersectConfig, +) -> Result< + Box + Send + Sync>, + WorkerError, +> { + let starting_points = + get_starting_points(immutable_path, config).map_err(|_| WorkerError::Panic)?; + + for point in starting_points { + match pallas::storage::hardano::immutable::read_blocks_from_point(immutable_path, point) { + Ok(iter) => return Ok(iter), + Err(_) => continue, + } + } + + // If all points fail (or if the list was empty), try from Origin + pallas::storage::hardano::immutable::read_blocks_from_point(immutable_path, Point::Origin) + .map_err(|_| WorkerError::Panic) +} + + #[derive(Stage)] #[stage(name = "source", unit = "()", worker = "Worker")] pub struct Stage { config: Config, - + intersect: IntersectConfig, pub output: SourceOutputPort, } @@ -208,21 +253,36 @@ impl gasket::framework::Worker for Worker { .map_err(|_| WorkerError::Panic)?; } - fetch_snapshot(&stage.config, feedback.clone()) - .await + // Check if the directory is empty + let is_dir_empty = target_directory + .read_dir() .map_err(|err| miette::miette!(err.to_string())) - .context("fetching and validating mithril snapshot") - .map_err(|_| WorkerError::Panic)?; + .context("Failed to read target directory") + .map_err(|_| WorkerError::Panic)? + .next() + .is_none(); + + if is_dir_empty { + // Directory is empty, fetch the snapshot + fetch_snapshot(&stage.config, feedback.clone()) + .await + .map_err(|err| miette::miette!(err.to_string())) + .context("fetching and validating mithril snapshot") + .map_err(|_| WorkerError::Panic)?; + } else { + println!("Snapshot directory is not empty. Assuming existing snapshot data."); + // You might want to add some validation here to ensure the existing data is valid + } Ok(Self { config: stage.config.clone(), - is_bootstrapped: false, + is_bootstrapped: false, // Set to true since we either fetched or found existing data }) } async fn schedule(&mut self, _stage: &mut Stage) -> Result, WorkerError> { if self.is_bootstrapped { - return Ok(WorkSchedule::Done); + Ok(WorkSchedule::Done) } else { Ok(WorkSchedule::Unit(())) } @@ -231,7 +291,7 @@ impl gasket::framework::Worker for Worker { async fn execute(&mut self, _unit: &(), stage: &mut Stage) -> Result<(), WorkerError> { let immutable_path = Path::new(&self.config.snapshot_download_dir).join("immutable"); - let iter = pallas::storage::hardano::immutable::read_blocks(&immutable_path) + let iter = read_blocks_with_config(&immutable_path, &stage.intersect) .into_diagnostic() .context("reading immutable db") .map_err(|_| WorkerError::Panic)?; @@ -275,9 +335,10 @@ pub struct Config { } impl Config { - pub fn bootstrapper(self, _ctx: &Context) -> Result { + pub fn bootstrapper(self, ctx: &Context) -> Result { let stage = Stage { config: self, + intersect: ctx.intersect.clone(), output: Default::default(), }; From ff94ab877ed64826fb725762ca007876856f0b80 Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Mon, 8 Jul 2024 19:48:13 +0800 Subject: [PATCH 5/6] breadcrumbs example --- examples/mithril/daemon.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/mithril/daemon.toml b/examples/mithril/daemon.toml index 24796ab5..da00f074 100644 --- a/examples/mithril/daemon.toml +++ b/examples/mithril/daemon.toml @@ -6,11 +6,8 @@ snapshot_download_dir = "./snapshot" skip_validation = false [intersect] -type = "Point" -value = [ - 46243673, - "75af0dbb19e37a5efa0e1d5109efffb4ffddae04d45008e88e63951fb3337fb6", -] +type = "Breadcrumbs" +value = [[ 50948461, "6aae3090ba47b555cdf5afbbc188babcbf195f171757a3b67007adb5ed6f0c0c" ], [ 45965940, "be02b0f25facb2fd31a85cd10bca54eb0252421a00ac2d164f6794aa312f0ab6" ], ] [sink] type = "Stdout" From 108fb3c5dc12eb4ba2a22b4012fb3f15b6834af6 Mon Sep 17 00:00:00 2001 From: Clark Alesna Date: Tue, 9 Jul 2024 09:18:10 +0800 Subject: [PATCH 6/6] removed AI comment suggestion kekw --- src/sources/mithril.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/mithril.rs b/src/sources/mithril.rs index 96623662..bafe059b 100644 --- a/src/sources/mithril.rs +++ b/src/sources/mithril.rs @@ -271,7 +271,7 @@ impl gasket::framework::Worker for Worker { .map_err(|_| WorkerError::Panic)?; } else { println!("Snapshot directory is not empty. Assuming existing snapshot data."); - // You might want to add some validation here to ensure the existing data is valid + // @TODO add data validation (?) or just assume it's correct } Ok(Self {