Merge branch 'main' into feature/column_inc

Cargo.toml

@@ -1,10 +1,6 @@
 
 [workspace]
-members = [
-    "crates/cli",
-    "crates/freeze",
-    "crates/python",
-]
+members = ["crates/cli", "crates/freeze", "crates/python"]
 
 # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 but not virtual workspaces.
 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html
@@ -16,12 +12,48 @@ edition = "2021"
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/paradigmxyz/cryo"
 repository = "https://github.com/paradigmxyz/cryo"
-exclude = [
-    ".github/",
-]
+exclude = [".github/"]
+
+[workspace.dependencies]
+anstyle = "1.0.1"
+async-trait = "0.1.68"
+chrono = { version = "0.4.26", features = ["serde"] }
+clap_cryo = { version = "4.3.21-cryo", features = [
+    "derive",
+    "color",
+    "unstable-styles",
+] }
+colored = "2.0.0"
+color-print = "0.3.4"
+ethers = { version = "2.0.8", features = ["rustls", "ws", "ipc"] }
+ethers-core = "2.0.8"
+eyre = "0.6.8"
+futures = "0.3.28"
+governor = "0.5.1"
+hex = "0.4.3"
+indexmap = "2.0.0"
+indicatif = "0.17.5"
+polars = { version = "0.32.1", features = [
+    "parquet",
+    "string_encoding",
+    "polars-lazy",
+    "lazy",
+    "binary_encoding",
+    "json",
+    "dtype-struct",
+] }
+prefix-hex = "0.7.0"
+pyo3 = { version = "0.19.2", features = ["extension-module"] }
+pyo3-build-config = "0.19.0"
+pyo3-asyncio = { version = "0.19.0", features = ["tokio-runtime"] }
+pyo3-polars = "0.6.0"
+serde = { version = "1.0.183", features = ["derive"] }
+serde_json = "1.0.104"
+thiserror = "1.0.40"
+thousands = "0.2.0"
+tokio = { version = "1.29.0", features = ["macros", "rt-multi-thread", "sync"] }
 
 [profile.dev]
 incremental = true
 debug = 1
 codegen-units = 32
-

README.md

@@ -87,6 +87,24 @@ Many `cryo` cli options will affect output schemas by adding/removing columns or
 
 `cryo` will always print out data schemas before collecting any data. To view these schemas without collecting data, use `--dry` to perform a dry run.
 
+#### Schema Design Guide
+
+An attempt is made to ensure that the dataset schemas conform to a common set of design guidelines:
+- By default, rows should contain enough information be order-able
+- Columns should be named by their JSON-RPC or ethers.rs defaults, except in cases where a much more explicit name is available
+- To make joins across tables easier, a given piece of information should use the same datatype and column name across tables when possible
+- Large ints such as `u256` should allow multiple conversions. A `value` column of type `u256` should allow: `value_binary`, `value_string`, `value_f64`, `value_decimal128`, `value_u64_high`, and `value_u64_low`
+- By default, columns related to non-identifying cryptographic signatures are omitted by default. For example, `state_root` of a block or `v`/`r`/`s` of a transaction
+- Integer values that can never be negative should be stored as unsigned integers
+
+Standard types across tables:
+- `block_number`: `u32`
+- `transaction_index`: `u32`
+- `nonce`: `u32`
+- `gas_used`: `u32`
+- `gas_limit`: `u32`
+- `chain_id`: `u64`
+
 #### JSON-RPC
 
 `cryo` currently obtains all of its data using the [JSON-RPC](https://ethereum.org/en/developers/docs/apis/json-rpc/) protocol standard.

crates/cli/Cargo.toml

@@ -19,19 +19,20 @@ path = "src/main.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-clap_cryo = { version = "4.3.21-cryo", features = ["derive", "color", "unstable-styles"] }
-color-print = "0.3.4"
-ethers = "2.0.7"
-hex = "0.4.3"
-indicatif = "0.17.5"
-polars = "0.32.1"
-tokio = "1.29.0"
 cryo_freeze = { version = "0.2.0", path = "../freeze" }
-colored = "2.0.0"
-thousands = "0.2.0"
-chrono = "0.4.26"
-anstyle = "1.0.1"
-eyre = "0.6.8"
-governor = "0.5.1"
-serde = { version = "1.0.183", features = ["derive"] }
-serde_json = "1.0.104"
+
+anstyle = { workspace = true }
+chrono = { workspace = true }
+clap_cryo = { workspace = true }
+colored = { workspace = true }
+color-print = { workspace = true }
+ethers = { workspace = true }
+eyre = { workspace = true }
+governor = { workspace = true }
+hex = { workspace = true }
+indicatif = { workspace = true }
+polars = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thousands = { workspace = true }
+tokio = { workspace = true }