chore: Improve local execution (#91)

* transactions module

* TxEssence trait

* Generic Transaction struct

* tx essence as a generic parameter

* OptimismTxEssence

* op chain spec

* OpTxExecStrategy

* redundant block builder type params

* strategy bundles

* host binary parameters

* profiling flag

* optimism

* new derivation binary skeleton, copy over libs

* providers and conversion utils

* epoch transitioning

* basic derive flow

* host-side derivation

* read metadata from op head

* op-derive guest

* disable guest memory leaks

* heapless batch derivation

* Add missing import

* Remove heapless BinaryHeap

* Remove heapless

* Introduce op-derive tool

* Remove ethers types from BatcherDb trait

* Verify new op block has correct transaction list

* Move derive logic into library

* Fix bug in transaction trie reconstruction

* Introduce get_op_header() to BatcherDb

* Clippy warning

* Default Serde value for FileProvider::receipts

* Disable bloom filter checks

* Clippy

* Clippy

* Fix parsing of from and to fields for deposits

* Reintroduce filtering by log bloom

* fmt

* Add support for local exec to op-derive

* Enforce block_number is correct in MemDb

* Split derive() into multiple functions

* Remove redundant check for batch parent hash

* Remove redundant copy of system config

* Remove redundant block number check

* Cleanup

* Remove redundant vector of eth blocks

* Add base_fee_per_gas to Epoch

* Store deposits in Epoch

* Add Eth tail to DeriveOutput

* Cleanup

* Move deque_next_epoch_if_none to State

* Move eth block processing to Batches

* update zkvm, basic guest

* more compose guest code, comment bonsai code

* untested composition draft guest code

* Reorg and cleanup

* More cleanup

* run cargo fmt --all

* fix clippy warnings

* remove unused imports

* basic prep/left/fin in-memory flow

* in memory aggregation workflow

* format guest code

* composition with receipts

* add op-derive cmd test

* add cmd tests

* cleanup optimism/mod

* Rename command line args

* Fix test arguments

* Rename config field to max_channel_bank_size

* Enforce decompression limit of MAX_RLP_BYTES_PER_CHANNEL

* Use constant OPTIMISM_DEPOSITED_TX_TYPE when checking batch validity

* Import from std instead of alloc/core

* Re-enable core::mem::forget() optimization

* Replace asserts with ensures; enforce absence of receipts for Op blocks

* Simplfy iteration through derived transactions

* Add Bonsai support to op-derive. Also add Bonsai session status to output when polling

* More println

* Rework Batcher initialization

* More logging

* More log output if Bonsai workflow fails

* refactor rpc db

* variable derive step support

* re-enable profiling

* ignore rpc_cache dir

* add -profile support to compose binary

* upgrade zkvm, modify code comments

* bump zkvm, fix CI

* clippies

* fixes and changes

* vs code change

* disambiguate merkle ranges and proofs

Co-authored-by: Wolfgang Welz <welzwo@gmail.com>

* update risc0 to release v0.20

* update GH action to 0.20

* use old actions

* changes

* refactor: Unified zeth utility (#72)

* feat: Provably build derived op blocks (#78)

* feat: Composition via Bonsai (#79)

* feat: Deterministic builds with receipt cache (+misc) (#80)

* cleanup toml files

* remove unused imports

* cli cleanup

* change block count to u32

* test composition

* fix arguments

* update risc0 to 0.20.1

* compact json files

* update lock files

* improve execution

* test for warnings

* nits

* docker in readme

* create cache file if not found

* update copyright

* update copyright

---------

Co-authored-by: Rami Khalil <the.rami.khalil@gmail.com>
Co-authored-by: Timothy Carstens <intoverflow@gmail.com>
Co-authored-by: Rami <2418646+hashcashier@users.noreply.github.com>

host/Cargo.toml

@@ -27,6 +27,7 @@ serde = "1.0"
 tempfile = "3.6"
 tokio = { version = "1.23", features = ["full"] }
 tracing = { version = "0.1", features = ["log"] }
+typetag = "0.2.15"
 zeth-guests = { path = "../guests" }
 zeth-lib = { path = "../lib" }
 zeth-primitives = { path = "../primitives" }

host/src/operations/mod.rs

@@ -21,12 +21,12 @@ use std::fmt::Debug;
 use bonsai_sdk::alpha::responses::SnarkReceipt;
 use log::{debug, error, info, warn};
 use risc0_zkvm::{
-    compute_image_id, default_prover,
+    compute_image_id,
     serde::to_vec,
     sha::{Digest, Digestible},
-    Assumption, ExecutorEnv, ExecutorImpl, FileSegmentRef, Receipt, Session,
+    Assumption, ExecutorEnv, ExecutorImpl, FileSegmentRef, Receipt, Segment, SegmentRef,
 };
-use serde::{de::DeserializeOwned, Serialize};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use tempfile::tempdir;
 use zeth_primitives::keccak::keccak;
 
@@ -295,6 +295,8 @@ pub async fn prove_bonsai<O: Eq + Debug + DeserializeOwned>(
     verify_bonsai_receipt(image_id, expected_output, session.uuid.clone(), 8).await
 }
 
+/// Prove the given ELF locally with the given input and assumptions. The segments are
+/// stored in a temporary directory, to allow for proofs larger than the available memory.
 pub fn prove_locally(
     segment_limit_po2: u32,
     encoded_input: Vec<u32>,
@@ -311,34 +313,55 @@ pub fn prove_locally(
     );
 
     info!("Running the prover...");
-    let mut env_builder = ExecutorEnv::builder();
+    let session = {
+        let mut env_builder = ExecutorEnv::builder();
+        env_builder
+            .session_limit(None)
+            .segment_limit_po2(segment_limit_po2)
+            .write_slice(&encoded_input);
 
-    env_builder
-        .session_limit(None)
-        .segment_limit_po2(segment_limit_po2)
-        .write_slice(&encoded_input);
+        if profile {
+            info!("Profiling enabled.");
+            env_builder.enable_profiler(format!("profile_{}.pb", profile_reference));
+        }
 
-    if profile {
-        info!("Profiling enabled.");
-        env_builder.enable_profiler(format!("profile_{}.pb", profile_reference));
-    }
+        for assumption in assumptions {
+            env_builder.add_assumption(assumption);
+        }
 
-    for assumption in assumptions {
-        env_builder.add_assumption(assumption);
-    }
+        let env = env_builder.build().unwrap();
+        let mut exec = ExecutorImpl::from_elf(env, elf).unwrap();
+
+        let segment_dir = tempdir().unwrap();
+
+        exec.run_with_callback(|segment| {
+            Ok(Box::new(FileSegmentRef::new(&segment, segment_dir.path())?))
+        })
+        .unwrap()
+    };
+    session.prove().unwrap()
+}
 
-    let prover = default_prover();
-    prover.prove(env_builder.build().unwrap(), elf).unwrap()
+const NULL_SEGMENT_REF: NullSegmentRef = NullSegmentRef {};
+#[derive(Serialize, Deserialize)]
+struct NullSegmentRef {}
+
+#[typetag::serde]
+impl SegmentRef for NullSegmentRef {
+    fn resolve(&self) -> anyhow::Result<Segment> {
+        unimplemented!()
+    }
 }
 
-pub fn execute<T: serde::Serialize + ?Sized, O: Eq + Debug + DeserializeOwned>(
+/// Execute the guest code with the given input and verify the output.
+pub fn execute<T: Serialize, O: Eq + Debug + DeserializeOwned>(
     input: &T,
     segment_limit_po2: u32,
     profile: bool,
     elf: &[u8],
     expected_output: &O,
     profile_reference: &String,
-) -> Session {
+) {
     debug!(
         "Running in executor with segment_limit_po2 = {:?}",
         segment_limit_po2
@@ -352,31 +375,31 @@ pub fn execute<T: serde::Serialize + ?Sized, O: Eq + Debug + DeserializeOwned>(
     );
 
     info!("Running the executor...");
-    let start_time = std::time::Instant::now();
     let session = {
-        let mut builder = ExecutorEnv::builder();
-        builder
+        let mut env_builder = ExecutorEnv::builder();
+        env_builder
             .session_limit(None)
             .segment_limit_po2(segment_limit_po2)
             .write_slice(&input);
 
         if profile {
             info!("Profiling enabled.");
-            builder.enable_profiler(format!("profile_{}.pb", profile_reference));
+            env_builder.enable_profiler(format!("profile_{}.pb", profile_reference));
         }
 
-        let env = builder.build().unwrap();
+        let env = env_builder.build().unwrap();
         let mut exec = ExecutorImpl::from_elf(env, elf).unwrap();
 
-        let segment_dir = tempdir().unwrap();
-
-        exec.run_with_callback(|segment| {
-            Ok(Box::new(FileSegmentRef::new(&segment, segment_dir.path())?))
-        })
-        .unwrap()
+        exec.run_with_callback(|_| Ok(Box::new(NULL_SEGMENT_REF)))
+            .unwrap()
     };
+    println!(
+        "Executor ran in (roughly) {} cycles",
+        session.segments.len() * (1 << segment_limit_po2)
+    );
     // verify output
-    let output_guest: O = session.journal.clone().unwrap().decode().unwrap();
+    let journal = session.journal.unwrap();
+    let output_guest: O = journal.decode().expect("Could not decode journal");
     if expected_output == &output_guest {
         info!("Executor succeeded");
     } else {
@@ -385,16 +408,4 @@ pub fn execute<T: serde::Serialize + ?Sized, O: Eq + Debug + DeserializeOwned>(
             output_guest, expected_output,
         );
     }
-    // report performance
-    println!(
-        "Generated {:?} segments; elapsed time: {:?}",
-        session.segments.len(),
-        start_time.elapsed()
-    );
-    println!(
-        "Executor ran in (roughly) {} cycles",
-        session.segments.len() * (1 << segment_limit_po2)
-    );
-    // return result
-    session
 }