Extension into branch

bin/mpt-test/README.md

@@ -4,6 +4,11 @@ This tool aims to verify mainnet blocks for the MPT circuit.
 
 ## Running tests
 
+Set the environment variable `WEB3_PROVIDER_URL` to a mainnet JSON-RPC provider, for example:
+```
+export WEB3_PROVIDER_URL=https://mainnet.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161
+```
+
 Just run `./test_mainnet_blocks.sh`
 
 NOTE: this run the tests with keccak testing disabled, because it takes SO MUCH to test with keccaks enables. If you want to run them with keccak, just run `cargo run --release --no-default-features`.
@@ -13,7 +18,6 @@ NOTE: this run the tests with keccak testing disabled, because it takes SO MUCH
 In order to add more blocks to prove you have to:
 
 - Add new entry in the `access-lists` folder
-- Set the environment variable `WEB3_SERVICE_PROVIDER` to a mainnet JSON-RPC provider
 - Run the tests again
 - You will have to upload the cache file again (web3_rpc_cache.bin) and update the `test_mainnet_blocks.sh` file
 

bin/mpt-test/src/circuit/witness.rs

@@ -220,6 +220,48 @@ impl<F: Field> Witness<F> {
         let mut initial_values = Vec::new();
         let mut changed_values = Vec::new();
 
+        // Put the read proofs first:
+        if include_initial_values {
+            for entry in access_list.clone().0 {
+                let AccessListItem {
+                    address,
+                    storage_keys,
+                } = entry;
+
+                let old = provider
+                    .get_proof(
+                        address,
+                        storage_keys.clone(),
+                        Some(BlockId::Number(BlockNumber::Number(block_no - 1))),
+                    )
+                    .await?;
+
+                // Skip if the account doesn't exist in the old block.
+                if old.balance.is_zero()
+                    && old.code_hash.is_zero()
+                    && old.nonce.is_zero()
+                    && old.storage_hash.is_zero()
+                {
+                    continue;
+                }
+
+                initial_values.push(TrieModification::balance(address, old.balance));
+                initial_values.push(TrieModification::nonce(address, old.nonce));
+                initial_values.push(TrieModification::codehash(address, old.code_hash));
+
+                for key in storage_keys.iter() {
+                    let old = old.storage_proof.iter().find(|p| p.key == *key).unwrap();
+                    if old.value == U256::zero() {
+                        initial_values.push(TrieModification::storage_does_not_exist(
+                            address, *key, old.value,
+                        ));
+                    } else {
+                        initial_values.push(TrieModification::storage(address, *key, old.value));
+                    }
+                }
+            }
+        }
+
         for entry in access_list.0 {
             let AccessListItem {
                 address,
@@ -252,42 +294,30 @@ impl<F: Field> Witness<F> {
                 continue;
             }
 
-            if include_initial_values {
-                initial_values.push(TrieModification::balance(address, old.balance));
-                initial_values.push(TrieModification::nonce(address, old.nonce));
-                initial_values.push(TrieModification::codehash(address, old.code_hash));
-
-                for key in storage_keys.iter() {
-                    let old = old.storage_proof.iter().find(|p| p.key == *key).unwrap();
-                    if old.value == U256::zero() {
-                        initial_values.push(TrieModification::storage_does_not_exist(
-                            address, *key, old.value,
-                        ));
-                    } else {
-                        initial_values.push(TrieModification::storage(address, *key, old.value));
-                    }
-                }
-            }
-
             // check for this address changes
             if old.nonce != new.nonce {
                 changed_values.push(TrieModification::nonce(address, new.nonce));
             }
             if old.balance != new.balance {
                 changed_values.push(TrieModification::balance(address, new.balance));
             }
-            if old.code_hash != new.code_hash
-            // && new.code_hash != *DEFAULT_CODE_HASH
-            {
+
+            if old.code_hash != new.code_hash && new.code_hash != *DEFAULT_CODE_HASH {
                 changed_values.push(TrieModification::codehash(address, new.code_hash));
             }
 
             for key in storage_keys {
                 let new = new.storage_proof.iter().find(|p| p.key == key).unwrap();
-                changed_values.push(TrieModification::storage(address, key, new.value));
+                let old = old.storage_proof.iter().find(|p| p.key == key).unwrap();
+                if !(old.value == U256::zero() && new.value == U256::zero()) {
+                    changed_values.push(TrieModification::storage(address, key, new.value));
+                }
             }
         }
 
+        println!("initial_values.len(): {}", initial_values.len());
+        println!("changed_values.len(): {}", changed_values.len());
+
         let mut trie_modifications = initial_values;
         trie_modifications.append(&mut changed_values);
 

bin/mpt-test/src/main.rs

@@ -16,19 +16,20 @@ async fn mock_prove(block_no: u64, access_list: &str) -> Result<()> {
 
     let access_list: AccessList = serde_json::from_str(access_list)?;
     let proof_count = 2 * access_list.0.len() * 3
-        + access_list
+        + 2 * access_list
             .0
             .iter()
             .map(|k| k.storage_keys.len())
             .sum::<usize>();
 
-    let max_nodes = 40000;
+    let max_nodes = 1000000;
+    let degree = 20;
 
     let witness = Witness::<Fr>::build(provider_url, U64::from(block_no), Some(access_list), true)
         .await?
         .unwrap();
 
-    let circuit = StateUpdateCircuit::new(witness, 16, max_nodes, proof_count + 10)?;
+    let circuit = StateUpdateCircuit::new(witness, degree, max_nodes, proof_count + 10)?;
     circuit.assert_satisfied();
     Ok(())
 }

geth-utils/gethutil/mpt/README.md

@@ -0,0 +1,33 @@
+# MPT witness generator
+
+## Generate witnesses
+
+To generate witnesses for the MPT circuit, execute
+
+```
+go test -v ./...
+```
+
+To generate the tests that use a local blockchain you need a local `geth`. You would
+need to run something like:
+```
+geth --dev --http --ipcpath ~/Library/Ethereum/geth.ipc
+
+```
+The local `geth` is used to generate some tests that have a small number of accounts so that
+these accounts appear in the first or second level of the trie. You might need to remove the
+database if you already have some accounts:
+
+```
+geth removedb
+```
+
+The witness files will appear in generated_witnesses folder.
+
+## Format the code
+
+To format the code use:
+
+```
+gofmt -w ./*
+```

geth-utils/gethutil/mpt/oracle/prefetch.go

@@ -81,7 +81,8 @@ var RemoteUrl = "https://mainnet.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161"
 var LocalUrl = "http://localhost:8545"
 
 // For generating special tests for MPT circuit:
-var PreventHashingInSecureTrie = false
+var PreventHashingInSecureTrie = false // storage
+var AccountPreventHashingInSecureTrie = false
 
 func toFilename(key string) string {
 	return fmt.Sprintf("/tmp/eth/json_%s", key)

geth-utils/gethutil/mpt/state/database.go

@@ -52,7 +52,7 @@ func (db *Database) CopyTrie(t Trie) Trie {
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *Database) OpenTrie(root common.Hash) (Trie, error) {
-	tr, err := trie.NewSecure(root, db.db)
+	tr, err := trie.NewSecure(root, db.db, false)
 	if err != nil {
 		return nil, err
 	}
@@ -62,7 +62,7 @@ func (db *Database) OpenTrie(root common.Hash) (Trie, error) {
 // OpenStorageTrie opens the storage trie of an account.
 func (db *Database) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
 	//return SimpleTrie{db.BlockNumber, root, true, addrHash}, nil
-	tr, err := trie.NewSecure(root, db.db)
+	tr, err := trie.NewSecure(root, db.db, true)
 	if err != nil {
 		return nil, err
 	}

geth-utils/gethutil/mpt/state/statedb.go

@@ -298,7 +298,12 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash {
 
 // GetProof returns the Merkle proof for a given account.
 func (s *StateDB) GetProof(addr common.Address) ([][]byte, []byte, [][]byte, bool, bool, error) {
-	return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes()))
+	newAddr := crypto.Keccak256Hash(addr.Bytes())
+	if oracle.AccountPreventHashingInSecureTrie {
+		bytes := append(addr.Bytes(), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...)
+		newAddr = common.BytesToHash(bytes)
+	}
+	return s.GetProofByHash(newAddr)
 }
 
 // GetProofByHash returns the Merkle proof for a given account.
@@ -311,7 +316,13 @@ func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, []byte, [][]by
 // GetStorageProof returns the Merkle proof for given storage slot.
 func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, []byte, [][]byte, bool, bool, error) {
 	var proof proofList
-	trie := s.StorageTrie(a)
+	newAddr := a
+	if oracle.AccountPreventHashingInSecureTrie {
+		bytes := append(a.Bytes(), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...)
+		newAddr = common.BytesToAddress(bytes)
+	}
+
+	trie := s.StorageTrie(newAddr)
 	if trie == nil {
 		return proof, nil, nil, false, false, errors.New("storage trie for requested address does not exist")
 	}
@@ -457,7 +468,7 @@ func (s *StateDB) SetStateObjectIfExists(addr common.Address) {
 			}
 
 			/*
-				When an account that does not exist is tried to be fetched by PrefetchAccount and when the some other account
+				When an account that does not exist is being fetched by PrefetchAccount and when some other account
 				exist at the overlapping address (the beginning of it), this (wrong) account is obtained by PrefetchAccount
 				and needs to be ignored.
 			*/
@@ -533,8 +544,15 @@ func (s *StateDB) updateStateObject(obj *stateObject) {
 	if err != nil {
 		panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err))
 	}
-	if err = s.trie.TryUpdateAlwaysHash(addr[:], data); err != nil {
-		s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err))
+
+	if !oracle.AccountPreventHashingInSecureTrie {
+		if err = s.trie.TryUpdateAlwaysHash(addr[:], data); err != nil {
+			s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err))
+		}
+	} else {
+		if err = s.trie.TryUpdate(addr[:], data); err != nil {
+			s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err))
+		}
 	}
 
 	// If state snapshotting is active, cache the data til commit. Note, this

geth-utils/gethutil/mpt/trie/proof.go

@@ -102,33 +102,27 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) (
 			fromLevel--
 			continue
 		}
-		// var hn Node
 
 		// We need nibbles in witness for extension keys.
 		// copy n.Key before it gets changed in ProofHash
 		var nCopy []byte
 		if short, ok := n.(*ShortNode); ok {
-			if !hasTerm(short.Key) { // only for extension keys
+			if !hasTerm(short.Key) { // extension keys
 				nCopy = make([]byte, len(short.Key))
 				copy(nCopy, short.Key)
 				extNibbles = append(extNibbles, nCopy)
+			} else {
+				extNibbles = append(extNibbles, []byte{})
 			}
+		} else {
+			extNibbles = append(extNibbles, []byte{})
 		}
 
-		// n, hn = hasher.ProofHash(n)
 		n, _ = hasher.ProofHash(n)
-		// if hash, ok := hn.(HashNode); ok || i == 0 {
 		// If the node's database encoding is a hash (or is the
 		// root node), it becomes a proof element.
 		enc, _ := rlp.EncodeToBytes(n)
-		/*
-			if !ok {
-				hash = hasher.HashData(enc)
-			}
-		*/
-		// proofDb.Put(hash, enc)
 		proofDb.Put([]byte{1, 1, 1}, enc)
-		// }
 	}
 
 	isNeighbourNodeHashed := false

geth-utils/gethutil/mpt/trie/secure_trie.go

@@ -40,6 +40,7 @@ type SecureTrie struct {
 	hashKeyBuf       [common.HashLength]byte
 	secKeyCache      map[string][]byte
 	secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch
+	isStorageTrie    bool
 }
 
 // NewSecure creates a trie with an existing root node from a backing database
@@ -53,15 +54,15 @@ type SecureTrie struct {
 // Loaded nodes are kept around until their 'cache generation' expires.
 // A new cache generation is created by each call to Commit.
 // cachelimit sets the number of past cache generations to keep.
-func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) {
+func NewSecure(root common.Hash, db *Database, isStorageTrie bool) (*SecureTrie, error) {
 	if db == nil {
 		panic("trie.NewSecure called without a database")
 	}
 	trie, err := New(root, db)
 	if err != nil {
 		return nil, err
 	}
-	return &SecureTrie{trie: *trie}, nil
+	return &SecureTrie{trie: *trie, isStorageTrie: isStorageTrie}, nil
 }
 
 // Get returns the value for key stored in the trie.
@@ -202,7 +203,8 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator {
 // The caller must not hold onto the return value because it will become
 // invalid on the next call to hashKey or secKey.
 func (t *SecureTrie) hashKey(key []byte) []byte {
-	if !oracle.PreventHashingInSecureTrie {
+	preventHashing := (oracle.PreventHashingInSecureTrie && t.isStorageTrie) || (oracle.AccountPreventHashingInSecureTrie && !t.isStorageTrie)
+	if !preventHashing {
 		h := NewHasher(false)
 		h.sha.Reset()
 		h.sha.Write(key)
@@ -211,6 +213,9 @@ func (t *SecureTrie) hashKey(key []byte) []byte {
 		return t.hashKeyBuf[:]
 	} else {
 		// For generating special tests for MPT circuit.
+		if len(key) < 32 { // accounts
+			key = append(key, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}...)
+		}
 		return key
 	}
 }