Skip to content
This repository has been archived by the owner on Sep 23, 2023. It is now read-only.

[wip] E35 #957

Open
wants to merge 872 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
872 commits
Select commit Hold shift + click to select a range
885f582
save
awskii Aug 11, 2023
f38e59c
Merge branch 'e35' of github.com:ledgerwatch/erigon-lib into e35
awskii Aug 11, 2023
b28e99a
save
awskii Aug 11, 2023
e7bf936
restore locality
awskii Aug 11, 2023
5763664
Merge branch 'e35' into e35_ex
AskAlexSharov Aug 11, 2023
6feb0b3
Merge branch 'e35_ex' into e35
AskAlexSharov Aug 11, 2023
4b898f3
save
AskAlexSharov Aug 12, 2023
51230ba
Merge branch 'e35' into e35_ex
AskAlexSharov Aug 12, 2023
ad6b685
save
AskAlexSharov Aug 12, 2023
5433291
save
AskAlexSharov Aug 12, 2023
93ae1cd
Merge branch 'e35_ex' into e35
AskAlexSharov Aug 12, 2023
d120170
save
AskAlexSharov Aug 12, 2023
ad9aa02
Merge branch 'e35_ex' into e35
AskAlexSharov Aug 12, 2023
e9b81d5
save
AskAlexSharov Aug 13, 2023
15ffe97
save
awskii Aug 14, 2023
dcc3801
save
awskii Aug 14, 2023
feb6b6d
Merge branch 'e35' into e35_ex
AskAlexSharov Aug 15, 2023
a0ac976
save
AskAlexSharov Aug 15, 2023
fdcaa16
Merge branch 'e35_ex' into e35
AskAlexSharov Aug 15, 2023
2616d48
save
AskAlexSharov Aug 15, 2023
9b33109
save
AskAlexSharov Aug 15, 2023
105237f
save
AskAlexSharov Aug 15, 2023
6e7fb4b
save
AskAlexSharov Aug 15, 2023
97f86de
save
awskii Aug 16, 2023
3c14955
save
awskii Aug 16, 2023
97fd4d0
working save
awskii Aug 16, 2023
44fa0ff
working save
awskii Aug 16, 2023
9199d80
save
awskii Aug 17, 2023
c371577
revert
awskii Aug 17, 2023
74be43f
save fix iter
awskii Aug 17, 2023
cb48738
save
awskii Aug 17, 2023
8804662
return nil instead of ErrBtIndexLookupBounds
AskAlexSharov Aug 18, 2023
5d968ec
save
awskii Aug 18, 2023
d112192
save nolocality cold
awskii Aug 18, 2023
80d3daa
save
awskii Aug 18, 2023
641ec33
save
AskAlexSharov Aug 19, 2023
6d51aba
save
AskAlexSharov Aug 20, 2023
80e2558
save
awskii Aug 21, 2023
bdcc8a9
save
awskii Aug 21, 2023
54c03dd
save
awskii Aug 22, 2023
f6ce779
save
awskii Aug 22, 2023
ff3b3ce
Merge branch 'main' into e35
AskAlexSharov Aug 23, 2023
be8602e
save
AskAlexSharov Aug 23, 2023
2739578
save
AskAlexSharov Aug 23, 2023
6d9bfdd
save
AskAlexSharov Aug 23, 2023
3689fda
testcase: locality index to kv file
AskAlexSharov Aug 23, 2023
a78c04b
disable ii compression
AskAlexSharov Aug 23, 2023
2e4eac4
disable ii compression
AskAlexSharov Aug 23, 2023
fa6c925
save
awskii Aug 23, 2023
369dbfa
save
AskAlexSharov Aug 24, 2023
71fc9ec
step towards 1 seed
AskAlexSharov Aug 24, 2023
016edb2
save
awskii Aug 24, 2023
b607c49
Merge branch 'e35' of github.com:ledgerwatch/erigon-lib into e35
awskii Aug 24, 2023
f91714e
save
awskii Aug 24, 2023
b0a2cd6
save
AskAlexSharov Aug 25, 2023
944198e
save
AskAlexSharov Aug 25, 2023
9e17cce
save
AskAlexSharov Aug 25, 2023
a2ec8df
fix
awskii Aug 25, 2023
78b85da
save
awskii Aug 25, 2023
bb5e756
save
AskAlexSharov Aug 26, 2023
53dddc7
save
AskAlexSharov Aug 26, 2023
51653cc
save
AskAlexSharov Aug 26, 2023
101b635
save
AskAlexSharov Aug 26, 2023
0e5bf49
Merge branch 'test_non_nil_next' into e35
AskAlexSharov Aug 26, 2023
a5c1a4a
save
AskAlexSharov Aug 26, 2023
35f5e97
save
AskAlexSharov Aug 26, 2023
7be8df2
save
AskAlexSharov Aug 26, 2023
f0fc0ff
save
AskAlexSharov Aug 28, 2023
7480b7f
save
AskAlexSharov Aug 28, 2023
07ffeda
save
AskAlexSharov Aug 28, 2023
a6eeb8b
save
AskAlexSharov Aug 28, 2023
295bd6a
save
AskAlexSharov Aug 28, 2023
cc78840
save
AskAlexSharov Aug 28, 2023
2b401f2
Merge branch 'e35_same_seed3' into e35
AskAlexSharov Aug 28, 2023
9c40c43
save
AskAlexSharov Aug 28, 2023
4819815
save
AskAlexSharov Aug 28, 2023
8524e56
save
AskAlexSharov Aug 28, 2023
7fb4f80
save
AskAlexSharov Aug 28, 2023
f4229a9
save
awskii Aug 29, 2023
5bc437c
save
awskii Aug 29, 2023
97dbf97
save
awskii Aug 29, 2023
c46f349
try commit
AskAlexSharov Aug 30, 2023
e9e7888
save
AskAlexSharov Aug 30, 2023
6650930
merge devel
AskAlexSharov Aug 30, 2023
2849934
merge devel
AskAlexSharov Aug 30, 2023
62f88f7
merge devel
AskAlexSharov Aug 30, 2023
6b543ed
TestSharedDomain_Unwind: deadlock on panic fix
AskAlexSharov Aug 30, 2023
4c38a42
save
AskAlexSharov Aug 30, 2023
ad6f2ab
fix
awskii Aug 30, 2023
2ddbc46
fix
awskii Aug 30, 2023
6062c62
fix
awskii Aug 31, 2023
1a57a1c
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Aug 31, 2023
735a4c7
save
AskAlexSharov Aug 31, 2023
a3ada23
save
AskAlexSharov Aug 31, 2023
ff85056
Merge branch 'e35' into e35_sharded_bloom_ef
AskAlexSharov Aug 31, 2023
140926e
save
AskAlexSharov Aug 31, 2023
12ba46c
save
AskAlexSharov Aug 31, 2023
9323ea9
save
AskAlexSharov Aug 31, 2023
47eb6d5
save
AskAlexSharov Aug 31, 2023
eac1d9b
save
AskAlexSharov Aug 31, 2023
b68b5b0
fix
awskii Aug 31, 2023
0624f88
save
awskii Aug 31, 2023
9b3574e
save
AskAlexSharov Sep 1, 2023
7e87a9e
merge devel
AskAlexSharov Sep 1, 2023
327ec84
merge devel
AskAlexSharov Sep 1, 2023
800c40d
merge devel
AskAlexSharov Sep 1, 2023
950ad04
merge devel
AskAlexSharov Sep 1, 2023
a1d9bc0
merge devel
AskAlexSharov Sep 1, 2023
743fc3c
merge devel
AskAlexSharov Sep 1, 2023
9295c09
merge devel
AskAlexSharov Sep 1, 2023
c114e94
merge devel
AskAlexSharov Sep 1, 2023
d43618e
merge devel
AskAlexSharov Sep 1, 2023
aab2457
merge devel
AskAlexSharov Sep 1, 2023
597274d
merge devel
AskAlexSharov Sep 1, 2023
ceabf68
merge devel
AskAlexSharov Sep 1, 2023
586a5c7
merge devel
AskAlexSharov Sep 1, 2023
f4949da
Merge branch 'main' into e35
AskAlexSharov Sep 1, 2023
a2a52a5
save
AskAlexSharov Sep 1, 2023
349455e
save
AskAlexSharov Sep 1, 2023
ac722ba
save
AskAlexSharov Sep 1, 2023
4224c57
save
AskAlexSharov Sep 1, 2023
c43e8c5
save
AskAlexSharov Sep 1, 2023
ea908b9
save
AskAlexSharov Sep 1, 2023
379768f
save
AskAlexSharov Sep 1, 2023
b549129
save
AskAlexSharov Sep 1, 2023
e21e9c9
save
AskAlexSharov Sep 1, 2023
34e8d4c
save
AskAlexSharov Sep 1, 2023
3cf8d78
save
AskAlexSharov Sep 1, 2023
3291ec1
save
AskAlexSharov Sep 1, 2023
61a7c73
save
AskAlexSharov Sep 1, 2023
3777ab9
save
AskAlexSharov Sep 1, 2023
0560634
save
AskAlexSharov Sep 1, 2023
58d6420
save
AskAlexSharov Sep 1, 2023
6de0809
save
AskAlexSharov Sep 1, 2023
a07a29a
Merge branch 'e35' into e35_sharded_bloom_ef
AskAlexSharov Sep 1, 2023
de283e4
save
AskAlexSharov Sep 1, 2023
00ae04b
save
AskAlexSharov Sep 1, 2023
84a7537
save
AskAlexSharov Sep 1, 2023
9406a45
save
AskAlexSharov Sep 1, 2023
7165c4d
save
AskAlexSharov Sep 1, 2023
fb86bf9
save
awskii Sep 1, 2023
e12dc4b
Merge branch 'e35' of github.com:ledgerwatch/erigon-lib into e35
awskii Sep 1, 2023
644a7f7
save
AskAlexSharov Sep 3, 2023
cc4975e
save
AskAlexSharov Sep 3, 2023
c9b7a74
save
AskAlexSharov Sep 3, 2023
fdee665
save
AskAlexSharov Sep 3, 2023
f682f7b
save
AskAlexSharov Sep 3, 2023
a264b7a
save
AskAlexSharov Sep 3, 2023
c051e01
save
AskAlexSharov Sep 3, 2023
353f2a8
save
AskAlexSharov Sep 3, 2023
4168239
save
AskAlexSharov Sep 3, 2023
a52adc0
save
AskAlexSharov Sep 3, 2023
c2759c1
save
AskAlexSharov Sep 3, 2023
c2f8313
save
AskAlexSharov Sep 3, 2023
5df6a8f
save
AskAlexSharov Sep 3, 2023
6f67204
save
AskAlexSharov Sep 3, 2023
d8160fc
save
AskAlexSharov Sep 3, 2023
757a345
save
AskAlexSharov Sep 3, 2023
1b452f2
save
AskAlexSharov Sep 3, 2023
6dea833
save
AskAlexSharov Sep 3, 2023
a3b9169
save
AskAlexSharov Sep 3, 2023
bbd9c14
save
AskAlexSharov Sep 3, 2023
62d2679
save
AskAlexSharov Sep 3, 2023
d87f6d1
save
AskAlexSharov Sep 3, 2023
757b433
save
AskAlexSharov Sep 3, 2023
9284db6
save
AskAlexSharov Sep 3, 2023
983877f
save
AskAlexSharov Sep 3, 2023
13e2a07
save
AskAlexSharov Sep 4, 2023
43bfe49
save
AskAlexSharov Sep 4, 2023
3a5e7e8
save
AskAlexSharov Sep 4, 2023
c7b7cb1
save
AskAlexSharov Sep 4, 2023
55355f1
save
AskAlexSharov Sep 4, 2023
0a4edb8
save
AskAlexSharov Sep 4, 2023
2fbf618
domain: avoid extra bt.Has()
AskAlexSharov Sep 4, 2023
b5dc3f6
domain_shared: replace bt by map.
AskAlexSharov Sep 4, 2023
dc04305
domain: use pre-allocated cursors
AskAlexSharov Sep 4, 2023
7b3ff6e
simplify MakeContext
AskAlexSharov Sep 4, 2023
12873d3
save
AskAlexSharov Sep 4, 2023
2fe1e7b
save
AskAlexSharov Sep 4, 2023
07125b3
save
AskAlexSharov Sep 4, 2023
f6d8116
save
AskAlexSharov Sep 4, 2023
0540385
mend
AskAlexSharov Sep 4, 2023
20e9465
save
AskAlexSharov Sep 4, 2023
f1197a2
save
AskAlexSharov Sep 4, 2023
e948b7a
save
AskAlexSharov Sep 4, 2023
c7419e9
save
AskAlexSharov Sep 4, 2023
a8cc33d
save
AskAlexSharov Sep 5, 2023
60ec6bb
inlinable select
AskAlexSharov Sep 5, 2023
e6e17a0
inlinable select
AskAlexSharov Sep 5, 2023
27eb871
save
AskAlexSharov Sep 5, 2023
d31d95b
Revert "save"
AskAlexSharov Sep 6, 2023
b777ff8
Revert "inlinable select"
AskAlexSharov Sep 6, 2023
ebbcd4d
save
AskAlexSharov Sep 6, 2023
38d6197
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Sep 6, 2023
9920be7
save
AskAlexSharov Sep 6, 2023
b8e852c
save
AskAlexSharov Sep 6, 2023
8cf2a89
save
AskAlexSharov Sep 6, 2023
7b1e0bf
save
AskAlexSharov Sep 6, 2023
2ebbf43
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Sep 6, 2023
2f0c118
save
AskAlexSharov Sep 6, 2023
f038568
optimize MakeContext
AskAlexSharov Sep 7, 2023
53b1d19
save
AskAlexSharov Sep 7, 2023
0ed5593
save
AskAlexSharov Sep 7, 2023
786afca
save
AskAlexSharov Sep 8, 2023
548db2d
save
AskAlexSharov Sep 8, 2023
4ea8d26
save
AskAlexSharov Sep 8, 2023
0e613a9
save
AskAlexSharov Sep 8, 2023
5bd422e
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Sep 9, 2023
e29fd56
save
AskAlexSharov Sep 10, 2023
100ca79
save
AskAlexSharov Sep 10, 2023
55ff13d
save
AskAlexSharov Sep 12, 2023
e20074b
save
AskAlexSharov Sep 12, 2023
68228db
save
AskAlexSharov Sep 12, 2023
4024a0f
save
AskAlexSharov Sep 12, 2023
4279c09
save
AskAlexSharov Sep 12, 2023
3cbe0a1
save
AskAlexSharov Sep 12, 2023
45d39e6
save
AskAlexSharov Sep 12, 2023
0991661
save
AskAlexSharov Sep 12, 2023
267bcb2
save
AskAlexSharov Sep 12, 2023
2b84d8a
save
AskAlexSharov Sep 12, 2023
fc46c9b
save
AskAlexSharov Sep 12, 2023
c7a3063
save
AskAlexSharov Sep 12, 2023
9760f6f
save
AskAlexSharov Sep 12, 2023
5e87ab7
save
AskAlexSharov Sep 12, 2023
84de6df
save
AskAlexSharov Sep 13, 2023
9a238b3
Merge branch 'downloader_faster_scan' into e35_sharded_bloom_ef
AskAlexSharov Sep 13, 2023
8bb948c
save
AskAlexSharov Sep 13, 2023
a1d7205
save
AskAlexSharov Sep 13, 2023
139fe8f
save
AskAlexSharov Sep 13, 2023
fd5aa78
save
AskAlexSharov Sep 13, 2023
0a502dd
save
AskAlexSharov Sep 13, 2023
d4cfe81
save
AskAlexSharov Sep 13, 2023
2e7538a
save
AskAlexSharov Sep 13, 2023
a1c1438
save
AskAlexSharov Sep 13, 2023
33e59e3
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Sep 14, 2023
992036d
save
AskAlexSharov Sep 14, 2023
61e149f
save
AskAlexSharov Sep 15, 2023
13118be
save
AskAlexSharov Sep 15, 2023
2eaedcf
Merge branch 't3' into e35_sharded_bloom_ef
AskAlexSharov Sep 15, 2023
433dbe8
Merge branch 'e35_sharded_bloom_ef' into e35
AskAlexSharov Sep 15, 2023
70de2d2
save
AskAlexSharov Sep 15, 2023
681acb9
fix linter error
AskAlexSharov Sep 16, 2023
fee880d
save
AskAlexSharov Sep 17, 2023
45fd248
save
AskAlexSharov Sep 17, 2023
6b8a5db
save
AskAlexSharov Sep 17, 2023
ec1068a
save
AskAlexSharov Sep 17, 2023
4a67e92
Merge branch 'main' into e35_sharded_bloom_ef
AskAlexSharov Sep 18, 2023
26acc4e
Merge branch 'e35_sharded_bloom_ef' into e35
AskAlexSharov Sep 18, 2023
ac8f5ee
save
AskAlexSharov Sep 19, 2023
b14c8ac
save
awskii Sep 19, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 35 additions & 18 deletions commitment/bin_patricia_hashed.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"fmt"
"io"
"math/bits"
"sort"

"github.com/holiman/uint256"
"github.com/ledgerwatch/log/v3"
Expand Down Expand Up @@ -746,7 +747,7 @@ func (bph *BinPatriciaHashed) computeBinaryCellHash(cell *BinaryCell, depth int,
var valBuf [128]byte
valLen := cell.accountForHashing(valBuf[:], storageRootHash)
if bph.trace {
fmt.Printf("accountLeafHashWithKey for [%x]=>[%x]\n", bph.hashAuxBuffer[:halfKeySize+1-depth], valBuf[:valLen])
fmt.Printf("accountLeafHashWithKey for [%x]=>[%x]\n", cell.downHashedKey[:halfKeySize+1-depth], rlp.RlpEncodedBytes(valBuf[:valLen]))
}
return bph.accountLeafHashWithKey(buf, cell.downHashedKey[:halfKeySize+1-depth], rlp.RlpEncodedBytes(valBuf[:valLen]))
}
Expand Down Expand Up @@ -1274,9 +1275,19 @@ func (bph *BinPatriciaHashed) RootHash() ([]byte, error) {
return hash[1:], nil // first byte is 128+hash_len
}

func (bph *BinPatriciaHashed) ReviewKeys(plainKeys, hashedKeys [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
func (bph *BinPatriciaHashed) ProcessKeys(plainKeys [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
branchNodeUpdates = make(map[string]BranchData)

pks := make(map[string]int, len(plainKeys))
hashedKeys := make([][]byte, len(plainKeys))
for i, pk := range plainKeys {
hashedKeys[i] = hexToBin(pk)
pks[string(hashedKeys[i])] = i
}

sort.Slice(hashedKeys, func(i, j int) bool {
return bytes.Compare(hashedKeys[i], hashedKeys[j]) < 0
})
stagedBinaryCell := new(BinaryCell)
for i, hashedKey := range hashedKeys {
plainKey := plainKeys[i]
Expand Down Expand Up @@ -1310,7 +1321,7 @@ func (bph *BinPatriciaHashed) ReviewKeys(plainKeys, hashedKeys [][]byte) (rootHa
cell.setAccountFields(stagedBinaryCell.CodeHash[:], &stagedBinaryCell.Balance, stagedBinaryCell.Nonce)

if bph.trace {
fmt.Printf("accountFn reading key %x => balance=%v nonce=%v codeHash=%x\n", cell.apk, cell.Balance.Uint64(), cell.Nonce, cell.CodeHash)
fmt.Printf("accountFn reading key %x => balance=%d nonce=%v codeHash=%x\n", cell.apk, &cell.Balance, cell.Nonce, cell.CodeHash)
}
}
} else {
Expand Down Expand Up @@ -1503,12 +1514,10 @@ func (bph *BinPatriciaHashed) SetState(buf []byte) error {
return err
}

bph.currentKeyLen = int(s.CurrentKeyLen)
bph.rootChecked = s.RootChecked
bph.rootTouched = s.RootTouched
bph.rootPresent = s.RootPresent

copy(bph.currentKey[:], s.CurrentKey[:])
copy(bph.depths[:], s.Depths[:])
copy(bph.branchBefore[:], s.BranchBefore[:])
copy(bph.touchMap[:], s.TouchMap[:])
Expand All @@ -1517,44 +1526,52 @@ func (bph *BinPatriciaHashed) SetState(buf []byte) error {
return nil
}

func (bph *BinPatriciaHashed) ProcessUpdates(plainKeys, hashedKeys [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
func (bph *BinPatriciaHashed) ProcessUpdates(plainKeys [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error) {
branchNodeUpdates = make(map[string]BranchData)

for i, pk := range plainKeys {
updates[i].hashedKey = hexToBin(pk)
updates[i].plainKey = pk
}

sort.Slice(updates, func(i, j int) bool {
return bytes.Compare(updates[i].hashedKey, updates[j].hashedKey) < 0
})

for i, plainKey := range plainKeys {
hashedKey := hashedKeys[i]
update := updates[i]
if bph.trace {
fmt.Printf("plainKey=[%x], hashedKey=[%x], currentKey=[%x]\n", plainKey, hashedKey, bph.currentKey[:bph.currentKeyLen])
fmt.Printf("plainKey=[%x], hashedKey=[%x], currentKey=[%x]\n", update.plainKey, update.hashedKey, bph.currentKey[:bph.currentKeyLen])
}
// Keep folding until the currentKey is the prefix of the key we modify
for bph.needFolding(hashedKey) {
for bph.needFolding(update.hashedKey) {
if branchData, updateKey, err := bph.fold(); err != nil {
return nil, nil, fmt.Errorf("fold: %w", err)
} else if branchData != nil {
branchNodeUpdates[string(updateKey)] = branchData
}
}
// Now unfold until we step on an empty cell
for unfolding := bph.needUnfolding(hashedKey); unfolding > 0; unfolding = bph.needUnfolding(hashedKey) {
if err := bph.unfold(hashedKey, unfolding); err != nil {
for unfolding := bph.needUnfolding(update.hashedKey); unfolding > 0; unfolding = bph.needUnfolding(update.hashedKey) {
if err := bph.unfold(update.hashedKey, unfolding); err != nil {
return nil, nil, fmt.Errorf("unfold: %w", err)
}
}

update := updates[i]
// Update the cell
if update.Flags == DeleteUpdate {
bph.deleteBinaryCell(hashedKey)
bph.deleteBinaryCell(update.hashedKey)
if bph.trace {
fmt.Printf("key %x deleted\n", plainKey)
fmt.Printf("key %x deleted\n", update.plainKey)
}
} else {
cell := bph.updateBinaryCell(plainKey, hashedKey)
cell := bph.updateBinaryCell(update.plainKey, update.hashedKey)
if bph.trace {
fmt.Printf("accountFn updated key %x =>", plainKey)
}
if update.Flags&BalanceUpdate != 0 {
if bph.trace {
fmt.Printf(" balance=%d", update.Balance.Uint64())
fmt.Printf(" balance=%d", &update.Balance)
}
cell.Balance.Set(&update.Balance)
}
Expand Down Expand Up @@ -1603,13 +1620,13 @@ func (bph *BinPatriciaHashed) hashAndNibblizeKey2(key []byte) []byte { //nolint

bph.keccak.Reset()
bph.keccak.Write(key[:length.Addr])
copy(hashedKey[:length.Hash], bph.keccak.Sum(nil))
bph.keccak.Read(hashedKey[:length.Hash])

if len(key[length.Addr:]) > 0 {
hashedKey = append(hashedKey, make([]byte, length.Hash)...)
bph.keccak.Reset()
bph.keccak.Write(key[length.Addr:])
copy(hashedKey[length.Hash:], bph.keccak.Sum(nil))
bph.keccak.Read(hashedKey[length.Hash:])
}

nibblized := make([]byte, len(hashedKey)*2)
Expand Down
48 changes: 24 additions & 24 deletions commitment/bin_patricia_hashed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
trie := NewBinPatriciaHashed(length.Addr, ms.branchFn, ms.accountFn, ms.storageFn)
trieBatch := NewBinPatriciaHashed(length.Addr, ms2.branchFn, ms2.accountFn, ms2.storageFn)

plainKeys, hashedKeys, updates := NewUpdateBuilder().
plainKeys, updates := NewUpdateBuilder().
Balance("e25652aaa6b9417973d325f9a1246b48ff9420bf", 12).
Balance("cdd0a12034e978f7eccda72bd1bd89a8142b704e", 120000).
Balance("5bb6abae12c87592b940458437526cb6cad60d50", 170).
Expand All @@ -43,25 +43,25 @@ func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
fmt.Println("1. Running sequential updates over the bin trie")
var seqHash []byte
for i := 0; i < len(updates); i++ {
sh, branchNodeUpdates, err := trie.ReviewKeys(plainKeys[i:i+1], hashedKeys[i:i+1])
sh, branchNodeUpdates, err := trie.ProcessKeys(plainKeys[i : i+1])
require.NoError(t, err)
require.Len(t, sh, length.Hash)
ms.applyBranchNodeUpdates(branchNodeUpdates)
// WARN! provided sequential branch updates are incorrect - lead to deletion of prefixes (afterMap is zero)
// while root hashes are equal
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)

fmt.Printf("h=%x\n", sh)
seqHash = sh
}

fmt.Println("2. Running batch updates over the bin trie")

batchHash, branchBatchUpdates, err := trieBatch.ReviewKeys(plainKeys, hashedKeys)
batchHash, branchBatchUpdates, err := trieBatch.ProcessKeys(plainKeys)
require.NoError(t, err)
ms2.applyBranchNodeUpdates(branchBatchUpdates)

renderUpdates(branchBatchUpdates)
//renderUpdates(branchBatchUpdates)

require.EqualValues(t, seqHash, batchHash)
// require.EqualValues(t, seqHash, batchHash)
Expand All @@ -88,7 +88,7 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
ms := NewMockState(t)
ms2 := NewMockState(t)

plainKeys, hashedKeys, updates := NewUpdateBuilder().
plainKeys, updates := NewUpdateBuilder().
Balance("f5", 4).
Balance("ff", 900234).
Balance("04", 1233).
Expand Down Expand Up @@ -122,22 +122,22 @@ func Test_BinPatriciaHashed_UniqueRepresentation(t *testing.T) {
t.Fatal(err)
}

sequentialRoot, branchNodeUpdates, err := trieOne.ReviewKeys(plainKeys[i:i+1], hashedKeys[i:i+1])
sequentialRoot, branchNodeUpdates, err := trieOne.ProcessKeys(plainKeys[i : i+1])
require.NoError(t, err)
roots = append(roots, sequentialRoot)

ms.applyBranchNodeUpdates(branchNodeUpdates)
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)
}

err := ms2.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

fmt.Printf("\n2. Trie batch update generated following branch updates\n")
// batch update
batchRoot, branchNodeUpdatesTwo, err := trieTwo.ReviewKeys(plainKeys, hashedKeys)
batchRoot, branchNodeUpdatesTwo, err := trieTwo.ProcessKeys(plainKeys)
require.NoError(t, err)
renderUpdates(branchNodeUpdatesTwo)
//renderUpdates(branchNodeUpdatesTwo)

fmt.Printf("\n sequential roots:\n")
for i, rh := range roots {
Expand All @@ -154,7 +154,7 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
ms := NewMockState(t)
hph := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
hph.SetTrace(false)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Balance("01", 5).
Balance("02", 6).
Expand All @@ -171,56 +171,56 @@ func Test_BinPatriciaHashed_EmptyState(t *testing.T) {
err := ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

firstRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
firstRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
require.NoError(t, err)

t.Logf("root hash %x\n", firstRootHash)

ms.applyBranchNodeUpdates(branchNodeUpdates)

fmt.Printf("1. Generated updates\n")
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)

// More updates
hph.Reset()
hph.SetTrace(false)
plainKeys, hashedKeys, updates = NewUpdateBuilder().
plainKeys, updates = NewUpdateBuilder().
Storage("03", "58", "050505").
Build()
err = ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

secondRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
secondRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
require.NoError(t, err)
require.NotEqualValues(t, firstRootHash, secondRootHash)

ms.applyBranchNodeUpdates(branchNodeUpdates)
fmt.Printf("2. Generated single update\n")
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)

// More updates
hph.Reset()
hph.SetTrace(false)
plainKeys, hashedKeys, updates = NewUpdateBuilder().
plainKeys, updates = NewUpdateBuilder().
Storage("03", "58", "070807").
Build()
err = ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

thirdRootHash, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
thirdRootHash, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
require.NoError(t, err)
require.NotEqualValues(t, secondRootHash, thirdRootHash)

ms.applyBranchNodeUpdates(branchNodeUpdates)
fmt.Printf("3. Generated single update\n")
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)
}

func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) {
ms := NewMockState(t)
hph := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
hph.SetTrace(false)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
plainKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Nonce("00", 246462653).
Balance("01", 5).
Expand All @@ -233,24 +233,24 @@ func Test_BinPatriciaHashed_EmptyUpdateState(t *testing.T) {
err := ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

hashBeforeEmptyUpdate, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
hashBeforeEmptyUpdate, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
require.NoError(t, err)
require.NotEmpty(t, hashBeforeEmptyUpdate)

ms.applyBranchNodeUpdates(branchNodeUpdates)

fmt.Println("1. Updates applied")
renderUpdates(branchNodeUpdates)
//renderUpdates(branchNodeUpdates)

// generate empty updates and do NOT reset tree
hph.SetTrace(true)

plainKeys, hashedKeys, updates = NewUpdateBuilder().Build()
plainKeys, updates = NewUpdateBuilder().Build()

err = ms.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)

hashAfterEmptyUpdate, branchNodeUpdates, err := hph.ReviewKeys(plainKeys, hashedKeys)
hashAfterEmptyUpdate, branchNodeUpdates, err := hph.ProcessKeys(plainKeys)
require.NoError(t, err)

ms.applyBranchNodeUpdates(branchNodeUpdates)
Expand Down
12 changes: 8 additions & 4 deletions commitment/commitment.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ type Trie interface {
// Reset Drops everything from the trie
Reset()

ReviewKeys(pk, hk [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
// Reads updates from storage
ProcessKeys(pk [][]byte) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)

ProcessUpdates(pk, hk [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)
ProcessUpdates(pk [][]byte, updates []Update) (rootHash []byte, branchNodeUpdates map[string]BranchData, err error)

ResetFns(
branchFn func(prefix []byte) ([]byte, error),
Expand Down Expand Up @@ -70,6 +71,9 @@ const (
type BranchData []byte

func (branchData BranchData) String() string {
if len(branchData) == 0 {
return ""
}
touchMap := binary.BigEndian.Uint16(branchData[0:])
afterMap := binary.BigEndian.Uint16(branchData[2:])
pos := 4
Expand Down Expand Up @@ -468,10 +472,10 @@ func NewHexBranchMerger(capacity uint64) *BranchMerger {

// MergeHexBranches combines two branchData, number 2 coming after (and potentially shadowing) number 1
func (m *BranchMerger) Merge(branch1 BranchData, branch2 BranchData) (BranchData, error) {
if branch2 == nil {
if len(branch2) == 0 {
return branch1, nil
}
if branch1 == nil {
if len(branch1) == 0 {
return branch2, nil
}

Expand Down
Loading
Loading