From b23251e5ab55ef7d1303d8c61d22c1da1b7d897d Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 3 Jan 2024 12:11:31 -0300 Subject: [PATCH 01/35] feat(pkg/trie): Add basic structures --- pkg/trie/hashdb/hashdb.go | 49 +++++++++++++++++++++ pkg/trie/triedb/cache.go | 30 +++++++++++++ pkg/trie/triedb/nibble/nibble.go | 8 ++++ pkg/trie/triedb/node.go | 73 ++++++++++++++++++++++++++++++++ pkg/trie/triedb/node_codec.go | 6 +++ pkg/trie/triedb/triedb.go | 12 ++++++ 6 files changed, 178 insertions(+) create mode 100644 pkg/trie/hashdb/hashdb.go create mode 100644 pkg/trie/triedb/cache.go create mode 100644 pkg/trie/triedb/nibble/nibble.go create mode 100644 pkg/trie/triedb/node.go create mode 100644 pkg/trie/triedb/node_codec.go create mode 100644 pkg/trie/triedb/triedb.go diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go new file mode 100644 index 0000000000..4426890678 --- /dev/null +++ b/pkg/trie/hashdb/hashdb.go @@ -0,0 +1,49 @@ +package hashdb + +type HasherOut interface { + comparable + ToBytes() []byte +} + +// / A trie node prefix, it is the nibble path from the trie root +// / to the trie node. +// / For a node containing no partial key value it is the full key. +// / For a value node or node containing a partial key, it is the full key minus its node partial +// / nibbles (the node key can be split into prefix and node partial). +// / Therefore it is always the leftmost portion of the node key, so its internal representation +// / is a non expanded byte slice followed by a last padded byte representation. +// / The padded byte is an optional padded value. +type Prefix struct { + partialKey []byte + paddedByte byte +} + +type Hasher[Out HasherOut] interface { + Length() int + Hash(value []byte) Out +} + +type PlainDB[K any, V any] interface { + Get(key K) *V + Contains(key K) bool + Emplace(key K, value V) + Remove(key K) +} + +type PlainDBReadOnly[K any, V any] interface { + Get(key K) *V + Contains(key K) bool +} + +type HashDB[Out HasherOut, T any] interface { + Get(key Out, prefix Prefix) *T + Contains(key Out, prefix Prefix) bool + Insert(prefix Prefix, value []byte) Out + Emplace(key Out, prefix Prefix, value T) + remove(key Out, prefix Prefix) +} + +type HashDBReadOnly[Out HasherOut, T any] interface { + Get(key Out, prefix Prefix) *T + Contains(key Out, prefix Prefix) bool +} diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go new file mode 100644 index 0000000000..e79bb6ca83 --- /dev/null +++ b/pkg/trie/triedb/cache.go @@ -0,0 +1,30 @@ +package triedb + +// CachedValue a value as cached by TrieCache +type CachedValue[H comparable] interface { + Type() string +} +type ( + // The value doesn't exists in ithe trie + NonExisting struct{} + // We cached the hash, because we did not yet accessed the data + ExistingHash[H comparable] struct { + hash H + } + // The value xists in the trie + Existing[H comparable] struct { + hash H // The hash of the value + data []byte // The actual data of the value + } +) + +func (v NonExisting) Type() string { return "NonExisting" } +func (v ExistingHash[H]) Type() string { return "ExistingHash" } +func (v Existing[H]) Type() string { return "Existing" } + +type TrieCache[Out HashOut] interface { + LookupValueForKey(key []byte) *CachedValue[Out] + CacheValueForKey(key []byte, value CachedValue[Out]) + GetOrInsertNode(hash Out, fetchNode func() (Node[Out], error)) + GetNode(hash Out) Node[Out] +} diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go new file mode 100644 index 0000000000..75682773c1 --- /dev/null +++ b/pkg/trie/triedb/nibble/nibble.go @@ -0,0 +1,8 @@ +package nibble + +const NibbleLength = 16 + +type NibbleVec struct { + inner []byte + len uint +} diff --git a/pkg/trie/triedb/node.go b/pkg/trie/triedb/node.go new file mode 100644 index 0000000000..f70c692c53 --- /dev/null +++ b/pkg/trie/triedb/node.go @@ -0,0 +1,73 @@ +package triedb + +import ( + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" +) + +// Nodes + +// Node is a trie node +type Node[H HashOut] interface { + Type() string +} + +type ( + // NodeEmptyNode represents an empty node + Empty struct{} + // NodeLeaf represents a leaf node + Leaf[H HashOut] struct { + nibble nibble.NibbleVec + value Value[H] + } + // NodeNibbledBranch represents a branch node + NibbledBranch[H HashOut] struct { + nibble nibble.NibbleVec + childs [nibble.NibbleLength]NodeHandle[H] + value Value[H] + } +) + +func (n Empty) Type() string { return "Empty" } +func (n Leaf[H]) Type() string { return "Leaf" } +func (n NibbledBranch[H]) Type() string { return "NibbledBranch" } + +// Value is a trie node value +type Value[H HashOut] interface { + Type() string + Hash() H + Value() []byte +} +type ( + // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value + InlineValue[H HashOut] struct { + bytes []byte + hash H + } + // HashedNodeValue is a trie node pointer to a hashed node + HashedValue[H comparable] struct { + hash H + } +) + +func (v InlineValue[H]) Type() string { return "Inline" } +func (v InlineValue[H]) Hash() H { return v.hash } +func (v InlineValue[H]) Value() []byte { return v.bytes } +func (v HashedValue[H]) Type() string { return "Node" } +func (v HashedValue[H]) Hash() H { return v.hash } +func (v HashedValue[H]) Value() []byte { return nil } + +// NodeHandle is a reference to a trie node which may be stored within another trie node. +type NodeHandle[H HashOut] interface { + Type() string +} +type ( + HashNodeHandle[H HashOut] struct { + value H + } + InlineNodeHandle[H HashOut] struct { + node Node[H] + } +) + +func (h HashNodeHandle[H]) Type() string { return "Hash" } +func (h InlineNodeHandle[H]) Type() string { return "Inline" } diff --git a/pkg/trie/triedb/node_codec.go b/pkg/trie/triedb/node_codec.go new file mode 100644 index 0000000000..e3e654e4bc --- /dev/null +++ b/pkg/trie/triedb/node_codec.go @@ -0,0 +1,6 @@ +package triedb + +type HashOut interface { + comparable + ToBytes() []byte +} diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go new file mode 100644 index 0000000000..378748513e --- /dev/null +++ b/pkg/trie/triedb/triedb.go @@ -0,0 +1,12 @@ +package triedb + +import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + +type DBValue = []byte + +type TrieDBBuilder[Out HashOut] struct { + db hashdb.HashDBReadOnly[Out, DBValue] + root Out + cache TrieCache[Out] + //recorder +} From af43f27c9c8833e97665b9e3efccec1d4afa5bc9 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 3 Jan 2024 12:35:27 -0300 Subject: [PATCH 02/35] feat(pkg/trie): Add recorder interface --- pkg/trie/triedb/recorder.go | 68 +++++++++++++++++++++++++++++++++++++ pkg/trie/triedb/triedb.go | 8 ++--- 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 pkg/trie/triedb/recorder.go diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go new file mode 100644 index 0000000000..7becbe5484 --- /dev/null +++ b/pkg/trie/triedb/recorder.go @@ -0,0 +1,68 @@ +package triedb + +type RecordedForKey uint8 + +const ( + RecordedForKeyValue RecordedForKey = iota + RecordedForKeyHash + RecordedForKeyNone +) + +type TrieRecorder[Out HashOut] interface { + record(access TrieAccess[Out]) + trieNodesRecordedForKey(key []byte) RecordedForKey +} + +// TrieAccess is used to report the trie access to the TrieRecorder +type TrieAccess[Out HashOut] interface { + Type() string +} + +type ( + // TrieAccessNode means that the given node was accessed using its hash + TrieAccessNode[H HashOut] struct { + hash H + node Node[H] + } + + // TrieAccessEncodedNode means that the given encodedNode was accessed using its hash + TrieAccessEncodedNode[H HashOut] struct { + hash H + encodedNode []byte + } + + // TrieAccessValue means that the given value was accessed using its hash + // fullKey is the key to access this value in the trie + // Should map to RecordedForKeyValue when checking the recorder + TrieAccessValue[H HashOut] struct { + hash H + value []byte + fullKey []byte + } + + // TrieAccessInlineValue means that a value stored in an inlined node was accessed + // The given fullKey is the key to access this value in the trie + // Should map to RecordedForKeyValue when checking the recorder + TrieAccessInlineValue[H HashOut] struct { + fullKey []byte + } + + // TrieAccessHash means that the hash of the value for a given fullKey was accessed + // Should map to RecordedForKeyHash when checking the recorder + TrieAccessHash[H HashOut] struct { + fullKey []byte + } + + // TrieAccessNotExisting means that the value/hash for fullKey was accessed, but it couldn't be found in the trie + // Should map to RecordedForKeyValue when checking the recorder + TrieAccessNotExisting struct { + fullKey []byte + } +) + +func (a TrieAccessNode[H]) Type() string { return "Node" } +func (a TrieAccessEncodedNode[H]) Type() string { return "EncodedNode" } +func (a TrieAccessValue[H]) Type() string { return "Value" } +func (a TrieAccessInlineValue[H]) Type() string { return "InlineValue" } +func (a TrieAccessHash[H]) Type() string { return "Hash" } +func (a TrieAccessNotExisting) Type() string { return "NotExisting" } diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 378748513e..ffd844955b 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -5,8 +5,8 @@ import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" type DBValue = []byte type TrieDBBuilder[Out HashOut] struct { - db hashdb.HashDBReadOnly[Out, DBValue] - root Out - cache TrieCache[Out] - //recorder + db hashdb.HashDBReadOnly[Out, DBValue] + root Out + cache TrieCache[Out] + recorder TrieRecorder[Out] } From e2baf3f16b5cb6afa44361e2a3d9d383718498ae Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 3 Jan 2024 14:36:05 -0300 Subject: [PATCH 03/35] feat(pkg/trie): Finish recorder impl and add node codec and layout --- pkg/trie/triedb/layout.go | 11 ++++++ pkg/trie/triedb/nibble/nibble.go | 8 +++++ pkg/trie/triedb/node.go | 10 +++--- pkg/trie/triedb/node_codec.go | 22 ++++++++++++ pkg/trie/triedb/recorder.go | 61 ++++++++++++++++++++++++++++++-- 5 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 pkg/trie/triedb/layout.go diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go new file mode 100644 index 0000000000..4586ee2457 --- /dev/null +++ b/pkg/trie/triedb/layout.go @@ -0,0 +1,11 @@ +package triedb + +import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + +type TrieLayout[Out HashOut] interface { + UseExtension() bool + AllowEmpty() bool + MaxInlineValue() *uint + Hasher() hashdb.Hasher[Out] + Codec() NodeCodec[Out] +} diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 75682773c1..7012060ecc 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -6,3 +6,11 @@ type NibbleVec struct { inner []byte len uint } + +func (n NibbleVec) RightIter() []byte { + return n.inner +} + +func (n NibbleVec) Len() uint { + return n.len +} diff --git a/pkg/trie/triedb/node.go b/pkg/trie/triedb/node.go index f70c692c53..67f28abea7 100644 --- a/pkg/trie/triedb/node.go +++ b/pkg/trie/triedb/node.go @@ -16,14 +16,14 @@ type ( Empty struct{} // NodeLeaf represents a leaf node Leaf[H HashOut] struct { - nibble nibble.NibbleVec - value Value[H] + partialKey nibble.NibbleVec + value Value[H] } // NodeNibbledBranch represents a branch node NibbledBranch[H HashOut] struct { - nibble nibble.NibbleVec - childs [nibble.NibbleLength]NodeHandle[H] - value Value[H] + partialKey nibble.NibbleVec + childs [nibble.NibbleLength]NodeHandle[H] + value Value[H] } ) diff --git a/pkg/trie/triedb/node_codec.go b/pkg/trie/triedb/node_codec.go index e3e654e4bc..8d77396d50 100644 --- a/pkg/trie/triedb/node_codec.go +++ b/pkg/trie/triedb/node_codec.go @@ -1,6 +1,28 @@ package triedb +import "fmt" + type HashOut interface { comparable ToBytes() []byte } + +type NodeCodec[H HashOut] interface { + HashedNullNode() H + EmptyNode() []byte + LeafNode(partialKey []byte, numberNibble uint, value Value[H]) []byte + BranchNodeNibbled(partialKey []byte, numberNibble uint, children [16]NodeHandle[H], value Value[H]) []byte +} + +func EncodeNode[H HashOut](node Node[H], codec NodeCodec[H]) []byte { + switch n := node.(type) { + case Empty: + return codec.EmptyNode() + case Leaf[H]: + return codec.LeafNode(n.partialKey.RightIter(), n.partialKey.Len(), n.value) + case NibbledBranch[H]: + return codec.BranchNodeNibbled(n.partialKey.RightIter(), n.partialKey.Len(), n.childs, n.value) + default: + panic(fmt.Sprintf("unknown node type %s", n.Type())) + } +} diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index 7becbe5484..c13f128bd1 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -1,5 +1,7 @@ package triedb +import "fmt" + type RecordedForKey uint8 const ( @@ -53,9 +55,9 @@ type ( fullKey []byte } - // TrieAccessNotExisting means that the value/hash for fullKey was accessed, but it couldn't be found in the trie + // TrieAccessNonExisting means that the value/hash for fullKey was accessed, but it couldn't be found in the trie // Should map to RecordedForKeyValue when checking the recorder - TrieAccessNotExisting struct { + TrieAccessNonExisting struct { fullKey []byte } ) @@ -65,4 +67,57 @@ func (a TrieAccessEncodedNode[H]) Type() string { return "EncodedNode" } func (a TrieAccessValue[H]) Type() string { return "Value" } func (a TrieAccessInlineValue[H]) Type() string { return "InlineValue" } func (a TrieAccessHash[H]) Type() string { return "Hash" } -func (a TrieAccessNotExisting) Type() string { return "NotExisting" } +func (a TrieAccessNonExisting) Type() string { return "NotExisting" } + +// Recorder implementation + +type Record[H HashOut] struct { + /// The hash of the node. + Hash H + /// The data representing the node. + Data []byte +} + +type Recorder[H HashOut] struct { + nodes []Record[H] + recorderKeys map[string]RecordedForKey // TODO: revisit this later, it should be a BTreeMap + layout TrieLayout[H] +} + +// NewRecorder creates a new Recorder which records all given nodes +func NewRecorder[H HashOut]() *Recorder[H] { + return &Recorder[H]{ + nodes: make([]Record[H], 0), + recorderKeys: make(map[string]RecordedForKey), + } +} + +// Drain drains all visited records +func (r *Recorder[H]) Drain() { + clear(r.nodes) + clear(r.recorderKeys) +} + +// Impl of TrieRecorder for Recorder +func (r *Recorder[H]) record(access TrieAccess[H]) { + switch access := access.(type) { + case TrieAccessEncodedNode[H]: + r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.encodedNode}) + case TrieAccessNode[H]: + r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: EncodeNode(access.node, r.layout.Codec())}) + case TrieAccessValue[H]: + r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.value}) + r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue + case TrieAccessHash[H]: + if _, inserted := r.recorderKeys[string(access.fullKey)]; !inserted { + r.recorderKeys[string(access.fullKey)] = RecordedForKeyHash + } + case TrieAccessNonExisting: + // We handle the non existing value/hash like having recorded the value. + r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue + case TrieAccessInlineValue[H]: + r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue + default: + panic(fmt.Sprintf("unknown access type %s", access.Type())) + } +} From 1fd73469e0cd779dac82c6f0203bde037ae78169 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 3 Jan 2024 14:41:45 -0300 Subject: [PATCH 04/35] fix(pkg/trie): fix recorder drain method --- pkg/trie/triedb/recorder.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index c13f128bd1..1c7bbcb2d2 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -93,9 +93,16 @@ func NewRecorder[H HashOut]() *Recorder[H] { } // Drain drains all visited records -func (r *Recorder[H]) Drain() { +func (r *Recorder[H]) Drain() []Record[H] { + // Store temporal nodes + nodes := make([]Record[H], len(r.nodes)) + copy(nodes, r.nodes) + + // Clean up internal data and return the nodes clear(r.nodes) clear(r.recorderKeys) + + return nodes } // Impl of TrieRecorder for Recorder From 509b51d1b1ed5adcc7115c7a447d60ea01f915da Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 3 Jan 2024 17:35:47 -0300 Subject: [PATCH 05/35] feat(pkg/trie): Add lookup --- pkg/trie/hashdb/hashdb.go | 31 +++--- pkg/trie/memorydb/memorydb.go | 19 ++++ pkg/trie/triedb/errors.go | 8 ++ pkg/trie/triedb/lookup.go | 137 ++++++++++++++++++++++++++ pkg/trie/triedb/nibble/nibble.go | 49 +++++++-- pkg/trie/triedb/nibble/nibbleslice.go | 110 +++++++++++++++++++++ pkg/trie/triedb/node.go | 121 +++++++++++++++++------ pkg/trie/triedb/node_codec.go | 19 ++-- pkg/trie/triedb/trie.go | 12 +++ pkg/trie/triedb/triedb.go | 20 +++- pkg/trie/triedb/triedbbuilder.go | 38 +++++++ 11 files changed, 496 insertions(+), 68 deletions(-) create mode 100644 pkg/trie/memorydb/memorydb.go create mode 100644 pkg/trie/triedb/errors.go create mode 100644 pkg/trie/triedb/lookup.go create mode 100644 pkg/trie/triedb/nibble/nibbleslice.go create mode 100644 pkg/trie/triedb/trie.go create mode 100644 pkg/trie/triedb/triedbbuilder.go diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go index 4426890678..be1125e6da 100644 --- a/pkg/trie/hashdb/hashdb.go +++ b/pkg/trie/hashdb/hashdb.go @@ -14,13 +14,14 @@ type HasherOut interface { // / is a non expanded byte slice followed by a last padded byte representation. // / The padded byte is an optional padded value. type Prefix struct { - partialKey []byte - paddedByte byte + PartialKey []byte + PaddedByte *byte } -type Hasher[Out HasherOut] interface { +type Hasher[Hash HasherOut] interface { Length() int - Hash(value []byte) Out + Hash(value []byte) Hash + FromBytes(value []byte) Hash } type PlainDB[K any, V any] interface { @@ -30,20 +31,10 @@ type PlainDB[K any, V any] interface { Remove(key K) } -type PlainDBReadOnly[K any, V any] interface { - Get(key K) *V - Contains(key K) bool -} - -type HashDB[Out HasherOut, T any] interface { - Get(key Out, prefix Prefix) *T - Contains(key Out, prefix Prefix) bool - Insert(prefix Prefix, value []byte) Out - Emplace(key Out, prefix Prefix, value T) - remove(key Out, prefix Prefix) -} - -type HashDBReadOnly[Out HasherOut, T any] interface { - Get(key Out, prefix Prefix) *T - Contains(key Out, prefix Prefix) bool +type HashDB[Hash HasherOut, T any] interface { + Get(key Hash, prefix Prefix) *T + Contains(key Hash, prefix Prefix) bool + Insert(prefix Prefix, value []byte) Hash + Emplace(key Hash, prefix Prefix, value T) + remove(key Hash, prefix Prefix) } diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go new file mode 100644 index 0000000000..23cb846f0d --- /dev/null +++ b/pkg/trie/memorydb/memorydb.go @@ -0,0 +1,19 @@ +package memorydb + +import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + +type MemoryDBValue[T any] struct { + value T + rc int32 +} + +type MemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any] struct { + data map[Hash]MemoryDBValue[T] + hashedNullNode Hash + nullNodeData T + keyFunction KF +} + +type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { + Key(hash Hash, prefix hashdb.Prefix) Hash +} diff --git a/pkg/trie/triedb/errors.go b/pkg/trie/triedb/errors.go new file mode 100644 index 0000000000..085f07bc9b --- /dev/null +++ b/pkg/trie/triedb/errors.go @@ -0,0 +1,8 @@ +package triedb + +import "errors" + +var ErrInvalidStateRoot = errors.New("invalid state root") +var ErrIncompleteDB = errors.New("incomplete database") +var DecoderError = errors.New("corrupt trie item") +var InvalidHash = errors.New("hash is not value") diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go new file mode 100644 index 0000000000..ca8730f467 --- /dev/null +++ b/pkg/trie/triedb/lookup.go @@ -0,0 +1,137 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package triedb + +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" +) + +var EmptyValue = []byte{} + +type Lookup[Hash HashOut] struct { + db hashdb.HashDB[Hash, DBValue] + hash Hash + cache TrieCache[Hash] + recorder TrieRecorder[Hash] + layout TrieLayout[Hash] +} + +func NewLookup[H HashOut]( + db hashdb.HashDB[H, DBValue], hash H, cache TrieCache[H], recorder TrieRecorder[H]) *Lookup[H] { + return &Lookup[H]{ + db: db, + hash: hash, + cache: cache, + recorder: recorder, + } +} + +func (l Lookup[H]) Lookup(nibbleKey *nibble.NibbleSlice) ([]byte, error) { + return l.lookupWithoutCache(nibbleKey) +} + +func (l Lookup[H]) record(access TrieAccess[H]) { + if l.recorder != nil { + l.recorder.record(access) + } +} + +func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, error) { + partial := nibbleKey + hash := l.hash + keyNibbles := uint(0) + + depth := 0 + + for { + // Get node from DB + nodeData := l.db.Get(hash, nibbleKey.Mid(keyNibbles).Left()) + if nodeData == nil { + if depth == 0 { + return nil, ErrInvalidStateRoot + } else { + return nil, ErrIncompleteDB + } + } + + l.record(TrieAccessEncodedNode[H]{ + hash: hash, + encodedNode: *nodeData, + }) + + // Iterates children + for { + // Decode node + decodedNode, err := l.layout.Codec().Decode(*nodeData) + if err != nil { + return nil, DecoderError + } + + var nextNode NodeHandle = nil + + switch node := decodedNode.(type) { + case Empty: + return EmptyValue, nil + case Leaf: + // If leaf and matches return value + if partial.Eq(&node.partialKey) { + return l.loadValue(node.value, nibbleKey.OriginalDataAsPrefix()) + } + return EmptyValue, nil + case NibbledBranch: + slice := node.partialKey + children := node.children + value := node.value + // Get next node + if !partial.StartsWith(&slice) { + return EmptyValue, nil + } + + if partial.Len() == slice.Len() { + if value != nil { + return l.loadValue(value, nibbleKey.OriginalDataAsPrefix()) + } + } + + nextNode = children[partial.At(slice.Len())] + if nextNode == nil { + return EmptyValue, nil + } + + partial = partial.Mid(slice.Len() + 1) + keyNibbles += slice.Len() + 1 + } + + switch node := nextNode.(type) { + case Hash: + nextHash := DecodeHash(node.value, l.layout.Hasher()) + if nextHash == nil { + return nil, InvalidHash + } + hash = *nextHash + break + case Inline: + nodeData = &node.value + } + } + depth++ + } +} + +func (l Lookup[H]) loadValue(value Value, prefix hashdb.Prefix) ([]byte, error) { + switch v := value.(type) { + case InlineValue: + return v.bytes, nil + case NodeValue: + hash := l.layout.Hasher().FromBytes(v.bytes) + bytes := l.db.Get(hash, prefix) + if bytes == nil { + return nil, ErrIncompleteDB + } + return *bytes, nil + default: + panic("unknown value type") + } +} diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 7012060ecc..8389e4ec68 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -1,16 +1,51 @@ package nibble +const NibblePerByte uint = 2 +const PaddingBitmask byte = 0x0F +const BitPerNibble = 4 const NibbleLength = 16 -type NibbleVec struct { - inner []byte - len uint +func padLeft(b byte) byte { + padded := (b & ^PaddingBitmask) + return padded } -func (n NibbleVec) RightIter() []byte { - return n.inner +func padRight(b byte) byte { + padded := (b & PaddingBitmask) + return padded } -func (n NibbleVec) Len() uint { - return n.len +func NumberPadding(i uint) uint { + return i % NibblePerByte +} + +// Count the biggest common depth between two left aligned packed nibble slice +func biggestDepth(v1, v2 []byte) uint { + upperBound := minLength(v1, v2) + + for i := uint(0); i < upperBound; i++ { + if v1[i] != v2[i] { + return i*NibblePerByte + leftCommon(v1[i], v2[i]) + } + } + return upperBound * NibblePerByte +} + +// LeftCommon the number of common nibble between two left aligned bytes +func leftCommon(a, b byte) uint { + if a == b { + return 2 + } + if padLeft(a) == padLeft(b) { + return 1 + } else { + return 0 + } +} + +func minLength(v1, v2 []byte) uint { + if len(v1) < len(v2) { + return uint(len(v1)) + } + return uint(len(v2)) } diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go new file mode 100644 index 0000000000..6cf93093e0 --- /dev/null +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -0,0 +1,110 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package nibble + +import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + +// NibbleSlice is a helper structure to store a slice of nibbles and a moving offset +// this is helpful to use it for example while we are looking for a key, we can define the full key in the data and +// moving the offset while we are going deep in the trie +type NibbleSlice struct { + data []byte + offset uint +} + +func NewNibbleSlice(data []byte) *NibbleSlice { + return &NibbleSlice{data, 0} +} + +func NewNibbleSliceWithPadding(data []byte, padding uint) *NibbleSlice { + return &NibbleSlice{data, padding} +} + +func (ns *NibbleSlice) Data() []byte { + return ns.data +} + +func (ns *NibbleSlice) Offset() uint { + return ns.offset +} + +func (ns *NibbleSlice) Mid(i uint) *NibbleSlice { + return &NibbleSlice{ns.data, ns.offset + i} +} + +func (ns *NibbleSlice) Len() uint { + return uint(len(ns.data))*NibblePerByte - ns.offset +} + +func (ns *NibbleSlice) At(i uint) byte { + ix := (ns.offset + i) / NibblePerByte + pad := (ns.offset + i) % NibblePerByte + b := ns.data[ix] + if pad == 1 { + return b & PaddingBitmask + } + return b >> BitPerNibble +} + +func (ns *NibbleSlice) StartsWith(other *NibbleSlice) bool { + return ns.commonPrefix(other) == other.Len() +} + +func (ns *NibbleSlice) Eq(other *NibbleSlice) bool { + return ns.Len() == other.Len() && ns.StartsWith(other) +} + +func (ns *NibbleSlice) commonPrefix(other *NibbleSlice) uint { + selfAlign := ns.offset % NibblePerByte + otherAlign := other.offset % NibblePerByte + if selfAlign == otherAlign { + selfStart := ns.offset / NibblePerByte + otherStart := other.offset / NibblePerByte + first := uint(0) + if selfAlign != 0 { + if padRight(ns.data[selfStart]) != padRight(other.data[otherStart]) { + return 0 + } + selfStart++ + otherStart++ + first++ + } + return biggestDepth(ns.data[selfStart:], other.data[otherStart:]) + first + } + + s := minLength(ns.data, other.data) + i := uint(0) + for i < s { + if ns.At(i) != other.At(i) { + break + } + i++ + } + return i +} + +func (ns *NibbleSlice) Left() hashdb.Prefix { + split := ns.offset / NibblePerByte + ix := (ns.offset % NibblePerByte) + if ix == 0 { + return hashdb.Prefix{ + PartialKey: ns.data[:split], + PaddedByte: nil, + } + } else { + padded := padRight(ns.data[split]) + + return hashdb.Prefix{ + PartialKey: ns.data[:split], + PaddedByte: &padded, + } + } +} + +func (ns *NibbleSlice) OriginalDataAsPrefix() hashdb.Prefix { + return hashdb.Prefix{ + PartialKey: ns.data, + PaddedByte: nil, + } +} diff --git a/pkg/trie/triedb/node.go b/pkg/trie/triedb/node.go index 67f28abea7..5f5fe2af53 100644 --- a/pkg/trie/triedb/node.go +++ b/pkg/trie/triedb/node.go @@ -1,12 +1,30 @@ package triedb import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) -// Nodes +// Value +type Value interface { + Type() string +} -// Node is a trie node +type ( + // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value + InlineValue struct { + bytes []byte + } + // HashedNodeValue is a trie node pointer to a hashed node + NodeValue struct { + bytes []byte + } +) + +func (v InlineValue) Type() string { return "Inline" } +func (v NodeValue) Type() string { return "Node" } + +// Nodes type Node[H HashOut] interface { Type() string } @@ -15,59 +33,102 @@ type ( // NodeEmptyNode represents an empty node Empty struct{} // NodeLeaf represents a leaf node - Leaf[H HashOut] struct { - partialKey nibble.NibbleVec - value Value[H] + Leaf struct { + partialKey nibble.NibbleSlice + value Value + } + // NodeNibbledBranch represents a branch node + NibbledBranch struct { + partialKey nibble.NibbleSlice + children [nibble.NibbleLength]NodeHandle + value Value + } +) + +func (n Empty) Type() string { return "Empty" } +func (n Leaf) Type() string { return "Leaf" } +func (n NibbledBranch) Type() string { return "NibbledBranch" } + +// NodeOwned is a trie node +type NodeOwned[H HashOut] interface { + Type() string +} + +type ( + // NodeEmptyNode represents an empty node + NodeOwnedEmpty struct{} + // NodeLeaf represents a leaf node + NodeOwnedLeaf[H HashOut] struct { + partialKey nibble.NibbleSlice + value ValueOwned[H] } // NodeNibbledBranch represents a branch node - NibbledBranch[H HashOut] struct { - partialKey nibble.NibbleVec - childs [nibble.NibbleLength]NodeHandle[H] - value Value[H] + NodeOwnedNibbledBranch[H HashOut] struct { + partialKey nibble.NibbleSlice + children [nibble.NibbleLength]NodeHandleOwned[H] + value ValueOwned[H] } ) -func (n Empty) Type() string { return "Empty" } -func (n Leaf[H]) Type() string { return "Leaf" } -func (n NibbledBranch[H]) Type() string { return "NibbledBranch" } +func (n NodeOwnedEmpty) Type() string { return "Empty" } +func (n NodeOwnedLeaf[H]) Type() string { return "Leaf" } +func (n NodeOwnedNibbledBranch[H]) Type() string { return "NibbledBranch" } // Value is a trie node value -type Value[H HashOut] interface { +type ValueOwned[H HashOut] interface { Type() string - Hash() H - Value() []byte } type ( // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value - InlineValue[H HashOut] struct { + InlineValueOwned[H HashOut] struct { bytes []byte hash H } // HashedNodeValue is a trie node pointer to a hashed node - HashedValue[H comparable] struct { + NodeValueOwned[H comparable] struct { hash H } ) -func (v InlineValue[H]) Type() string { return "Inline" } -func (v InlineValue[H]) Hash() H { return v.hash } -func (v InlineValue[H]) Value() []byte { return v.bytes } -func (v HashedValue[H]) Type() string { return "Node" } -func (v HashedValue[H]) Hash() H { return v.hash } -func (v HashedValue[H]) Value() []byte { return nil } +func (v InlineValueOwned[H]) Type() string { return "Inline" } +func (v NodeValueOwned[H]) Type() string { return "Node" } // NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandle[H HashOut] interface { +type NodeHandleOwned[H HashOut] interface { Type() string } type ( - HashNodeHandle[H HashOut] struct { - value H + NodeHandleOwnedHash[H HashOut] struct { + ValueOwned H } - InlineNodeHandle[H HashOut] struct { - node Node[H] + NodeHandleOwnedInline[H HashOut] struct { + node NodeOwned[H] } ) -func (h HashNodeHandle[H]) Type() string { return "Hash" } -func (h InlineNodeHandle[H]) Type() string { return "Inline" } +func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } +func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } + +// NodeHandle is a reference to a trie node which may be stored within another trie node. +type NodeHandle interface { + Type() string +} +type ( + Hash struct { + value []byte + } + Inline struct { + value []byte + } +) + +func (h Hash) Type() string { return "Hash" } +func (h Inline) Type() string { return "Inline" } + +func DecodeHash[H HashOut](data []byte, hasher hashdb.Hasher[H]) *H { + if len(data) != hasher.Length() { + return nil + } + hash := hasher.FromBytes(data) + return &hash +} diff --git a/pkg/trie/triedb/node_codec.go b/pkg/trie/triedb/node_codec.go index 8d77396d50..be648acfb5 100644 --- a/pkg/trie/triedb/node_codec.go +++ b/pkg/trie/triedb/node_codec.go @@ -1,6 +1,10 @@ package triedb -import "fmt" +import ( + "fmt" + + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" +) type HashOut interface { comparable @@ -10,18 +14,19 @@ type HashOut interface { type NodeCodec[H HashOut] interface { HashedNullNode() H EmptyNode() []byte - LeafNode(partialKey []byte, numberNibble uint, value Value[H]) []byte - BranchNodeNibbled(partialKey []byte, numberNibble uint, children [16]NodeHandle[H], value Value[H]) []byte + LeafNode(partialKey nibble.NibbleSlice, numberNibble uint, value Value) []byte + BranchNodeNibbled(partialKey nibble.NibbleSlice, numberNibble uint, children [16]NodeHandle, value Value) []byte + Decode(data []byte) (Node[H], error) } func EncodeNode[H HashOut](node Node[H], codec NodeCodec[H]) []byte { switch n := node.(type) { case Empty: return codec.EmptyNode() - case Leaf[H]: - return codec.LeafNode(n.partialKey.RightIter(), n.partialKey.Len(), n.value) - case NibbledBranch[H]: - return codec.BranchNodeNibbled(n.partialKey.RightIter(), n.partialKey.Len(), n.childs, n.value) + case Leaf: + return codec.LeafNode(n.partialKey, n.partialKey.Len(), n.value) + case NibbledBranch: + return codec.BranchNodeNibbled(n.partialKey, n.partialKey.Len(), n.children, n.value) default: panic(fmt.Sprintf("unknown node type %s", n.Type())) } diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go new file mode 100644 index 0000000000..77d0a40439 --- /dev/null +++ b/pkg/trie/triedb/trie.go @@ -0,0 +1,12 @@ +package triedb + +type Trie[Hash HashOut] interface { + Root() Hash + IsEmpty() bool + Contains(key []byte) (bool, error) + GetHash(key []byte) (*Hash, error) + Get(key []byte) (*DBValue, error) + //TODO: + //get_with + //lookup_first_descendant +} diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index ffd844955b..ea55e4b4ee 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -2,11 +2,23 @@ package triedb import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" -type DBValue = []byte - -type TrieDBBuilder[Out HashOut] struct { - db hashdb.HashDBReadOnly[Out, DBValue] +type TrieDB[Out HashOut] struct { + db hashdb.HashDB[Out, DBValue] root Out cache TrieCache[Out] recorder TrieRecorder[Out] } + +func NewTrieDB[H HashOut]( + db hashdb.HashDB[H, DBValue], + root H, + cache TrieCache[H], + recorder TrieRecorder[H], +) *TrieDB[H] { + return &TrieDB[H]{ + db: db, + root: root, + cache: cache, + recorder: recorder, + } +} diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go new file mode 100644 index 0000000000..3c0ad4eb35 --- /dev/null +++ b/pkg/trie/triedb/triedbbuilder.go @@ -0,0 +1,38 @@ +package triedb + +import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + +type DBValue = []byte + +type TrieDBBuilder[Out HashOut] struct { + db hashdb.HashDB[Out, DBValue] + root Out + cache TrieCache[Out] + recorder TrieRecorder[Out] +} + +func NewTrieDBBuilder[Out HashOut]( + db hashdb.HashDB[Out, DBValue], + root Out, +) *TrieDBBuilder[Out] { + return &TrieDBBuilder[Out]{ + db: db, + root: root, + cache: nil, + recorder: nil, + } +} + +func (tdbb *TrieDBBuilder[Out]) WithCache(cache TrieCache[Out]) *TrieDBBuilder[Out] { + tdbb.cache = cache + return tdbb +} + +func (tdbb *TrieDBBuilder[Out]) WithRecorder(recorder TrieRecorder[Out]) *TrieDBBuilder[Out] { + tdbb.recorder = recorder + return tdbb +} + +func (tdbb *TrieDBBuilder[Out]) Build() *TrieDB[Out] { + return NewTrieDB(tdbb.db, tdbb.root, tdbb.cache, tdbb.recorder) +} From 3e5e0189255fa5da726ac3beac08bc5c2f83abef Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 8 Jan 2024 11:17:43 -0300 Subject: [PATCH 06/35] feat(pkg/trie): Add insert and remove methods --- go.mod | 1 + go.sum | 2 + pkg/trie/memorydb/memorydb.go | 22 +++ pkg/trie/triedb/cache.go | 8 +- pkg/trie/triedb/layout.go | 9 +- pkg/trie/triedb/lookup.go | 43 ++-- pkg/trie/triedb/{ => node}/node.go | 20 +- pkg/trie/triedb/{ => node}/node_codec.go | 6 +- pkg/trie/triedb/recorder.go | 30 +-- pkg/trie/triedb/trie.go | 30 ++- pkg/trie/triedb/triedb.go | 239 ++++++++++++++++++++++- pkg/trie/triedb/triedbbuilder.go | 9 +- 12 files changed, 355 insertions(+), 64 deletions(-) rename pkg/trie/triedb/{ => node}/node.go (93%) rename pkg/trie/triedb/{ => node}/node_codec.go (80%) diff --git a/go.mod b/go.mod index c96b84004b..66f09d4d43 100644 --- a/go.mod +++ b/go.mod @@ -73,6 +73,7 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gammazero/deque v0.2.1 github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 25beb3f6c9..5a4aa3615e 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= +github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index 23cb846f0d..e057fcf656 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -14,6 +14,28 @@ type MemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[ keyFunction KF } +func newFromNullNode[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any]( + nullKey []byte, + nullNodeData T, + hasher Hasher, + keyFunction KF, +) *MemoryDB[Hash, Hasher, KF, T] { + return &MemoryDB[Hash, Hasher, KF, T]{ + data: make(map[Hash]MemoryDBValue[T]), + hashedNullNode: hasher.Hash(nullKey), + nullNodeData: nullNodeData, + keyFunction: keyFunction, + } +} + +func NewMemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any]( + data []byte, + hasher Hasher, + keyFunction KF, +) *MemoryDB[Hash, Hasher, KF, []byte] { + return newFromNullNode[Hash](data, data, hasher, keyFunction) +} + type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { Key(hash Hash, prefix hashdb.Prefix) Hash } diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go index e79bb6ca83..a5e8f6d502 100644 --- a/pkg/trie/triedb/cache.go +++ b/pkg/trie/triedb/cache.go @@ -1,5 +1,7 @@ package triedb +import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + // CachedValue a value as cached by TrieCache type CachedValue[H comparable] interface { Type() string @@ -22,9 +24,9 @@ func (v NonExisting) Type() string { return "NonExisting" } func (v ExistingHash[H]) Type() string { return "ExistingHash" } func (v Existing[H]) Type() string { return "Existing" } -type TrieCache[Out HashOut] interface { +type TrieCache[Out node.HashOut] interface { LookupValueForKey(key []byte) *CachedValue[Out] CacheValueForKey(key []byte, value CachedValue[Out]) - GetOrInsertNode(hash Out, fetchNode func() (Node[Out], error)) - GetNode(hash Out) Node[Out] + GetOrInsertNode(hash Out, fetchNode func() (node.Node[Out], error)) + GetNode(hash Out) node.Node[Out] } diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go index 4586ee2457..cbf07df3d8 100644 --- a/pkg/trie/triedb/layout.go +++ b/pkg/trie/triedb/layout.go @@ -1,11 +1,14 @@ package triedb -import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +) -type TrieLayout[Out HashOut] interface { +type TrieLayout[Out node.HashOut] interface { UseExtension() bool AllowEmpty() bool MaxInlineValue() *uint Hasher() hashdb.Hasher[Out] - Codec() NodeCodec[Out] + Codec() node.NodeCodec[Out] } diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index ca8730f467..894f5f71bb 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -6,11 +6,12 @@ package triedb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" ) var EmptyValue = []byte{} -type Lookup[Hash HashOut] struct { +type Lookup[Hash node.HashOut] struct { db hashdb.HashDB[Hash, DBValue] hash Hash cache TrieCache[Hash] @@ -18,7 +19,7 @@ type Lookup[Hash HashOut] struct { layout TrieLayout[Hash] } -func NewLookup[H HashOut]( +func NewLookup[H node.HashOut]( db hashdb.HashDB[H, DBValue], hash H, cache TrieCache[H], recorder TrieRecorder[H]) *Lookup[H] { return &Lookup[H]{ db: db, @@ -69,21 +70,21 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er return nil, DecoderError } - var nextNode NodeHandle = nil + var nextNode node.NodeHandle = nil switch node := decodedNode.(type) { - case Empty: + case node.Empty: return EmptyValue, nil - case Leaf: + case node.Leaf: // If leaf and matches return value - if partial.Eq(&node.partialKey) { - return l.loadValue(node.value, nibbleKey.OriginalDataAsPrefix()) + if partial.Eq(&node.PartialKey) { + return l.loadValue(node.Value, nibbleKey.OriginalDataAsPrefix()) } return EmptyValue, nil - case NibbledBranch: - slice := node.partialKey - children := node.children - value := node.value + case node.NibbledBranch: + slice := node.PartialKey + children := node.Children + value := node.Value // Get next node if !partial.StartsWith(&slice) { return EmptyValue, nil @@ -104,28 +105,28 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er keyNibbles += slice.Len() + 1 } - switch node := nextNode.(type) { - case Hash: - nextHash := DecodeHash(node.value, l.layout.Hasher()) + switch n := nextNode.(type) { + case node.Hash: + nextHash := node.DecodeHash(n.Value, l.layout.Hasher()) if nextHash == nil { return nil, InvalidHash } hash = *nextHash break - case Inline: - nodeData = &node.value + case node.Inline: + nodeData = &n.Value } } depth++ } } -func (l Lookup[H]) loadValue(value Value, prefix hashdb.Prefix) ([]byte, error) { +func (l Lookup[H]) loadValue(value node.Value, prefix hashdb.Prefix) ([]byte, error) { switch v := value.(type) { - case InlineValue: - return v.bytes, nil - case NodeValue: - hash := l.layout.Hasher().FromBytes(v.bytes) + case node.InlineValue: + return v.Bytes, nil + case node.NodeValue: + hash := l.layout.Hasher().FromBytes(v.Bytes) bytes := l.db.Get(hash, prefix) if bytes == nil { return nil, ErrIncompleteDB diff --git a/pkg/trie/triedb/node.go b/pkg/trie/triedb/node/node.go similarity index 93% rename from pkg/trie/triedb/node.go rename to pkg/trie/triedb/node/node.go index 5f5fe2af53..6ef44f1508 100644 --- a/pkg/trie/triedb/node.go +++ b/pkg/trie/triedb/node/node.go @@ -1,4 +1,4 @@ -package triedb +package node import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" @@ -13,11 +13,11 @@ type Value interface { type ( // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value InlineValue struct { - bytes []byte + Bytes []byte } // HashedNodeValue is a trie node pointer to a hashed node NodeValue struct { - bytes []byte + Bytes []byte } ) @@ -34,14 +34,14 @@ type ( Empty struct{} // NodeLeaf represents a leaf node Leaf struct { - partialKey nibble.NibbleSlice - value Value + PartialKey nibble.NibbleSlice + Value Value } // NodeNibbledBranch represents a branch node NibbledBranch struct { - partialKey nibble.NibbleSlice - children [nibble.NibbleLength]NodeHandle - value Value + PartialKey nibble.NibbleSlice + Children [nibble.NibbleLength]NodeHandle + Value Value } ) @@ -115,10 +115,10 @@ type NodeHandle interface { } type ( Hash struct { - value []byte + Value []byte } Inline struct { - value []byte + Value []byte } ) diff --git a/pkg/trie/triedb/node_codec.go b/pkg/trie/triedb/node/node_codec.go similarity index 80% rename from pkg/trie/triedb/node_codec.go rename to pkg/trie/triedb/node/node_codec.go index be648acfb5..4ec58338eb 100644 --- a/pkg/trie/triedb/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -1,4 +1,4 @@ -package triedb +package node import ( "fmt" @@ -24,9 +24,9 @@ func EncodeNode[H HashOut](node Node[H], codec NodeCodec[H]) []byte { case Empty: return codec.EmptyNode() case Leaf: - return codec.LeafNode(n.partialKey, n.partialKey.Len(), n.value) + return codec.LeafNode(n.PartialKey, n.PartialKey.Len(), n.Value) case NibbledBranch: - return codec.BranchNodeNibbled(n.partialKey, n.partialKey.Len(), n.children, n.value) + return codec.BranchNodeNibbled(n.PartialKey, n.PartialKey.Len(), n.Children, n.Value) default: panic(fmt.Sprintf("unknown node type %s", n.Type())) } diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index 1c7bbcb2d2..dcfde757ec 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -1,6 +1,10 @@ package triedb -import "fmt" +import ( + "fmt" + + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +) type RecordedForKey uint8 @@ -10,25 +14,25 @@ const ( RecordedForKeyNone ) -type TrieRecorder[Out HashOut] interface { +type TrieRecorder[Out node.HashOut] interface { record(access TrieAccess[Out]) trieNodesRecordedForKey(key []byte) RecordedForKey } // TrieAccess is used to report the trie access to the TrieRecorder -type TrieAccess[Out HashOut] interface { +type TrieAccess[Out node.HashOut] interface { Type() string } type ( // TrieAccessNode means that the given node was accessed using its hash - TrieAccessNode[H HashOut] struct { + TrieAccessNode[H node.HashOut] struct { hash H - node Node[H] + node node.Node[H] } // TrieAccessEncodedNode means that the given encodedNode was accessed using its hash - TrieAccessEncodedNode[H HashOut] struct { + TrieAccessEncodedNode[H node.HashOut] struct { hash H encodedNode []byte } @@ -36,7 +40,7 @@ type ( // TrieAccessValue means that the given value was accessed using its hash // fullKey is the key to access this value in the trie // Should map to RecordedForKeyValue when checking the recorder - TrieAccessValue[H HashOut] struct { + TrieAccessValue[H node.HashOut] struct { hash H value []byte fullKey []byte @@ -45,13 +49,13 @@ type ( // TrieAccessInlineValue means that a value stored in an inlined node was accessed // The given fullKey is the key to access this value in the trie // Should map to RecordedForKeyValue when checking the recorder - TrieAccessInlineValue[H HashOut] struct { + TrieAccessInlineValue[H node.HashOut] struct { fullKey []byte } // TrieAccessHash means that the hash of the value for a given fullKey was accessed // Should map to RecordedForKeyHash when checking the recorder - TrieAccessHash[H HashOut] struct { + TrieAccessHash[H node.HashOut] struct { fullKey []byte } @@ -71,21 +75,21 @@ func (a TrieAccessNonExisting) Type() string { return "NotExisting" } // Recorder implementation -type Record[H HashOut] struct { +type Record[H node.HashOut] struct { /// The hash of the node. Hash H /// The data representing the node. Data []byte } -type Recorder[H HashOut] struct { +type Recorder[H node.HashOut] struct { nodes []Record[H] recorderKeys map[string]RecordedForKey // TODO: revisit this later, it should be a BTreeMap layout TrieLayout[H] } // NewRecorder creates a new Recorder which records all given nodes -func NewRecorder[H HashOut]() *Recorder[H] { +func NewRecorder[H node.HashOut]() *Recorder[H] { return &Recorder[H]{ nodes: make([]Record[H], 0), recorderKeys: make(map[string]RecordedForKey), @@ -111,7 +115,7 @@ func (r *Recorder[H]) record(access TrieAccess[H]) { case TrieAccessEncodedNode[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.encodedNode}) case TrieAccessNode[H]: - r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: EncodeNode(access.node, r.layout.Codec())}) + r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: node.EncodeNode(access.node, r.layout.Codec())}) case TrieAccessValue[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.value}) r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go index 77d0a40439..e8149f8836 100644 --- a/pkg/trie/triedb/trie.go +++ b/pkg/trie/triedb/trie.go @@ -1,6 +1,29 @@ package triedb -type Trie[Hash HashOut] interface { +import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + +type TrieValue interface { + Type() string +} + +type ( + InlineTrieValue struct { + Bytes []byte + } + NodeTrieValue[H node.HashOut] struct { + Hash H + } + NewNodeTrie[H node.HashOut] struct { + Hash *H + Bytes []byte + } +) + +func (v InlineTrieValue) Type() string { return "Inline" } +func (v NodeTrieValue[H]) Type() string { return "Node" } +func (v NewNodeTrie[H]) Type() string { return "NewNode" } + +type Trie[Hash node.HashOut] interface { Root() Hash IsEmpty() bool Contains(key []byte) (bool, error) @@ -10,3 +33,8 @@ type Trie[Hash HashOut] interface { //get_with //lookup_first_descendant } + +type MutableTrie[Hash node.HashOut] interface { + insert(key []byte, value []byte) (*TrieValue, error) + remove(key []byte) (*TrieValue, error) +} diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index ea55e4b4ee..409c6a031a 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -1,15 +1,96 @@ package triedb -import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + "github.com/gammazero/deque" +) -type TrieDB[Out HashOut] struct { - db hashdb.HashDB[Out, DBValue] - root Out - cache TrieCache[Out] - recorder TrieRecorder[Out] +type StorageHandle = uint +type NibbleFullKey = nibble.NibbleSlice + +type Stored[H node.HashOut] interface { + Type() string +} + +type ( + StoredNew[Out node.HashOut] struct { + Node node.Node[Out] + } + StoredCached[Out node.HashOut] struct { + Node node.Node[Out] + Hash Out + } +) + +func (s StoredNew[Out]) Type() string { return "New" } +func (s StoredCached[Out]) Type() string { return "Cached" } + +type NodeHandle interface { + Type() string +} +type ( + Hash[H node.HashOut] struct { + Value H + } + InMemory struct { + Value StorageHandle + } +) + +func (h Hash[H]) Type() string { return "Hash" } +func (h InMemory) Type() string { return "InMemory" } + +// Compact storage for tree nodes +type NodeStorage[H node.HashOut] struct { + nodes []Stored[H] + freeIndices deque.Deque[uint] +} + +func NewEmptyNodeStorage[H node.HashOut]() *NodeStorage[H] { + return &NodeStorage[H]{ + nodes: make([]Stored[H], 0), + } +} + +func (ns *NodeStorage[H]) alloc(stored Stored[H]) StorageHandle { + if ns.freeIndices.Len() > 0 { + idx := ns.freeIndices.PopFront() + ns.nodes[idx] = stored + return idx + } + + ns.nodes = append(ns.nodes, stored) + return uint(len(ns.nodes) - 1) } -func NewTrieDB[H HashOut]( +func (ns *NodeStorage[H]) destroy(handle StorageHandle) Stored[H] { + idx := handle + + ns.freeIndices.PushBack(idx) + ns.nodes[idx] = StoredNew[H]{node.Empty{}} + return ns.nodes[idx] +} + +type deathRowValue struct { + backingByte [40]byte + b *[]byte +} + +type TrieDB[Out node.HashOut] struct { + storage NodeStorage[Out] + db hashdb.HashDB[Out, DBValue] + root Out + rootHandle node.NodeHandle + deathRow map[Out]struct{} + hashCount uint + cache TrieCache[Out] + recorder TrieRecorder[Out] + layout TrieLayout[Out] +} + +func NewTrieDB[H node.HashOut]( db hashdb.HashDB[H, DBValue], root H, cache TrieCache[H], @@ -22,3 +103,147 @@ func NewTrieDB[H HashOut]( recorder: recorder, } } + +type RemoveAtResult struct { + handle StorageHandle + changed bool +} + +// TODO: implement me +func (tdb *TrieDB[H]) lookupAndCache( + hash H, + key hashdb.Prefix, +) (StorageHandle, error) { + return 0, nil +} + +type PostInspectAction interface { + Type() string +} + +type ( + Replace[H node.HashOut] struct { + node node.Node[H] + } + Restore[H node.HashOut] struct { + node node.Node[H] + } + Delete struct{} +) + +func (r Replace[H]) Type() string { return "Replace" } +func (r Restore[H]) Type() string { return "Restore" } +func (r Delete) Type() string { return "Delete" } + +type InspectResult[H node.HashOut] struct { + stored Stored[H] + changed bool +} + +// TODO: implement me +func (tdb *TrieDB[H]) inspect( + stored Stored[H], + key NibbleFullKey, + inspector func( + node node.Node[H], + key NibbleFullKey, + ) (PostInspectAction, error), +) (InspectResult[H], error) { + panic("implement me") +} + +// TODO: implement me +func (tdb *TrieDB[H]) removeInspector( + node node.Node[H], + key NibbleFullKey, + oldVal *TrieValue, +) (PostInspectAction, error) { + panic("implement me") +} + +// Removes a node from the trie based on key +func (tdb *TrieDB[H]) removeAt( + handle NodeHandle, + key NibbleFullKey, + oldVal *TrieValue, +) (*RemoveAtResult, error) { + var stored Stored[H] + + switch h := handle.(type) { + case InMemory: + stored = tdb.storage.destroy(h.Value) + case Hash[H]: + fromCache, err := tdb.lookupAndCache(h.Value, key.Left()) + if err != nil { + return nil, err + } + stored = tdb.storage.destroy(fromCache) + } + + res, err := tdb.inspect(stored, key, func(node node.Node[H], key NibbleFullKey) (PostInspectAction, error) { + return tdb.removeInspector(node, key, oldVal) + }) + + if err != nil { + return nil, err + } + + return &RemoveAtResult{ + tdb.storage.alloc(res.stored), + res.changed, + }, nil +} + +func (tdb *TrieDB[H]) Remove(key []byte) (*TrieValue, error) { + rootHandle := tdb.rootHandle + keySlice := nibble.NewNibbleSlice(key) + var oldVal *TrieValue + + res, err := tdb.removeAt(rootHandle, *keySlice, oldVal) + + if err != nil { + return nil, err + } + + if res != nil { + tdb.rootHandle = InMemory{res.handle} + } else { + tdb.rootHandle = Hash[H]{tdb.layout.Codec().HashedNullNode()} + tdb.root = tdb.layout.Codec().HashedNullNode() + } + + return oldVal, nil +} + +type InsertAtResult struct { + handle StorageHandle + changed bool +} + +// TODO: implement me +func (tdb *TrieDB[H]) insertAt( + handle NodeHandle, + key NibbleFullKey, + value []byte, + oldVal *TrieValue, +) (InsertAtResult, error) { + panic("implement me") +} + +func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { + if !tdb.layout.AllowEmpty() && len(value) == 0 { + return tdb.Remove(key) + } + + var oldVal *TrieValue + + insertRes, err := tdb.insertAt(tdb.rootHandle, *nibble.NewNibbleSlice(key), value, oldVal) + + if err != nil { + return nil, err + } + + tdb.rootHandle = InMemory{insertRes.handle} + + return oldVal, nil +} diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go index 3c0ad4eb35..45e187f950 100644 --- a/pkg/trie/triedb/triedbbuilder.go +++ b/pkg/trie/triedb/triedbbuilder.go @@ -1,17 +1,20 @@ package triedb -import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +) type DBValue = []byte -type TrieDBBuilder[Out HashOut] struct { +type TrieDBBuilder[Out node.HashOut] struct { db hashdb.HashDB[Out, DBValue] root Out cache TrieCache[Out] recorder TrieRecorder[Out] } -func NewTrieDBBuilder[Out HashOut]( +func NewTrieDBBuilder[Out node.HashOut]( db hashdb.HashDB[Out, DBValue], root Out, ) *TrieDBBuilder[Out] { From 4188cec87aac9cda62ba3b87cbb5fd2bc46b28da Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 8 Jan 2024 12:13:11 -0300 Subject: [PATCH 07/35] feat(pkg/trie): Add license --- pkg/trie/hashdb/hashdb.go | 3 + pkg/trie/memorydb/memorydb.go | 3 + pkg/trie/triedb/cache.go | 3 + pkg/trie/triedb/errors.go | 3 + pkg/trie/triedb/layout.go | 3 + pkg/trie/triedb/nibble/nibble.go | 8 ++ pkg/trie/triedb/nibble/nibbleslice.go | 8 +- pkg/trie/triedb/node/node.go | 3 + pkg/trie/triedb/node/node_codec.go | 3 + pkg/trie/triedb/recorder.go | 3 + pkg/trie/triedb/trie.go | 3 + pkg/trie/triedb/triedb.go | 141 +++++++++++++++++++++----- pkg/trie/triedb/triedbbuilder.go | 3 + 13 files changed, 157 insertions(+), 30 deletions(-) diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go index be1125e6da..ad9f247e7a 100644 --- a/pkg/trie/hashdb/hashdb.go +++ b/pkg/trie/hashdb/hashdb.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package hashdb type HasherOut interface { diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index e057fcf656..0ca06db9d7 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package memorydb import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go index a5e8f6d502..733ef93821 100644 --- a/pkg/trie/triedb/cache.go +++ b/pkg/trie/triedb/cache.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" diff --git a/pkg/trie/triedb/errors.go b/pkg/trie/triedb/errors.go index 085f07bc9b..e548fe9355 100644 --- a/pkg/trie/triedb/errors.go +++ b/pkg/trie/triedb/errors.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import "errors" diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go index cbf07df3d8..4bb3da2504 100644 --- a/pkg/trie/triedb/layout.go +++ b/pkg/trie/triedb/layout.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import ( diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 8389e4ec68..49e2b80d6b 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package nibble const NibblePerByte uint = 2 @@ -5,6 +8,11 @@ const PaddingBitmask byte = 0x0F const BitPerNibble = 4 const NibbleLength = 16 +type NodeKey struct { + offset uint + nibbles NibbleSlice +} + func padLeft(b byte) byte { padded := (b & ^PaddingBitmask) return padded diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go index 6cf93093e0..29e031a609 100644 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -17,6 +17,10 @@ func NewNibbleSlice(data []byte) *NibbleSlice { return &NibbleSlice{data, 0} } +func NewFromStored(i NodeKey) *NibbleSlice { + return NewNibbleSliceWithPadding(i.nibbles.data, i.offset) +} + func NewNibbleSliceWithPadding(data []byte, padding uint) *NibbleSlice { return &NibbleSlice{data, padding} } @@ -48,14 +52,14 @@ func (ns *NibbleSlice) At(i uint) byte { } func (ns *NibbleSlice) StartsWith(other *NibbleSlice) bool { - return ns.commonPrefix(other) == other.Len() + return ns.CommonPrefix(other) == other.Len() } func (ns *NibbleSlice) Eq(other *NibbleSlice) bool { return ns.Len() == other.Len() && ns.StartsWith(other) } -func (ns *NibbleSlice) commonPrefix(other *NibbleSlice) uint { +func (ns *NibbleSlice) CommonPrefix(other *NibbleSlice) uint { selfAlign := ns.offset % NibblePerByte otherAlign := other.offset % NibblePerByte if selfAlign == otherAlign { diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 6ef44f1508..4303ee92b0 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package node import ( diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index 4ec58338eb..b4bb67f6e9 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package node import ( diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index dcfde757ec..3edded138d 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import ( diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go index e8149f8836..a2c4369e1d 100644 --- a/pkg/trie/triedb/trie.go +++ b/pkg/trie/triedb/trie.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 409c6a031a..803cd9180f 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -1,25 +1,74 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + n "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" "github.com/gammazero/deque" ) +type HashOut = n.HashOut + +// Value +type Value interface { + Type() string +} + +type ( + InlineValue struct { + Bytes []byte + } + NodeValue[H HashOut] struct { + Hash H + } + + NewNode[H HashOut] struct { + Hash *H + Bytes []byte + } +) + +// Node types in the Trie +type Node[H HashOut] interface { + Type() string +} + +type ( + // NodeEmptyNode represents an empty node + Empty struct{} + // NodeLeaf represents a leaf node + Leaf struct { + PartialKey nibble.NodeKey + Value Value + } + // NodeNibbledBranch represents a branch node + NibbledBranch struct { + PartialKey nibble.NodeKey + Children [nibble.NibbleLength]NodeHandle + Value Value + } +) + +func (n Empty) Type() string { return "Empty" } +func (n Leaf) Type() string { return "Leaf" } +func (n NibbledBranch) Type() string { return "NibbledBranch" } + type StorageHandle = uint type NibbleFullKey = nibble.NibbleSlice -type Stored[H node.HashOut] interface { +type Stored[H HashOut] interface { Type() string } type ( - StoredNew[Out node.HashOut] struct { - Node node.Node[Out] + StoredNew[Out HashOut] struct { + Node Node[Out] } - StoredCached[Out node.HashOut] struct { - Node node.Node[Out] + StoredCached[Out HashOut] struct { + Node Node[Out] Hash Out } ) @@ -31,7 +80,7 @@ type NodeHandle interface { Type() string } type ( - Hash[H node.HashOut] struct { + Hash[H HashOut] struct { Value H } InMemory struct { @@ -43,12 +92,12 @@ func (h Hash[H]) Type() string { return "Hash" } func (h InMemory) Type() string { return "InMemory" } // Compact storage for tree nodes -type NodeStorage[H node.HashOut] struct { +type NodeStorage[H HashOut] struct { nodes []Stored[H] freeIndices deque.Deque[uint] } -func NewEmptyNodeStorage[H node.HashOut]() *NodeStorage[H] { +func NewEmptyNodeStorage[H HashOut]() *NodeStorage[H] { return &NodeStorage[H]{ nodes: make([]Stored[H], 0), } @@ -69,7 +118,7 @@ func (ns *NodeStorage[H]) destroy(handle StorageHandle) Stored[H] { idx := handle ns.freeIndices.PushBack(idx) - ns.nodes[idx] = StoredNew[H]{node.Empty{}} + ns.nodes[idx] = StoredNew[H]{Empty{}} return ns.nodes[idx] } @@ -78,11 +127,11 @@ type deathRowValue struct { b *[]byte } -type TrieDB[Out node.HashOut] struct { +type TrieDB[Out HashOut] struct { storage NodeStorage[Out] db hashdb.HashDB[Out, DBValue] root Out - rootHandle node.NodeHandle + rootHandle NodeHandle deathRow map[Out]struct{} hashCount uint cache TrieCache[Out] @@ -90,7 +139,7 @@ type TrieDB[Out node.HashOut] struct { layout TrieLayout[Out] } -func NewTrieDB[H node.HashOut]( +func NewTrieDB[H HashOut]( db hashdb.HashDB[H, DBValue], root H, cache TrieCache[H], @@ -114,7 +163,7 @@ func (tdb *TrieDB[H]) lookupAndCache( hash H, key hashdb.Prefix, ) (StorageHandle, error) { - return 0, nil + panic("implement me") } type PostInspectAction interface { @@ -122,20 +171,20 @@ type PostInspectAction interface { } type ( - Replace[H node.HashOut] struct { - node node.Node[H] + PostInspectActionReplace[H HashOut] struct { + node Node[H] } - Restore[H node.HashOut] struct { - node node.Node[H] + PostInspectActionRestore[H HashOut] struct { + node Node[H] } - Delete struct{} + PostInspectActionDelete struct{} ) -func (r Replace[H]) Type() string { return "Replace" } -func (r Restore[H]) Type() string { return "Restore" } -func (r Delete) Type() string { return "Delete" } +func (r PostInspectActionReplace[H]) Type() string { return "Replace" } +func (r PostInspectActionRestore[H]) Type() string { return "Restore" } +func (r PostInspectActionDelete) Type() string { return "Delete" } -type InspectResult[H node.HashOut] struct { +type InspectResult[H HashOut] struct { stored Stored[H] changed bool } @@ -145,7 +194,7 @@ func (tdb *TrieDB[H]) inspect( stored Stored[H], key NibbleFullKey, inspector func( - node node.Node[H], + node Node[H], key NibbleFullKey, ) (PostInspectAction, error), ) (InspectResult[H], error) { @@ -154,13 +203,23 @@ func (tdb *TrieDB[H]) inspect( // TODO: implement me func (tdb *TrieDB[H]) removeInspector( - node node.Node[H], + node Node[H], key NibbleFullKey, oldVal *TrieValue, ) (PostInspectAction, error) { panic("implement me") } +// TODO: implement me +func (tdb *TrieDB[H]) insertInspector( + node Node[H], + key NibbleFullKey, + value []byte, + oldVal *TrieValue, +) (PostInspectAction, error) { + panic("Implement me") +} + // Removes a node from the trie based on key func (tdb *TrieDB[H]) removeAt( handle NodeHandle, @@ -180,7 +239,7 @@ func (tdb *TrieDB[H]) removeAt( stored = tdb.storage.destroy(fromCache) } - res, err := tdb.inspect(stored, key, func(node node.Node[H], key NibbleFullKey) (PostInspectAction, error) { + res, err := tdb.inspect(stored, key, func(node Node[H], key NibbleFullKey) (PostInspectAction, error) { return tdb.removeInspector(node, key, oldVal) }) @@ -220,14 +279,40 @@ type InsertAtResult struct { changed bool } -// TODO: implement me +// / Insert a key-value pair into the trie, creating new nodes if necessary. func (tdb *TrieDB[H]) insertAt( handle NodeHandle, key NibbleFullKey, value []byte, oldVal *TrieValue, ) (InsertAtResult, error) { - panic("implement me") + var storageHandle StorageHandle + var err error + + switch h := handle.(type) { + case InMemory: + storageHandle = h.Value + case Hash[H]: + storageHandle, err = tdb.lookupAndCache(h.Value, key.Left()) + if err != nil { + return InsertAtResult{}, err + } + } + + stored := tdb.storage.destroy(storageHandle) + + res, err := tdb.inspect(stored, key, func(node Node[H], key NibbleFullKey) (PostInspectAction, error) { + return tdb.insertInspector(node, key, value, oldVal) + }) + + if err != nil { + return InsertAtResult{}, err + } + + return InsertAtResult{ + tdb.storage.alloc(res.stored), + res.changed, + }, nil } func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go index 45e187f950..9ddeec5bae 100644 --- a/pkg/trie/triedb/triedbbuilder.go +++ b/pkg/trie/triedb/triedbbuilder.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import ( From f597bf8ce0af73ada6fd1238c251caacb4f04d45 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 8 Jan 2024 17:48:42 -0300 Subject: [PATCH 08/35] feat(pkg/trie): fixes and implements lookup and cache --- pkg/trie/triedb/cache.go | 4 +-- pkg/trie/triedb/node/node.go | 34 ++++++++++++++++----- pkg/trie/triedb/node/node_codec.go | 39 ++++++++++++++++++++---- pkg/trie/triedb/recorder.go | 12 ++++---- pkg/trie/triedb/triedb.go | 49 +++++++++++++++++++++++++++--- 5 files changed, 111 insertions(+), 27 deletions(-) diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go index 733ef93821..79c4167390 100644 --- a/pkg/trie/triedb/cache.go +++ b/pkg/trie/triedb/cache.go @@ -30,6 +30,6 @@ func (v Existing[H]) Type() string { return "Existing" } type TrieCache[Out node.HashOut] interface { LookupValueForKey(key []byte) *CachedValue[Out] CacheValueForKey(key []byte, value CachedValue[Out]) - GetOrInsertNode(hash Out, fetchNode func() (node.Node[Out], error)) - GetNode(hash Out) node.Node[Out] + GetOrInsertNode(hash Out, fetchNode func() (node.NodeOwned[Out], error)) + GetNode(hash Out) *node.NodeOwned[Out] } diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 4303ee92b0..3e6ddaee46 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -62,14 +62,14 @@ type ( NodeOwnedEmpty struct{} // NodeLeaf represents a leaf node NodeOwnedLeaf[H HashOut] struct { - partialKey nibble.NibbleSlice - value ValueOwned[H] + PartialKey nibble.NibbleSlice + Value ValueOwned[H] } // NodeNibbledBranch represents a branch node NodeOwnedNibbledBranch[H HashOut] struct { - partialKey nibble.NibbleSlice - children [nibble.NibbleLength]NodeHandleOwned[H] - value ValueOwned[H] + PartialKey nibble.NibbleSlice + Children [nibble.NibbleLength]NodeHandleOwned[H] + Value ValueOwned[H] } ) @@ -80,6 +80,7 @@ func (n NodeOwnedNibbledBranch[H]) Type() string { return "NibbledBranch" } // Value is a trie node value type ValueOwned[H HashOut] interface { Type() string + AsValue() Value } type ( // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value @@ -88,17 +89,24 @@ type ( hash H } // HashedNodeValue is a trie node pointer to a hashed node - NodeValueOwned[H comparable] struct { + NodeValueOwned[H HashOut] struct { hash H } ) func (v InlineValueOwned[H]) Type() string { return "Inline" } -func (v NodeValueOwned[H]) Type() string { return "Node" } +func (v InlineValueOwned[H]) AsValue() Value { + return InlineValue{Bytes: v.bytes} +} +func (v NodeValueOwned[H]) Type() string { return "Node" } +func (v NodeValueOwned[H]) AsValue() Value { + return NodeValue{Bytes: v.hash.ToBytes()} +} // NodeHandle is a reference to a trie node which may be stored within another trie node. type NodeHandleOwned[H HashOut] interface { Type() string + AsChildReference(codec NodeCodec[H]) ChildReference[H] } type ( NodeHandleOwnedHash[H HashOut] struct { @@ -109,8 +117,18 @@ type ( } ) -func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } +func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } +func (h NodeHandleOwnedHash[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { + return ChildReferenceHash[H]{hash: h.ValueOwned} +} func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } +func (h NodeHandleOwnedInline[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { + encoded := EncodeNodeOwned(h.node, codec) + if len(encoded) > codec.Hasher().Length() { + panic("Invalid inline node handle") + } + return ChildReferenceInline[H]{hash: codec.Hasher().FromBytes(encoded), length: uint(len(encoded))} +} // NodeHandle is a reference to a trie node which may be stored within another trie node. type NodeHandle interface { diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index b4bb67f6e9..4ded56b569 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -6,9 +6,27 @@ package node import ( "fmt" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) +type ChildReference[H HashOut] interface { + Type() string +} + +type ( + ChildReferenceHash[H HashOut] struct { + hash H + } + ChildReferenceInline[H HashOut] struct { + hash H + length uint + } +) + +func (c ChildReferenceHash[H]) Type() string { return "Hash" } +func (c ChildReferenceInline[H]) Type() string { return "Inline" } + type HashOut interface { comparable ToBytes() []byte @@ -16,20 +34,29 @@ type HashOut interface { type NodeCodec[H HashOut] interface { HashedNullNode() H + Hasher() hashdb.Hasher[H] EmptyNode() []byte LeafNode(partialKey nibble.NibbleSlice, numberNibble uint, value Value) []byte - BranchNodeNibbled(partialKey nibble.NibbleSlice, numberNibble uint, children [16]NodeHandle, value Value) []byte + BranchNodeNibbled(partialKey nibble.NibbleSlice, numberNibble uint, children [16]ChildReference[H], value *Value) []byte Decode(data []byte) (Node[H], error) } -func EncodeNode[H HashOut](node Node[H], codec NodeCodec[H]) []byte { +func EncodeNodeOwned[H HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { switch n := node.(type) { - case Empty: + case NodeOwnedEmpty: return codec.EmptyNode() - case Leaf: + case NodeOwnedLeaf[H]: return codec.LeafNode(n.PartialKey, n.PartialKey.Len(), n.Value) - case NibbledBranch: - return codec.BranchNodeNibbled(n.PartialKey, n.PartialKey.Len(), n.Children, n.Value) + case NodeOwnedNibbledBranch[H]: + var value = n.Value.AsValue() + + var children [16]ChildReference[H] + for i, c := range n.Children { + if c != nil { + children[i] = c.AsChildReference(codec) + } + } + return codec.BranchNodeNibbled(n.PartialKey, n.PartialKey.Len(), children, &value) default: panic(fmt.Sprintf("unknown node type %s", n.Type())) } diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index 3edded138d..6a28614215 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -29,9 +29,9 @@ type TrieAccess[Out node.HashOut] interface { type ( // TrieAccessNode means that the given node was accessed using its hash - TrieAccessNode[H node.HashOut] struct { - hash H - node node.Node[H] + TrieAccessNodeOwned[H node.HashOut] struct { + hash H + nodeOwned node.NodeOwned[H] } // TrieAccessEncodedNode means that the given encodedNode was accessed using its hash @@ -69,7 +69,7 @@ type ( } ) -func (a TrieAccessNode[H]) Type() string { return "Node" } +func (a TrieAccessNodeOwned[H]) Type() string { return "Node" } func (a TrieAccessEncodedNode[H]) Type() string { return "EncodedNode" } func (a TrieAccessValue[H]) Type() string { return "Value" } func (a TrieAccessInlineValue[H]) Type() string { return "InlineValue" } @@ -117,8 +117,8 @@ func (r *Recorder[H]) record(access TrieAccess[H]) { switch access := access.(type) { case TrieAccessEncodedNode[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.encodedNode}) - case TrieAccessNode[H]: - r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: node.EncodeNode(access.node, r.layout.Codec())}) + case TrieAccessNodeOwned[H]: + r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: node.EncodeNodeOwned(access.nodeOwned, r.layout.Codec())}) case TrieAccessValue[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.value}) r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 803cd9180f..762910b115 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -1,6 +1,3 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - package triedb import ( @@ -158,12 +155,34 @@ type RemoveAtResult struct { changed bool } -// TODO: implement me +func (tdb *TrieDB[H]) record( + access TrieAccess[H], +) { + if tdb.recorder != nil { + tdb.recorder.record(access) + } +} + func (tdb *TrieDB[H]) lookupAndCache( hash H, key hashdb.Prefix, ) (StorageHandle, error) { - panic("implement me") + var node Node[H] + + nodeFromCache := tdb.cache.GetNode(hash) + if nodeFromCache != nil { + tdb.record(TrieAccessNodeOwned[H]{hash: hash, nodeOwned: *nodeFromCache}) + node = NodeFromNodeOwned(*nodeFromCache, tdb.storage) + } else { + nodeEncoded := tdb.db.Get(hash, key) + if nodeEncoded == nil { + return 0, ErrIncompleteDB + } + + tdb.record(TrieAccessEncodedNode[H]{hash: hash, encodedNode: *nodeEncoded}) + } + + return tdb.storage.alloc(StoredCached[H]{Node: node, Hash: hash}), nil } type PostInspectAction interface { @@ -332,3 +351,23 @@ func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { return oldVal, nil } + +func NodeFromNodeOwned[H HashOut](nodeOwned n.NodeOwned[H], storage NodeStorage[H]) Node[H] { + switch node := nodeOwned.(type) { + case Empty: + return Empty{} + case Leaf: + return Leaf{ + PartialKey: node.PartialKey, + Value: node.Value, + } + case NibbledBranch: + return NibbledBranch{ + PartialKey: node.PartialKey, + Children: node.Children, + Value: node.Value, + } + default: + panic("Invalid node") + } +} From 1dd1f130f3c5fd74f75bd6261be5dcc7739ae7f6 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 10 Jan 2024 11:37:01 -0300 Subject: [PATCH 09/35] feat(pkg/trie): Implements remove inspector --- pkg/trie/hashdb/hashdb.go | 25 +--- pkg/trie/memorydb/memorydb.go | 7 +- pkg/trie/triedb/lookup.go | 2 +- pkg/trie/triedb/nibble/nibble.go | 14 +- pkg/trie/triedb/nibble/nibbleslice.go | 37 +++-- pkg/trie/triedb/node/node.go | 14 +- pkg/trie/triedb/node/node_codec.go | 2 +- pkg/trie/triedb/triedb.go | 198 +++++++++++++++++++++++--- 8 files changed, 234 insertions(+), 65 deletions(-) diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go index ad9f247e7a..2d33806f6c 100644 --- a/pkg/trie/hashdb/hashdb.go +++ b/pkg/trie/hashdb/hashdb.go @@ -3,24 +3,13 @@ package hashdb +import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + type HasherOut interface { comparable ToBytes() []byte } -// / A trie node prefix, it is the nibble path from the trie root -// / to the trie node. -// / For a node containing no partial key value it is the full key. -// / For a value node or node containing a partial key, it is the full key minus its node partial -// / nibbles (the node key can be split into prefix and node partial). -// / Therefore it is always the leftmost portion of the node key, so its internal representation -// / is a non expanded byte slice followed by a last padded byte representation. -// / The padded byte is an optional padded value. -type Prefix struct { - PartialKey []byte - PaddedByte *byte -} - type Hasher[Hash HasherOut] interface { Length() int Hash(value []byte) Hash @@ -35,9 +24,9 @@ type PlainDB[K any, V any] interface { } type HashDB[Hash HasherOut, T any] interface { - Get(key Hash, prefix Prefix) *T - Contains(key Hash, prefix Prefix) bool - Insert(prefix Prefix, value []byte) Hash - Emplace(key Hash, prefix Prefix, value T) - remove(key Hash, prefix Prefix) + Get(key Hash, prefix nibble.Prefix) *T + Contains(key Hash, prefix nibble.Prefix) bool + Insert(prefix nibble.Prefix, value []byte) Hash + Emplace(key Hash, prefix nibble.Prefix, value T) + remove(key Hash, prefix nibble.Prefix) } diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index 0ca06db9d7..47a34bb7e2 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -3,7 +3,10 @@ package memorydb -import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" +) type MemoryDBValue[T any] struct { value T @@ -40,5 +43,5 @@ func NewMemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFuncti } type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { - Key(hash Hash, prefix hashdb.Prefix) Hash + Key(hash Hash, prefix nibble.Prefix) Hash } diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 894f5f71bb..6e1096be97 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -121,7 +121,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er } } -func (l Lookup[H]) loadValue(value node.Value, prefix hashdb.Prefix) ([]byte, error) { +func (l Lookup[H]) loadValue(value node.Value, prefix nibble.Prefix) ([]byte, error) { switch v := value.(type) { case node.InlineValue: return v.Bytes, nil diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 49e2b80d6b..521f8fe549 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -8,9 +8,17 @@ const PaddingBitmask byte = 0x0F const BitPerNibble = 4 const NibbleLength = 16 -type NodeKey struct { - offset uint - nibbles NibbleSlice +// / A trie node prefix, it is the nibble path from the trie root +// / to the trie node. +// / For a node containing no partial key value it is the full key. +// / For a value node or node containing a partial key, it is the full key minus its node partial +// / nibbles (the node key can be split into prefix and node partial). +// / Therefore it is always the leftmost portion of the node key, so its internal representation +// / is a non expanded byte slice followed by a last padded byte representation. +// / The padded byte is an optional padded value. +type Prefix struct { + PartialKey []byte + PaddedByte *byte } func padLeft(b byte) byte { diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go index 29e031a609..f13c6d17d8 100644 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -3,8 +3,6 @@ package nibble -import "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - // NibbleSlice is a helper structure to store a slice of nibbles and a moving offset // this is helpful to use it for example while we are looking for a key, we can define the full key in the data and // moving the offset while we are going deep in the trie @@ -17,14 +15,31 @@ func NewNibbleSlice(data []byte) *NibbleSlice { return &NibbleSlice{data, 0} } -func NewFromStored(i NodeKey) *NibbleSlice { - return NewNibbleSliceWithPadding(i.nibbles.data, i.offset) -} - func NewNibbleSliceWithPadding(data []byte, padding uint) *NibbleSlice { return &NibbleSlice{data, padding} } +func NewFromStored(i Prefix) *NibbleSlice { + return &NibbleSlice{i.PartialKey, uint(*i.PaddedByte)} +} + +func (ns *NibbleSlice) Clone() *NibbleSlice { + data := make([]byte, len(ns.data)) + copy(data, ns.data) + return &NibbleSlice{data, ns.offset} +} + +func (ns *NibbleSlice) IsEmpty() bool { + return len(ns.Data()) == 0 +} + +func (ns *NibbleSlice) Advance(i uint) { + if ns.Len() < i { + panic("Cannot advance more than the length of the slice") + } + ns.offset += i +} + func (ns *NibbleSlice) Data() []byte { return ns.data } @@ -88,26 +103,26 @@ func (ns *NibbleSlice) CommonPrefix(other *NibbleSlice) uint { return i } -func (ns *NibbleSlice) Left() hashdb.Prefix { +func (ns *NibbleSlice) Left() Prefix { split := ns.offset / NibblePerByte ix := (ns.offset % NibblePerByte) if ix == 0 { - return hashdb.Prefix{ + return Prefix{ PartialKey: ns.data[:split], PaddedByte: nil, } } else { padded := padRight(ns.data[split]) - return hashdb.Prefix{ + return Prefix{ PartialKey: ns.data[:split], PaddedByte: &padded, } } } -func (ns *NibbleSlice) OriginalDataAsPrefix() hashdb.Prefix { - return hashdb.Prefix{ +func (ns *NibbleSlice) OriginalDataAsPrefix() Prefix { + return Prefix{ PartialKey: ns.data, PaddedByte: nil, } diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 3e6ddaee46..0245f074e6 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -67,9 +67,9 @@ type ( } // NodeNibbledBranch represents a branch node NodeOwnedNibbledBranch[H HashOut] struct { - PartialKey nibble.NibbleSlice - Children [nibble.NibbleLength]NodeHandleOwned[H] - Value ValueOwned[H] + PartialKey nibble.NibbleSlice + EncodedChildren [nibble.NibbleLength]NodeHandleOwned[H] + Value ValueOwned[H] } ) @@ -110,20 +110,20 @@ type NodeHandleOwned[H HashOut] interface { } type ( NodeHandleOwnedHash[H HashOut] struct { - ValueOwned H + Hash H } NodeHandleOwnedInline[H HashOut] struct { - node NodeOwned[H] + Node NodeOwned[H] } ) func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } func (h NodeHandleOwnedHash[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - return ChildReferenceHash[H]{hash: h.ValueOwned} + return ChildReferenceHash[H]{hash: h.Hash} } func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } func (h NodeHandleOwnedInline[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - encoded := EncodeNodeOwned(h.node, codec) + encoded := EncodeNodeOwned(h.Node, codec) if len(encoded) > codec.Hasher().Length() { panic("Invalid inline node handle") } diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index 4ded56b569..995616dc70 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -51,7 +51,7 @@ func EncodeNodeOwned[H HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { var value = n.Value.AsValue() var children [16]ChildReference[H] - for i, c := range n.Children { + for i, c := range n.EncodedChildren { if c != nil { children[i] = c.AsChildReference(codec) } diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 762910b115..0422bd4bf5 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -3,11 +3,11 @@ package triedb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - n "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + node_types "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" "github.com/gammazero/deque" ) -type HashOut = n.HashOut +type HashOut = node_types.HashOut // Value type Value interface { @@ -38,14 +38,14 @@ type ( Empty struct{} // NodeLeaf represents a leaf node Leaf struct { - PartialKey nibble.NodeKey - Value Value + encoded nibble.NibbleSlice + value Value } // NodeNibbledBranch represents a branch node NibbledBranch struct { - PartialKey nibble.NodeKey - Children [nibble.NibbleLength]NodeHandle - Value Value + encoded nibble.NibbleSlice + children [nibble.NibbleLength]NodeHandle + value Value } ) @@ -129,7 +129,7 @@ type TrieDB[Out HashOut] struct { db hashdb.HashDB[Out, DBValue] root Out rootHandle NodeHandle - deathRow map[Out]struct{} + deathRow map[Out]nibble.Prefix hashCount uint cache TrieCache[Out] recorder TrieRecorder[Out] @@ -165,7 +165,7 @@ func (tdb *TrieDB[H]) record( func (tdb *TrieDB[H]) lookupAndCache( hash H, - key hashdb.Prefix, + key nibble.Prefix, ) (StorageHandle, error) { var node Node[H] @@ -208,7 +208,6 @@ type InspectResult[H HashOut] struct { changed bool } -// TODO: implement me func (tdb *TrieDB[H]) inspect( stored Stored[H], key NibbleFullKey, @@ -216,8 +215,51 @@ func (tdb *TrieDB[H]) inspect( node Node[H], key NibbleFullKey, ) (PostInspectAction, error), -) (InspectResult[H], error) { - panic("implement me") +) (*InspectResult[H], error) { + var result InspectResult[H] + + switch s := stored.(type) { + case StoredNew[H]: + execution, err := inspector(s.Node, key) + if err != nil { + return nil, err + } + switch action := execution.(type) { + case PostInspectActionRestore[H]: + result = InspectResult[H]{StoredNew[H]{action.node}, false} + case PostInspectActionReplace[H]: + result = InspectResult[H]{StoredNew[H]{action.node}, true} + case PostInspectActionDelete: + return nil, nil + } + case StoredCached[H]: + execution, err := inspector(s.Node, key) + if err != nil { + return nil, err + } + + switch action := execution.(type) { + case PostInspectActionRestore[H]: + result = InspectResult[H]{StoredCached[H]{action.node, s.Hash}, false} + case PostInspectActionReplace[H]: + tdb.deathRow[s.Hash] = key.Left() + result = InspectResult[H]{StoredNew[H]{action.node}, true} + case PostInspectActionDelete: + tdb.deathRow[s.Hash] = key.Left() + return nil, nil + } + } + + return &result, nil +} + +// Given a node which may be in an invalid state, fix it such that it is then in a valid +// state. +// +// invalid state means: +// - Branch node where there is only a single entry; +func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) { + panic("TODO: implement me") } // TODO: implement me @@ -226,7 +268,86 @@ func (tdb *TrieDB[H]) removeInspector( key NibbleFullKey, oldVal *TrieValue, ) (PostInspectAction, error) { - panic("implement me") + switch n := node.(type) { + case Empty: + return PostInspectActionDelete{}, nil + case Leaf: + existingKey := n.encoded + if key.Eq(&existingKey) { + // We found the node we want to delete, so we are going to remove it + keyVal := key.Clone() + keyVal.Advance(existingKey.Len()) + tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) + return PostInspectActionDelete{}, nil + } else { + // Leaf the node alone, restoring leaf wrong partial + return PostInspectActionRestore[H]{ + Leaf{n.encoded, n.value}, + }, nil + } + case NibbledBranch: + if key.IsEmpty() { + if n.value == nil { + return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, nil}}, nil + } + tdb.replaceOldValue(oldVal, n.value, key.Left()) + fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, nil}, key) + if err != nil { + return nil, err + } + return PostInspectActionReplace[H]{fixedNode}, nil + } + common := n.encoded.CommonPrefix(&key) + existingLength := n.encoded.Len() + + if common == existingLength && common == key.Len() { + // Replace val + if n.value != nil { + keyVal := key.Clone() + keyVal.Advance(existingLength) + tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) + fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, nil}, key) + if err != nil { + return nil, err + } + return PostInspectActionReplace[H]{fixedNode}, nil + } + return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, nil}}, nil + } else if common < existingLength { + // Nothing to do here + return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, n.value}}, nil + } else { + // common == existing_length && common < partial.len() : check children + idx := key.At(common) + + child := n.children[idx] + if child != nil { + key.Advance(common + 1) + res, err := tdb.removeAt(child, key, oldVal) + if err != nil { + return nil, err + } + + if res != nil { + n.children[idx] = InMemory{res.handle} + branch := NibbledBranch{n.encoded, n.children, n.value} + if res.changed { + return PostInspectActionReplace[H]{branch}, nil + } else { + return PostInspectActionRestore[H]{branch}, nil + } + } + fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, n.value}, key) + if err != nil { + return nil, err + } + return PostInspectActionReplace[H]{fixedNode}, nil + } + return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, n.value}}, nil + } + default: + panic("Invalid node type") + } } // TODO: implement me @@ -239,6 +360,15 @@ func (tdb *TrieDB[H]) insertInspector( panic("Implement me") } +// TODO: implement me +func (tdb *TrieDB[H]) replaceOldValue( + oldVal *TrieValue, + newVal Value, + key nibble.Prefix, +) { + panic("Implement me") +} + // Removes a node from the trie based on key func (tdb *TrieDB[H]) removeAt( handle NodeHandle, @@ -352,20 +482,44 @@ func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { return oldVal, nil } -func NodeFromNodeOwned[H HashOut](nodeOwned n.NodeOwned[H], storage NodeStorage[H]) Node[H] { +func inlineOrHashOwned[H HashOut](child node_types.NodeHandleOwned[H], storage NodeStorage[H]) NodeHandle { + switch n := child.(type) { + case node_types.NodeHandleOwnedHash[H]: + return Hash[H]{n.Hash} + case node_types.NodeHandleOwnedInline[H]: + child := NodeFromNodeOwned[H](n.Node, storage) + return InMemory{storage.alloc(StoredNew[H]{child})} + default: + panic("Invalid child") + } +} + +func NodeFromNodeOwned[H HashOut](nodeOwned node_types.NodeOwned[H], storage NodeStorage[H]) Node[H] { switch node := nodeOwned.(type) { - case Empty: + case node_types.NodeOwnedEmpty: return Empty{} - case Leaf: + case node_types.NodeOwnedLeaf[H]: return Leaf{ - PartialKey: node.PartialKey, - Value: node.Value, + encoded: node.PartialKey, + value: node.Value, } - case NibbledBranch: + case node_types.NodeOwnedNibbledBranch[H]: + child := func(i uint) NodeHandle { + if node.EncodedChildren[i] != nil { + return inlineOrHashOwned(node.EncodedChildren[i], storage) + } + return nil + } + + var children [16]NodeHandle + for i := uint(0); i < 16; i++ { + children[i] = child(i) + } + return NibbledBranch{ - PartialKey: node.PartialKey, - Children: node.Children, - Value: node.Value, + encoded: node.PartialKey, + children: children, + value: node.Value, } default: panic("Invalid node") From 0fd4c491fbcce26d4c4fea2bc5581df9c982e509 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 10 Jan 2024 19:59:58 -0300 Subject: [PATCH 10/35] Fixes and insert inspector --- pkg/trie/triedb/layout.go | 1 - pkg/trie/triedb/lookup.go | 4 +- pkg/trie/triedb/nibble/nibble.go | 27 ++++ pkg/trie/triedb/nibble/nibbleslice.go | 71 +++++++++- pkg/trie/triedb/trie.go | 8 ++ pkg/trie/triedb/triedb.go | 188 ++++++++++++++++++++------ 6 files changed, 248 insertions(+), 51 deletions(-) diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go index 4bb3da2504..54a7704721 100644 --- a/pkg/trie/triedb/layout.go +++ b/pkg/trie/triedb/layout.go @@ -9,7 +9,6 @@ import ( ) type TrieLayout[Out node.HashOut] interface { - UseExtension() bool AllowEmpty() bool MaxInlineValue() *uint Hasher() hashdb.Hasher[Out] diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 6e1096be97..2f3a5225fb 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -77,7 +77,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er return EmptyValue, nil case node.Leaf: // If leaf and matches return value - if partial.Eq(&node.PartialKey) { + if partial.Eq(node.PartialKey) { return l.loadValue(node.Value, nibbleKey.OriginalDataAsPrefix()) } return EmptyValue, nil @@ -86,7 +86,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er children := node.Children value := node.Value // Get next node - if !partial.StartsWith(&slice) { + if !partial.StartsWith(slice) { return EmptyValue, nil } diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 521f8fe549..2b4a5206fd 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -7,6 +7,8 @@ const NibblePerByte uint = 2 const PaddingBitmask byte = 0x0F const BitPerNibble = 4 const NibbleLength = 16 +const SplitLeftShift = 4 +const SplitRightShift = 4 // / A trie node prefix, it is the nibble path from the trie root // / to the trie node. @@ -35,6 +37,31 @@ func NumberPadding(i uint) uint { return i % NibblePerByte } +func ShiftKey(key *NibbleSlice, offset uint) bool { + oldOffset := key.offset + key.offset = offset + + if oldOffset > offset { + // Shift left + kl := key.Len() + for i := uint(0); i < kl; i++ { + key.data[i] = key.data[i]<<2 | key.data[i+1]>>SplitLeftShift + } + key.data[kl-1] = key.data[kl-1] << SplitRightShift + return true + } else if oldOffset < offset { + // Shift right + key.data = append(key.data, 0) + for i := key.Len() - 1; i >= 1; i-- { + key.data[i] = key.data[i-1]<>SplitRightShift + } + key.data[0] = key.data[0] >> SplitRightShift + return true + } else { + return false + } +} + // Count the biggest common depth between two left aligned packed nibble slice func biggestDepth(v1, v2 []byte) uint { upperBound := minLength(v1, v2) diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go index f13c6d17d8..83e7c1a02c 100644 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -3,9 +3,30 @@ package nibble -// NibbleSlice is a helper structure to store a slice of nibbles and a moving offset -// this is helpful to use it for example while we are looking for a key, we can define the full key in the data and -// moving the offset while we are going deep in the trie +// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. +// +// This is an immutable struct. No operations actually change it. +// +// # Example (TODO: translate to go) +// ```snippet +// use patricia_trie::nibbleslice::NibbleSlice; +// +// fn main() { +// let d1 = &[0x01u8, 0x23, 0x45]; +// let d2 = &[0x34u8, 0x50, 0x12]; +// let d3 = &[0x00u8, 0x12]; +// let n1 = NibbleSlice::new(d1); // 0,1,2,3,4,5 +// let n2 = NibbleSlice::new(d2); // 3,4,5,0,1,2 +// let n3 = NibbleSlice::new_offset(d3, 1); // 0,1,2 +// assert!(n1 > n3); // 0,1,2,... > 0,1,2 +// assert!(n1 < n2); // 0,... < 3,... +// assert!(n2.mid(3) == n3); // 0,1,2 == 0,1,2 +// assert!(n1.starts_with(&n3)); +// assert_eq!(n1.common_prefix(&n3), 3); +// assert_eq!(n2.mid(3).common_prefix(&n1), 3); +// } +// +// ``` type NibbleSlice struct { data []byte offset uint @@ -29,6 +50,44 @@ func (ns *NibbleSlice) Clone() *NibbleSlice { return &NibbleSlice{data, ns.offset} } +func (ns *NibbleSlice) ToStored() NibbleSlice { + split := ns.offset / NibblePerByte + offset := ns.offset % NibblePerByte + return NibbleSlice{ + data: ns.data[split:], + offset: offset, + } +} + +func (ns *NibbleSlice) ToStoredRange(nb uint) NibbleSlice { + if nb > ns.Len() { + ns.ToStored() + } + if (ns.offset+nb)%NibblePerByte == 0 { + // aligned + start := ns.offset / NibblePerByte + end := (ns.offset + nb) / NibblePerByte + return NibbleSlice{ + data: ns.data[start:end], + offset: ns.offset % NibblePerByte, + } + } else { + // unaligned + start := ns.offset / NibblePerByte + end := (ns.offset + nb) / NibblePerByte + ea := ns.data[start : end+1] + eaOffset := ns.offset + nOffset := NumberPadding(nb) + result := NibbleSlice{ + ea, + eaOffset, + } + ShiftKey(&result, nOffset) + result.data = result.data[:len(result.data)-1] + return result + } +} + func (ns *NibbleSlice) IsEmpty() bool { return len(ns.Data()) == 0 } @@ -66,15 +125,15 @@ func (ns *NibbleSlice) At(i uint) byte { return b >> BitPerNibble } -func (ns *NibbleSlice) StartsWith(other *NibbleSlice) bool { +func (ns *NibbleSlice) StartsWith(other NibbleSlice) bool { return ns.CommonPrefix(other) == other.Len() } -func (ns *NibbleSlice) Eq(other *NibbleSlice) bool { +func (ns *NibbleSlice) Eq(other NibbleSlice) bool { return ns.Len() == other.Len() && ns.StartsWith(other) } -func (ns *NibbleSlice) CommonPrefix(other *NibbleSlice) uint { +func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) uint { selfAlign := ns.offset % NibblePerByte otherAlign := other.offset % NibblePerByte if selfAlign == otherAlign { diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go index a2c4369e1d..7a2561adaa 100644 --- a/pkg/trie/triedb/trie.go +++ b/pkg/trie/triedb/trie.go @@ -26,6 +26,14 @@ func (v InlineTrieValue) Type() string { return "Inline" } func (v NodeTrieValue[H]) Type() string { return "Node" } func (v NewNodeTrie[H]) Type() string { return "NewNode" } +func NewTrieValueFromBytes[H HashOut](value []byte, threshold *uint) TrieValue { + if threshold != nil && uint(len(value)) >= *threshold { + return NewNodeTrie[H]{nil, value} + } else { + return InlineTrieValue{Bytes: value} + } +} + type Trie[Hash node.HashOut] interface { Root() Hash IsEmpty() bool diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 0422bd4bf5..6864026b57 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -8,25 +8,7 @@ import ( ) type HashOut = node_types.HashOut - -// Value -type Value interface { - Type() string -} - -type ( - InlineValue struct { - Bytes []byte - } - NodeValue[H HashOut] struct { - Hash H - } - - NewNode[H HashOut] struct { - Hash *H - Bytes []byte - } -) +type Children = [nibble.NibbleLength]NodeHandle // Node types in the Trie type Node[H HashOut] interface { @@ -39,13 +21,13 @@ type ( // NodeLeaf represents a leaf node Leaf struct { encoded nibble.NibbleSlice - value Value + value TrieValue } // NodeNibbledBranch represents a branch node NibbledBranch struct { encoded nibble.NibbleSlice - children [nibble.NibbleLength]NodeHandle - value Value + children Children + value TrieValue } ) @@ -61,12 +43,12 @@ type Stored[H HashOut] interface { } type ( - StoredNew[Out HashOut] struct { - Node Node[Out] + StoredNew[H HashOut] struct { + Node Node[H] } - StoredCached[Out HashOut] struct { - Node Node[Out] - Hash Out + StoredCached[H HashOut] struct { + Hash H + Node Node[H] } ) @@ -119,11 +101,6 @@ func (ns *NodeStorage[H]) destroy(handle StorageHandle) Stored[H] { return ns.nodes[idx] } -type deathRowValue struct { - backingByte [40]byte - b *[]byte -} - type TrieDB[Out HashOut] struct { storage NodeStorage[Out] db hashdb.HashDB[Out, DBValue] @@ -240,7 +217,7 @@ func (tdb *TrieDB[H]) inspect( switch action := execution.(type) { case PostInspectActionRestore[H]: - result = InspectResult[H]{StoredCached[H]{action.node, s.Hash}, false} + result = InspectResult[H]{StoredCached[H]{s.Hash, action.node}, false} case PostInspectActionReplace[H]: tdb.deathRow[s.Hash] = key.Left() result = InspectResult[H]{StoredNew[H]{action.node}, true} @@ -262,7 +239,6 @@ func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) panic("TODO: implement me") } -// TODO: implement me func (tdb *TrieDB[H]) removeInspector( node Node[H], key NibbleFullKey, @@ -273,7 +249,7 @@ func (tdb *TrieDB[H]) removeInspector( return PostInspectActionDelete{}, nil case Leaf: existingKey := n.encoded - if key.Eq(&existingKey) { + if key.Eq(existingKey) { // We found the node we want to delete, so we are going to remove it keyVal := key.Clone() keyVal.Advance(existingKey.Len()) @@ -297,7 +273,7 @@ func (tdb *TrieDB[H]) removeInspector( } return PostInspectActionReplace[H]{fixedNode}, nil } - common := n.encoded.CommonPrefix(&key) + common := n.encoded.CommonPrefix(key) existingLength := n.encoded.Len() if common == existingLength && common == key.Len() { @@ -350,23 +326,151 @@ func (tdb *TrieDB[H]) removeInspector( } } -// TODO: implement me func (tdb *TrieDB[H]) insertInspector( node Node[H], key NibbleFullKey, value []byte, oldVal *TrieValue, ) (PostInspectAction, error) { - panic("Implement me") + partial := key + + switch n := node.(type) { + case Empty: + value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) + return PostInspectActionReplace[H]{Leaf{partial, value}}, nil + case Leaf: + existingKey := n.encoded + common := partial.CommonPrefix(existingKey) + if common == existingKey.Len() && common == partial.Len() { + // Equivalent leaf replace + value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) + unchanged := n.value == value + keyVal := key.Clone() + keyVal.Advance(existingKey.Len()) + tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) + if unchanged { + return PostInspectActionRestore[H]{Leaf{n.encoded, n.value}}, nil + } + return PostInspectActionReplace[H]{Leaf{n.encoded, n.value}}, nil + } else if common < existingKey.Len() { + // one of us isn't empty: transmute to branch here + children := Children{} + idx := existingKey.At(common) + newLeaf := Leaf{*existingKey.Mid(common + 1), n.value} + children[idx] = InMemory{tdb.storage.alloc(StoredNew[H]{newLeaf})} + branch := NibbledBranch{partial.ToStoredRange(common), children, nil} + + branchAction, err := tdb.insertInspector(branch, key, value, oldVal) + if err != nil { + return nil, err + } + return PostInspectActionReplace[H]{branchAction}, nil + } else { + branch := NibbledBranch{ + existingKey.ToStored(), + Children{}, + n.value, + } + + branchAction, err := tdb.insertInspector(branch, key, value, oldVal) + if err != nil { + return nil, err + } + return PostInspectActionReplace[H]{branchAction}, nil + } + case NibbledBranch: + existingKey := n.encoded + common := partial.CommonPrefix(existingKey) + if common == existingKey.Len() && common == partial.Len() { + value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) + unchanged := n.value == value + branch := NibbledBranch{n.encoded, n.children, n.value} + + keyVal := key.Clone() + keyVal.Advance(existingKey.Len()) + tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) + + if unchanged { + return PostInspectActionRestore[H]{branch}, nil + } + return PostInspectActionReplace[H]{branch}, nil + } else if common < existingKey.Len() { + nbranchPartial := existingKey.Mid(common + 1).ToStored() + low := NibbledBranch{nbranchPartial, n.children, n.value} + ix := existingKey.At(common) + children := Children{} + allocStorage := tdb.storage.alloc(StoredNew[H]{low}) + + children[ix] = InMemory{allocStorage} + value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) + + if partial.Len()-common == 0 { + return PostInspectActionReplace[H]{ + NibbledBranch{ + existingKey.ToStoredRange(common), + children, + value, + }, + }, nil + } else { + ix := partial.At(common) + storedLeaf := Leaf{partial.Mid(common + 1).ToStored(), value} + leaf := tdb.storage.alloc(StoredNew[H]{storedLeaf}) + + children[ix] = InMemory{leaf} + return PostInspectActionReplace[H]{ + NibbledBranch{ + existingKey.ToStoredRange(common), + children, + nil, + }, + }, nil + } + } else { + // Append after common == existing_key and partial > common + idx := partial.At(common) + key.Advance(common + 1) + child := n.children[idx] + if child != nil { + // Original had something there. recurse down into it. + res, err := tdb.insertAt(child, key, value, oldVal) + if err != nil { + return nil, err + } + n.children[idx] = InMemory{res.handle} + if !res.changed { + // The new node we composed didn't change. + // It means our branch is untouched too. + nBranch := NibbledBranch{n.encoded.ToStored(), n.children, n.value} + return PostInspectActionRestore[H]{nBranch}, nil + } + } else { + // Original had nothing there. compose a leaf. + value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) + leaf := tdb.storage.alloc(StoredNew[H]{Leaf{key.ToStored(), value}}) + n.children[idx] = InMemory{leaf} + } + return PostInspectActionReplace[H]{NibbledBranch{existingKey.ToStored(), n.children, n.value}}, nil + } + default: + panic("Invalid node type") + } } -// TODO: implement me func (tdb *TrieDB[H]) replaceOldValue( oldVal *TrieValue, - newVal Value, - key nibble.Prefix, + storedValue TrieValue, + prefix nibble.Prefix, ) { - panic("Implement me") + switch n := storedValue.(type) { + case NewNodeTrie[H]: + if n.Hash != nil { + tdb.deathRow[*n.Hash] = prefix + } + case NodeTrieValue[H]: + tdb.deathRow[n.Hash] = prefix + } + *oldVal = storedValue } // Removes a node from the trie based on key From 811638d719686628c24a2e76a5f53169cedac975 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 10 Jan 2024 22:43:08 -0300 Subject: [PATCH 11/35] feat(pkg/trie): Implements fix nodes method --- pkg/trie/triedb/lookup.go | 1 - pkg/trie/triedb/nibble/nibble.go | 11 ++- pkg/trie/triedb/nibble/nibbleslice.go | 19 +++++ pkg/trie/triedb/node/node_codec.go | 7 +- pkg/trie/triedb/recorder.go | 8 +- pkg/trie/triedb/triedb.go | 105 +++++++++++++++++++++++++- 6 files changed, 144 insertions(+), 7 deletions(-) diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 2f3a5225fb..e47653c0cf 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -112,7 +112,6 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er return nil, InvalidHash } hash = *nextHash - break case node.Inline: nodeData = &n.Value } diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 2b4a5206fd..e14f82cf3e 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -23,7 +23,7 @@ type Prefix struct { PaddedByte *byte } -func padLeft(b byte) byte { +func PadLeft(b byte) byte { padded := (b & ^PaddingBitmask) return padded } @@ -37,6 +37,13 @@ func NumberPadding(i uint) uint { return i % NibblePerByte } +func PushAtLeft(ix, v, into byte) byte { + if ix != 1 { + v = v << BitPerNibble + } + return into | v +} + func ShiftKey(key *NibbleSlice, offset uint) bool { oldOffset := key.offset key.offset = offset @@ -79,7 +86,7 @@ func leftCommon(a, b byte) uint { if a == b { return 2 } - if padLeft(a) == padLeft(b) { + if PadLeft(a) == PadLeft(b) { return 1 } else { return 0 diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go index 83e7c1a02c..ca71f16854 100644 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -186,3 +186,22 @@ func (ns *NibbleSlice) OriginalDataAsPrefix() Prefix { PaddedByte: nil, } } + +func CombineKeys(start *NibbleSlice, end NibbleSlice) { + if start.offset >= NibblePerByte || end.offset >= NibblePerByte { + panic("Cannot combine keys") + } + finalOffset := (start.offset + end.offset) % NibblePerByte + ShiftKey(start, finalOffset) + var st uint + if end.offset > 0 { + startLen := start.Len() + start.data[startLen-1] = padRight(end.data[0]) + st = 1 + } else { + st = 0 + } + for i := st; i < end.Len(); i++ { + start.data = append(start.data, end.data[i]) + } +} diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index 995616dc70..c4ba621105 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -37,7 +37,12 @@ type NodeCodec[H HashOut] interface { Hasher() hashdb.Hasher[H] EmptyNode() []byte LeafNode(partialKey nibble.NibbleSlice, numberNibble uint, value Value) []byte - BranchNodeNibbled(partialKey nibble.NibbleSlice, numberNibble uint, children [16]ChildReference[H], value *Value) []byte + BranchNodeNibbled( + partialKey nibble.NibbleSlice, + numberNibble uint, + children [16]ChildReference[H], + value *Value, + ) []byte Decode(data []byte) (Node[H], error) } diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index 6a28614215..bfe50c9cd6 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -118,7 +118,13 @@ func (r *Recorder[H]) record(access TrieAccess[H]) { case TrieAccessEncodedNode[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.encodedNode}) case TrieAccessNodeOwned[H]: - r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: node.EncodeNodeOwned(access.nodeOwned, r.layout.Codec())}) + r.nodes = append( + r.nodes, + Record[H]{ + Hash: access.hash, + Data: node.EncodeNodeOwned(access.nodeOwned, r.layout.Codec()), + }, + ) case TrieAccessValue[H]: r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.value}) r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 6864026b57..9b9ed9be93 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -1,6 +1,8 @@ package triedb import ( + "errors" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" node_types "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" @@ -107,7 +109,6 @@ type TrieDB[Out HashOut] struct { root Out rootHandle NodeHandle deathRow map[Out]nibble.Prefix - hashCount uint cache TrieCache[Out] recorder TrieRecorder[Out] layout TrieLayout[Out] @@ -236,7 +237,107 @@ func (tdb *TrieDB[H]) inspect( // invalid state means: // - Branch node where there is only a single entry; func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) { - panic("TODO: implement me") + switch n := node.(type) { + case NibbledBranch: + var ix byte + usedIndexes := 0 + for i := 0; i < 16; i++ { + if n.children[i] != nil { + ix = byte(i) + usedIndexes += 1 + } + } + if n.value == nil { + if usedIndexes == 0 { + panic("Branch with no subvalues. Something went wrong.") + } + if usedIndexes == 1 { + // only one onward node. use child instead + child := n.children[ix] + if child == nil { + return nil, errors.New("used_index only set if occupied") + } + key2 := key.Clone() + offset := uint(len(n.encoded.Data()))*nibble.NibblePerByte - n.encoded.Offset() + key2.Advance(offset) + prefix := key2.Left() + + start := prefix.PartialKey + var allocStart *[]byte + var prefixEnd *byte + if prefix.PaddedByte == nil { + allocStart = nil + pEnd := nibble.PushAtLeft(0, ix, 0) + prefixEnd = &pEnd + } else { + so := prefix.PartialKey + so = append(so, nibble.PadLeft(*prefix.PaddedByte)|ix) + allocStart = &so + prefixEnd = nil + } + childPrefix := nibble.Prefix{} + if allocStart != nil { + childPrefix.PartialKey = *allocStart + } else { + childPrefix.PartialKey = start + } + childPrefix.PaddedByte = prefixEnd + var stored Stored[H] + + switch c := child.(type) { + case InMemory: + stored = tdb.storage.destroy(c.Value) + case Hash[H]: + handle, err := tdb.lookupAndCache(c.Value, childPrefix) + if err != nil { + return nil, err + } + stored = tdb.storage.destroy(handle) + } + + var childNode Node[H] + + switch s := stored.(type) { + case StoredNew[H]: + childNode = s.Node + case StoredCached[H]: + tdb.deathRow[s.Hash] = childPrefix + childNode = s.Node + } + + switch cn := childNode.(type) { + case Leaf: + encNibble := n.encoded.Clone() + end := nibble.NewNibbleSliceWithPadding([]byte{ix}, nibble.NibblePerByte-1) + nibble.CombineKeys(encNibble, *end) + + end = nibble.NewNibbleSliceWithPadding(cn.encoded.Data(), cn.encoded.Offset()) + nibble.CombineKeys(encNibble, *end) + return Leaf{*encNibble, cn.value}, nil + case NibbledBranch: + encNibble := n.encoded.Clone() + end := nibble.NewNibbleSliceWithPadding([]byte{ix}, nibble.NibblePerByte-1) + nibble.CombineKeys(encNibble, *end) + end = nibble.NewNibbleSliceWithPadding(cn.encoded.Data(), cn.encoded.Offset()) + nibble.CombineKeys(encNibble, *end) + return NibbledBranch{*encNibble, cn.children, cn.value}, nil + default: + panic("Unreachable") + } + } + } + if n.value != nil { + if usedIndexes == 0 { + // Make a lift + // Fixing branch -> Leaf + return Leaf{n.encoded, n.value}, nil + } + } + // All is well, restoring branch + return NibbledBranch{n.encoded, n.children, n.value}, nil + default: + return node, nil + } } func (tdb *TrieDB[H]) removeInspector( From 490185f37f08bdfd45b8df3f3fc798ed21bdd823 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 10 Jan 2024 23:23:37 -0300 Subject: [PATCH 12/35] feat(pkg/trie): NibbleSlice tests --- pkg/trie/triedb/lookup.go | 2 +- pkg/trie/triedb/nibble/nibble.go | 40 +++++-- pkg/trie/triedb/nibble/nibbleslice.go | 89 +++++++--------- pkg/trie/triedb/nibble/nibbleslice_test.go | 118 +++++++++++++++++++++ pkg/trie/triedb/node/node_codec.go | 4 +- pkg/trie/triedb/triedb.go | 10 +- 6 files changed, 197 insertions(+), 66 deletions(-) create mode 100644 pkg/trie/triedb/nibble/nibbleslice_test.go diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index e47653c0cf..05d90fdc3e 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -42,7 +42,7 @@ func (l Lookup[H]) record(access TrieAccess[H]) { func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, error) { partial := nibbleKey hash := l.hash - keyNibbles := uint(0) + keyNibbles := 0 depth := 0 diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index e14f82cf3e..024944cf5e 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -3,7 +3,7 @@ package nibble -const NibblePerByte uint = 2 +const NibblePerByte int = 2 const PaddingBitmask byte = 0x0F const BitPerNibble = 4 const NibbleLength = 16 @@ -33,7 +33,7 @@ func padRight(b byte) byte { return padded } -func NumberPadding(i uint) uint { +func NumberPadding(i int) int { return i % NibblePerByte } @@ -44,14 +44,14 @@ func PushAtLeft(ix, v, into byte) byte { return into | v } -func ShiftKey(key *NibbleSlice, offset uint) bool { +func ShiftKey(key *NibbleSlice, offset int) bool { oldOffset := key.offset key.offset = offset if oldOffset > offset { // Shift left kl := key.Len() - for i := uint(0); i < kl; i++ { + for i := 0; i < kl; i++ { key.data[i] = key.data[i]<<2 | key.data[i+1]>>SplitLeftShift } key.data[kl-1] = key.data[kl-1] << SplitRightShift @@ -70,10 +70,10 @@ func ShiftKey(key *NibbleSlice, offset uint) bool { } // Count the biggest common depth between two left aligned packed nibble slice -func biggestDepth(v1, v2 []byte) uint { +func biggestDepth(v1, v2 []byte) int { upperBound := minLength(v1, v2) - for i := uint(0); i < upperBound; i++ { + for i := 0; i < upperBound; i++ { if v1[i] != v2[i] { return i*NibblePerByte + leftCommon(v1[i], v2[i]) } @@ -82,7 +82,7 @@ func biggestDepth(v1, v2 []byte) uint { } // LeftCommon the number of common nibble between two left aligned bytes -func leftCommon(a, b byte) uint { +func leftCommon(a, b byte) int { if a == b { return 2 } @@ -93,9 +93,29 @@ func leftCommon(a, b byte) uint { } } -func minLength(v1, v2 []byte) uint { +func minLength(v1, v2 []byte) int { if len(v1) < len(v2) { - return uint(len(v1)) + return len(v1) + } + return len(v2) +} + +// CombineKeys combines two node keys representd by nibble slices into the first one +func CombineKeys(start *NibbleSlice, end NibbleSlice) { + if start.offset >= NibblePerByte || end.offset >= NibblePerByte { + panic("Cannot combine keys") + } + finalOffset := (start.offset + end.offset) % NibblePerByte + ShiftKey(start, finalOffset) + var st int + if end.offset > 0 { + startLen := start.Len() + start.data[startLen-1] = padRight(end.data[0]) + st = 1 + } else { + st = 0 + } + for i := st; i < end.Len(); i++ { + start.data = append(start.data, end.data[i]) } - return uint(len(v2)) } diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go index ca71f16854..f8ec8012b5 100644 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ b/pkg/trie/triedb/nibble/nibbleslice.go @@ -29,27 +29,27 @@ package nibble // ``` type NibbleSlice struct { data []byte - offset uint + offset int } +// NewNibbleSlice creates a new nibble slice from a byte slice func NewNibbleSlice(data []byte) *NibbleSlice { return &NibbleSlice{data, 0} } -func NewNibbleSliceWithPadding(data []byte, padding uint) *NibbleSlice { - return &NibbleSlice{data, padding} -} - -func NewFromStored(i Prefix) *NibbleSlice { - return &NibbleSlice{i.PartialKey, uint(*i.PaddedByte)} +// NewNibbleSliceWithOffset creates a new nibble slice from a byte slice with an offset +func NewNibbleSliceWithOffset(data []byte, offset int) *NibbleSlice { + return &NibbleSlice{data, offset} } +// Clone creates a deep copy of the nibble slice func (ns *NibbleSlice) Clone() *NibbleSlice { data := make([]byte, len(ns.data)) copy(data, ns.data) return &NibbleSlice{data, ns.offset} } +// ToStored is a helper function to create a node key from this nibble slice func (ns *NibbleSlice) ToStored() NibbleSlice { split := ns.offset / NibblePerByte offset := ns.offset % NibblePerByte @@ -59,7 +59,8 @@ func (ns *NibbleSlice) ToStored() NibbleSlice { } } -func (ns *NibbleSlice) ToStoredRange(nb uint) NibbleSlice { +// ToStoredRange is a helper function to create a node key from this `NibbleSlice` and for a given number of nibble +func (ns *NibbleSlice) ToStoredRange(nb int) NibbleSlice { if nb > ns.Len() { ns.ToStored() } @@ -88,34 +89,41 @@ func (ns *NibbleSlice) ToStoredRange(nb uint) NibbleSlice { } } +// IsEmpty Return true if the slice contains no nibbles func (ns *NibbleSlice) IsEmpty() bool { - return len(ns.Data()) == 0 + return ns.Len() == 0 } -func (ns *NibbleSlice) Advance(i uint) { +// Advance the view on the slice by `i` nibbles +func (ns *NibbleSlice) Advance(i int) { if ns.Len() < i { panic("Cannot advance more than the length of the slice") } ns.offset += i } +// Data returns the underlying byte slice func (ns *NibbleSlice) Data() []byte { return ns.data } -func (ns *NibbleSlice) Offset() uint { +// Offset returns the offset of the nibble slice +func (ns *NibbleSlice) Offset() int { return ns.offset } -func (ns *NibbleSlice) Mid(i uint) *NibbleSlice { +// Mid returns a new nibble slice object which represents a view on to this slice (further) offset by `i` nibbles +func (ns *NibbleSlice) Mid(i int) *NibbleSlice { return &NibbleSlice{ns.data, ns.offset + i} } -func (ns *NibbleSlice) Len() uint { - return uint(len(ns.data))*NibblePerByte - ns.offset +// Len returns the length of the nibble slice considering the offset +func (ns *NibbleSlice) Len() int { + return len(ns.data)*NibblePerByte - ns.offset } -func (ns *NibbleSlice) At(i uint) byte { +// At returns the nibble at position `i` +func (ns *NibbleSlice) At(i int) byte { ix := (ns.offset + i) / NibblePerByte pad := (ns.offset + i) % NibblePerByte b := ns.data[ix] @@ -125,21 +133,24 @@ func (ns *NibbleSlice) At(i uint) byte { return b >> BitPerNibble } +// StartsWith returns true if this nibble slice start with the same same nibbles contained in `other` func (ns *NibbleSlice) StartsWith(other NibbleSlice) bool { return ns.CommonPrefix(other) == other.Len() } +// Eq returns true if this nibble slice is equal to `other` func (ns *NibbleSlice) Eq(other NibbleSlice) bool { return ns.Len() == other.Len() && ns.StartsWith(other) } -func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) uint { +// CommonPrefix return the amount of same nibbles at the beggining do we match with other +func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) int { selfAlign := ns.offset % NibblePerByte otherAlign := other.offset % NibblePerByte if selfAlign == otherAlign { selfStart := ns.offset / NibblePerByte otherStart := other.offset / NibblePerByte - first := uint(0) + first := 0 if selfAlign != 0 { if padRight(ns.data[selfStart]) != padRight(other.data[otherStart]) { return 0 @@ -152,7 +163,7 @@ func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) uint { } s := minLength(ns.data, other.data) - i := uint(0) + i := 0 for i < s { if ns.At(i) != other.At(i) { break @@ -162,46 +173,28 @@ func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) uint { return i } +// Left returns left portion of `NibbleSlice` +// if the slice originates from a full key it will be the `Prefix of the node`. func (ns *NibbleSlice) Left() Prefix { split := ns.offset / NibblePerByte ix := (ns.offset % NibblePerByte) - if ix == 0 { - return Prefix{ - PartialKey: ns.data[:split], - PaddedByte: nil, - } - } else { + prefix := Prefix{ + PartialKey: ns.data[:split], + PaddedByte: nil, + } + if ix != 0 { padded := padRight(ns.data[split]) - - return Prefix{ - PartialKey: ns.data[:split], - PaddedByte: &padded, - } + prefix.PaddedByte = &padded } + + return prefix } +// OriginalDataAsPrefix gets `Prefix` representation of the inner data. +// This means the entire inner data will be returned as `Prefix`, ignoring any `offset`. func (ns *NibbleSlice) OriginalDataAsPrefix() Prefix { return Prefix{ PartialKey: ns.data, PaddedByte: nil, } } - -func CombineKeys(start *NibbleSlice, end NibbleSlice) { - if start.offset >= NibblePerByte || end.offset >= NibblePerByte { - panic("Cannot combine keys") - } - finalOffset := (start.offset + end.offset) % NibblePerByte - ShiftKey(start, finalOffset) - var st uint - if end.offset > 0 { - startLen := start.Len() - start.data[startLen-1] = padRight(end.data[0]) - st = 1 - } else { - st = 0 - } - for i := st; i < end.Len(); i++ { - start.data = append(start.data, end.data[i]) - } -} diff --git a/pkg/trie/triedb/nibble/nibbleslice_test.go b/pkg/trie/triedb/nibble/nibbleslice_test.go new file mode 100644 index 0000000000..4c2167f6fe --- /dev/null +++ b/pkg/trie/triedb/nibble/nibbleslice_test.go @@ -0,0 +1,118 @@ +package nibble + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +var testData = []byte{0x01, 0x23, 0x45} + +func Test_Basic(t *testing.T) { + t.Run("Nibble slice with offset 0", func(t *testing.T) { + n := NewNibbleSlice(testData) + + require.Equal(t, 6, n.Len()) + require.False(t, n.IsEmpty()) + + for i := 0; i < n.Len(); i++ { + require.Equal(t, byte(i), n.At(i)) + } + }) + + t.Run("Nibble slice with offset 6", func(t *testing.T) { + n := NewNibbleSliceWithOffset(testData, 6) + require.True(t, n.IsEmpty()) + }) + + t.Run("Nibble slice with offset 3", func(t *testing.T) { + n := NewNibbleSliceWithOffset(testData, 3) + require.Equal(t, 3, n.Len()) + for i := 0; i < n.Len(); i++ { + require.Equal(t, byte(i)+3, n.At(i)) + } + }) +} + +func Test_Mid(t *testing.T) { + n := NewNibbleSlice(testData) + t.Run("Mid 2", func(t *testing.T) { + m := n.Mid(2) + for i := 0; i < m.Len(); i++ { + require.Equal(t, byte(i)+2, m.At(i)) + } + }) + t.Run("Mid 3", func(t *testing.T) { + m := n.Mid(3) + for i := 0; i < m.Len(); i++ { + require.Equal(t, byte(i)+3, m.At(i)) + } + }) +} + +func Test_EncodedPre(t *testing.T) { + n := NewNibbleSlice(testData) + + t.Run("Mid 0", func(t *testing.T) { + m := n.Mid(0) + expected := NibbleSlice{ + data: []byte{0x01, 0x23, 0x45}, + offset: 0, + } + + require.Equal(t, expected, m.ToStored()) + }) + + t.Run("Mid 1", func(t *testing.T) { + m := n.Mid(1) + expected := NibbleSlice{ + data: []byte{0x01, 0x23, 0x45}, + offset: 1, + } + + require.Equal(t, expected, m.ToStored()) + }) + + t.Run("Mid 2", func(t *testing.T) { + m := n.Mid(2) + expected := NibbleSlice{ + data: []byte{0x23, 0x45}, + offset: 0, + } + + require.Equal(t, expected, m.ToStored()) + }) + + t.Run("Mid 3", func(t *testing.T) { + m := n.Mid(3) + expected := NibbleSlice{ + data: []byte{0x23, 0x45}, + offset: 1, + } + + require.Equal(t, expected, m.ToStored()) + }) +} + +func Test_Shared(t *testing.T) { + n := NewNibbleSlice(testData) + + other := []byte{0x01, 0x23, 0x01, 0x23, 0x45, 0x67} + m := NewNibbleSlice(other) + + require.Equal(t, 4, n.CommonPrefix(*m)) + require.Equal(t, 4, m.CommonPrefix(*n)) + require.Equal(t, 3, n.Mid(1).CommonPrefix(*m.Mid(1))) + require.Equal(t, 0, n.Mid(1).CommonPrefix(*m.Mid(2))) + require.Equal(t, 6, n.CommonPrefix(*m.Mid(4))) + require.True(t, m.Mid(4).StartsWith(*n)) +} + +func Test_Compare(t *testing.T) { + other := []byte{0x01, 0x23, 0x01, 0x23, 0x45} + n := NewNibbleSlice(testData) + m := NewNibbleSlice(other) + + require.False(t, n.Eq(*m)) + require.True(t, n.Eq(*m.Mid(4))) +} diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index c4ba621105..1a56ac41d8 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -36,10 +36,10 @@ type NodeCodec[H HashOut] interface { HashedNullNode() H Hasher() hashdb.Hasher[H] EmptyNode() []byte - LeafNode(partialKey nibble.NibbleSlice, numberNibble uint, value Value) []byte + LeafNode(partialKey nibble.NibbleSlice, numberNibble int, value Value) []byte BranchNodeNibbled( partialKey nibble.NibbleSlice, - numberNibble uint, + numberNibble int, children [16]ChildReference[H], value *Value, ) []byte diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 9b9ed9be93..fc7a787e2b 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -258,7 +258,7 @@ func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) return nil, errors.New("used_index only set if occupied") } key2 := key.Clone() - offset := uint(len(n.encoded.Data()))*nibble.NibblePerByte - n.encoded.Offset() + offset := len(n.encoded.Data())*nibble.NibblePerByte - n.encoded.Offset() key2.Advance(offset) prefix := key2.Left() @@ -308,17 +308,17 @@ func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) switch cn := childNode.(type) { case Leaf: encNibble := n.encoded.Clone() - end := nibble.NewNibbleSliceWithPadding([]byte{ix}, nibble.NibblePerByte-1) + end := nibble.NewNibbleSliceWithOffset([]byte{ix}, nibble.NibblePerByte-1) nibble.CombineKeys(encNibble, *end) - end = nibble.NewNibbleSliceWithPadding(cn.encoded.Data(), cn.encoded.Offset()) + end = nibble.NewNibbleSliceWithOffset(cn.encoded.Data(), cn.encoded.Offset()) nibble.CombineKeys(encNibble, *end) return Leaf{*encNibble, cn.value}, nil case NibbledBranch: encNibble := n.encoded.Clone() - end := nibble.NewNibbleSliceWithPadding([]byte{ix}, nibble.NibblePerByte-1) + end := nibble.NewNibbleSliceWithOffset([]byte{ix}, nibble.NibblePerByte-1) nibble.CombineKeys(encNibble, *end) - end = nibble.NewNibbleSliceWithPadding(cn.encoded.Data(), cn.encoded.Offset()) + end = nibble.NewNibbleSliceWithOffset(cn.encoded.Data(), cn.encoded.Offset()) nibble.CombineKeys(encNibble, *end) return NibbledBranch{*encNibble, cn.children, cn.value}, nil default: From 869f9b1f7e014aa8fc0c2d709e419a068ec02eae Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 10 Jan 2024 23:31:35 -0300 Subject: [PATCH 13/35] chore(pkg/trie): Add missing licenses --- pkg/trie/triedb/nibble/nibbleslice_test.go | 3 +++ pkg/trie/triedb/triedb.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/trie/triedb/nibble/nibbleslice_test.go b/pkg/trie/triedb/nibble/nibbleslice_test.go index 4c2167f6fe..7cec57a8f4 100644 --- a/pkg/trie/triedb/nibble/nibbleslice_test.go +++ b/pkg/trie/triedb/nibble/nibbleslice_test.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package nibble import ( diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index fc7a787e2b..ed1c36c038 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package triedb import ( From d57528cc81aa7bbbc44b5d49954566e1f59e9628 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 11 Jan 2024 11:40:13 -0300 Subject: [PATCH 14/35] feat(pkg/trie): Implements keccak hasher --- pkg/trie/memorydb/keyfunction.go | 34 ++++++++++++ pkg/trie/memorydb/memorydb.go | 5 -- .../keccak_hasher/keccak_hasher.go | 52 +++++++++++++++++++ pkg/trie/triedb/recorder_test.go | 9 ++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 pkg/trie/memorydb/keyfunction.go create mode 100644 pkg/trie/test_support/keccak_hasher/keccak_hasher.go create mode 100644 pkg/trie/triedb/recorder_test.go diff --git a/pkg/trie/memorydb/keyfunction.go b/pkg/trie/memorydb/keyfunction.go new file mode 100644 index 0000000000..ba6bb6871e --- /dev/null +++ b/pkg/trie/memorydb/keyfunction.go @@ -0,0 +1,34 @@ +package memorydb + +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" +) + +type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { + Key(hash Hash, prefix nibble.Prefix) Hash +} + +type ( + HashKey[H hashdb.HasherOut] struct{} + PrefixKey[H []byte] struct{} +) + +func (h HashKey[H]) Key(key H, prefix nibble.Prefix) H { + return key +} + +func (h PrefixKey[H]) Key(key H, prefix nibble.Prefix) H { + newLen := len(key) + len(prefix.PartialKey) + 1 + prefixedKey := make([]byte, newLen) + + prefixedKey = append(prefixedKey, prefix.PartialKey...) + if prefix.PaddedByte != nil { + prefixedKey = append(prefixedKey, *prefix.PaddedByte) + } + prefixedKey = append(prefixedKey, key...) + + return H(prefixedKey) +} + + diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index 47a34bb7e2..c77c539bd8 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -5,7 +5,6 @@ package memorydb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) type MemoryDBValue[T any] struct { @@ -41,7 +40,3 @@ func NewMemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFuncti ) *MemoryDB[Hash, Hasher, KF, []byte] { return newFromNullNode[Hash](data, data, hasher, keyFunction) } - -type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { - Key(hash Hash, prefix nibble.Prefix) Hash -} diff --git a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go new file mode 100644 index 0000000000..4549e300cb --- /dev/null +++ b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go @@ -0,0 +1,52 @@ +package keccak_hasher + +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "golang.org/x/crypto/sha3" +) + +const KeccakHasherLength = 32 + +type KeccakHash struct { + bytes [KeccakHasherLength]byte +} + +func NewKeccakHash(bytes [KeccakHasherLength]byte) KeccakHash { + return KeccakHash{ + bytes: bytes, + } +} + +func (k KeccakHash) ToBytes() []byte { + return k.bytes[:] +} + +type KeccakHasher[H KeccakHash] struct{} + +func (k *KeccakHasher[H]) Length() int { + return KeccakHasherLength +} + +func (k *KeccakHasher[H]) FromBytes(in []byte) H { + var buf = [KeccakHasherLength]byte{} + copy(buf[:], in) + return H(NewKeccakHash(buf)) +} + +func (k *KeccakHasher[H]) Hash(in []byte) H { + h := sha3.NewLegacyKeccak256() + + _, err := h.Write(in) + if err != nil { + panic("Unexpected error hashing bytes") + } + + hash := h.Sum(nil) + return k.FromBytes(hash) +} + +func NewKeccakHasher[H KeccakHash]() KeccakHasher[H] { + return KeccakHasher[H]{} +} + +var _ hashdb.Hasher[KeccakHash] = (*KeccakHasher[KeccakHash])(nil) diff --git a/pkg/trie/triedb/recorder_test.go b/pkg/trie/triedb/recorder_test.go new file mode 100644 index 0000000000..6688d19516 --- /dev/null +++ b/pkg/trie/triedb/recorder_test.go @@ -0,0 +1,9 @@ +package triedb + +import ( + "testing" +) + +func Test_Record(t *testing.T) { + //TODO: implement +} From c292d8a068097899ea028fb13f56d2290f37d896 Mon Sep 17 00:00:00 2001 From: Diego Date: Fri, 12 Jan 2024 10:39:31 -0300 Subject: [PATCH 15/35] refa(pkg/trie): Simplify generics --- pkg/trie/hashdb/hashdb.go | 19 ++---- pkg/trie/memorydb/keyfunction.go | 23 +++---- pkg/trie/memorydb/memorydb.go | 61 +++++++++++++------ .../keccak_hasher/keccak_hasher.go | 32 +++++++--- pkg/trie/triedb/cache.go | 9 ++- pkg/trie/triedb/layout.go | 3 +- pkg/trie/triedb/lookup.go | 8 +-- pkg/trie/triedb/node/node.go | 24 ++++---- pkg/trie/triedb/node/node_codec.go | 15 ++--- pkg/trie/triedb/recorder.go | 21 ++++--- pkg/trie/triedb/recorder_test.go | 9 --- pkg/trie/triedb/trie.go | 12 ++-- pkg/trie/triedb/triedb.go | 32 +++------- pkg/trie/triedb/triedbbuilder.go | 51 ++++++++++------ 14 files changed, 164 insertions(+), 155 deletions(-) delete mode 100644 pkg/trie/triedb/recorder_test.go diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go index 2d33806f6c..091ab990ea 100644 --- a/pkg/trie/hashdb/hashdb.go +++ b/pkg/trie/hashdb/hashdb.go @@ -5,28 +5,21 @@ package hashdb import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -type HasherOut interface { - comparable - ToBytes() []byte +type HashOut interface { + Bytes() []byte + ComparableKey() string } -type Hasher[Hash HasherOut] interface { +type Hasher[Hash HashOut] interface { Length() int Hash(value []byte) Hash FromBytes(value []byte) Hash } -type PlainDB[K any, V any] interface { - Get(key K) *V - Contains(key K) bool - Emplace(key K, value V) - Remove(key K) -} - -type HashDB[Hash HasherOut, T any] interface { +type HashDB[Hash HashOut, T any] interface { Get(key Hash, prefix nibble.Prefix) *T Contains(key Hash, prefix nibble.Prefix) bool Insert(prefix nibble.Prefix, value []byte) Hash Emplace(key Hash, prefix nibble.Prefix, value T) - remove(key Hash, prefix nibble.Prefix) + Remove(key Hash, prefix nibble.Prefix) } diff --git a/pkg/trie/memorydb/keyfunction.go b/pkg/trie/memorydb/keyfunction.go index ba6bb6871e..47f686ff6e 100644 --- a/pkg/trie/memorydb/keyfunction.go +++ b/pkg/trie/memorydb/keyfunction.go @@ -5,30 +5,21 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) -type KeyFunction[Hash hashdb.HasherOut, H hashdb.Hasher[Hash]] interface { - Key(hash Hash, prefix nibble.Prefix) Hash -} - -type ( - HashKey[H hashdb.HasherOut] struct{} - PrefixKey[H []byte] struct{} -) +type KeyFunction[H hashdb.HashOut] func(key H, prefix nibble.Prefix, hasher hashdb.Hasher[H]) H -func (h HashKey[H]) Key(key H, prefix nibble.Prefix) H { +func HashKey[H hashdb.HashOut](key H, _ nibble.Prefix, _ hashdb.Hasher[H]) H { return key } -func (h PrefixKey[H]) Key(key H, prefix nibble.Prefix) H { - newLen := len(key) + len(prefix.PartialKey) + 1 - prefixedKey := make([]byte, newLen) +func PrefixKey[H hashdb.HashOut](key H, prefix nibble.Prefix, hasher hashdb.Hasher[H]) H { + newLen := len(key.Bytes()) + len(prefix.PartialKey) + 1 + prefixedKey := make([]byte, 0, newLen) prefixedKey = append(prefixedKey, prefix.PartialKey...) if prefix.PaddedByte != nil { prefixedKey = append(prefixedKey, *prefix.PaddedByte) } - prefixedKey = append(prefixedKey, key...) + prefixedKey = append(prefixedKey, key.Bytes()...) - return H(prefixedKey) + return hasher.FromBytes(prefixedKey) } - - diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index c77c539bd8..9eaf7fef5e 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -5,38 +5,59 @@ package memorydb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) -type MemoryDBValue[T any] struct { - value T +type MemoryDBValue struct { + value []byte rc int32 } -type MemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any] struct { - data map[Hash]MemoryDBValue[T] - hashedNullNode Hash - nullNodeData T - keyFunction KF +type MemoryDB[H hashdb.HashOut] struct { + data map[string]MemoryDBValue + hashedNullNode H + nullNodeData []byte + keyFunction KeyFunction[H] } -func newFromNullNode[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any]( +func newFromNullNode[H hashdb.HashOut]( nullKey []byte, - nullNodeData T, - hasher Hasher, - keyFunction KF, -) *MemoryDB[Hash, Hasher, KF, T] { - return &MemoryDB[Hash, Hasher, KF, T]{ - data: make(map[Hash]MemoryDBValue[T]), + nullNodeData []byte, + hasher hashdb.Hasher[H], + keyFunction KeyFunction[H], +) *MemoryDB[H] { + return &MemoryDB[H]{ + data: make(map[string]MemoryDBValue), hashedNullNode: hasher.Hash(nullKey), nullNodeData: nullNodeData, keyFunction: keyFunction, } } -func NewMemoryDB[Hash hashdb.HasherOut, Hasher hashdb.Hasher[Hash], KF KeyFunction[Hash, Hasher], T any]( - data []byte, - hasher Hasher, - keyFunction KF, -) *MemoryDB[Hash, Hasher, KF, []byte] { - return newFromNullNode[Hash](data, data, hasher, keyFunction) +func (db *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { + panic("Implement me") +} + +func (db *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { + panic("Implement me") +} + +func (db *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { + panic("Implement me") +} + +func (db *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { + panic("Implement me") +} + +func (db *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { + panic("Implement me") +} + +func NewMemoryDB[Hash hashdb.HashOut]( + hasher hashdb.Hasher[Hash], + keyFunction KeyFunction[Hash], +) *MemoryDB[Hash] { + data := []byte{0x0} + return newFromNullNode(data, data, hasher, keyFunction) } diff --git a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go index 4549e300cb..ffb1ae7cd3 100644 --- a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go +++ b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go @@ -17,23 +17,37 @@ func NewKeccakHash(bytes [KeccakHasherLength]byte) KeccakHash { } } -func (k KeccakHash) ToBytes() []byte { +func (k KeccakHash) Bytes() []byte { return k.bytes[:] } -type KeccakHasher[H KeccakHash] struct{} +func (k KeccakHash) ComparableKey() string { + return string(k.Bytes()) +} + +func KeccakHashFromBytes(b []byte) KeccakHash { + var newBytes [KeccakHasherLength]byte + copy(newBytes[:], b) + return KeccakHash{ + bytes: newBytes, + } +} + +var _ hashdb.HashOut = KeccakHash{} + +type KeccakHasher struct{} -func (k *KeccakHasher[H]) Length() int { +func (k KeccakHasher) Length() int { return KeccakHasherLength } -func (k *KeccakHasher[H]) FromBytes(in []byte) H { +func (k KeccakHasher) FromBytes(in []byte) KeccakHash { var buf = [KeccakHasherLength]byte{} copy(buf[:], in) - return H(NewKeccakHash(buf)) + return NewKeccakHash(buf) } -func (k *KeccakHasher[H]) Hash(in []byte) H { +func (k KeccakHasher) Hash(in []byte) KeccakHash { h := sha3.NewLegacyKeccak256() _, err := h.Write(in) @@ -45,8 +59,8 @@ func (k *KeccakHasher[H]) Hash(in []byte) H { return k.FromBytes(hash) } -func NewKeccakHasher[H KeccakHash]() KeccakHasher[H] { - return KeccakHasher[H]{} +func NewKeccakHasher() KeccakHasher { + return KeccakHasher{} } -var _ hashdb.Hasher[KeccakHash] = (*KeccakHasher[KeccakHash])(nil) +var _ hashdb.Hasher[KeccakHash] = (*KeccakHasher)(nil) diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go index 79c4167390..47ba81c9e8 100644 --- a/pkg/trie/triedb/cache.go +++ b/pkg/trie/triedb/cache.go @@ -3,10 +3,13 @@ package triedb -import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +) // CachedValue a value as cached by TrieCache -type CachedValue[H comparable] interface { +type CachedValue[H hashdb.HashOut] interface { Type() string } type ( @@ -27,7 +30,7 @@ func (v NonExisting) Type() string { return "NonExisting" } func (v ExistingHash[H]) Type() string { return "ExistingHash" } func (v Existing[H]) Type() string { return "Existing" } -type TrieCache[Out node.HashOut] interface { +type TrieCache[Out hashdb.HashOut] interface { LookupValueForKey(key []byte) *CachedValue[Out] CacheValueForKey(key []byte, value CachedValue[Out]) GetOrInsertNode(hash Out, fetchNode func() (node.NodeOwned[Out], error)) diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go index 54a7704721..adbc886309 100644 --- a/pkg/trie/triedb/layout.go +++ b/pkg/trie/triedb/layout.go @@ -8,9 +8,8 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" ) -type TrieLayout[Out node.HashOut] interface { +type TrieLayout[Out hashdb.HashOut] interface { AllowEmpty() bool MaxInlineValue() *uint - Hasher() hashdb.Hasher[Out] Codec() node.NodeCodec[Out] } diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 05d90fdc3e..5045db5761 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -11,7 +11,7 @@ import ( var EmptyValue = []byte{} -type Lookup[Hash node.HashOut] struct { +type Lookup[Hash hashdb.HashOut] struct { db hashdb.HashDB[Hash, DBValue] hash Hash cache TrieCache[Hash] @@ -19,7 +19,7 @@ type Lookup[Hash node.HashOut] struct { layout TrieLayout[Hash] } -func NewLookup[H node.HashOut]( +func NewLookup[H hashdb.HashOut]( db hashdb.HashDB[H, DBValue], hash H, cache TrieCache[H], recorder TrieRecorder[H]) *Lookup[H] { return &Lookup[H]{ db: db, @@ -107,7 +107,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er switch n := nextNode.(type) { case node.Hash: - nextHash := node.DecodeHash(n.Value, l.layout.Hasher()) + nextHash := node.DecodeHash(n.Value, l.layout.Codec().Hasher()) if nextHash == nil { return nil, InvalidHash } @@ -125,7 +125,7 @@ func (l Lookup[H]) loadValue(value node.Value, prefix nibble.Prefix) ([]byte, er case node.InlineValue: return v.Bytes, nil case node.NodeValue: - hash := l.layout.Hasher().FromBytes(v.Bytes) + hash := l.layout.Codec().Hasher().FromBytes(v.Bytes) bytes := l.db.Get(hash, prefix) if bytes == nil { return nil, ErrIncompleteDB diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 0245f074e6..0de39bd0d4 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -28,7 +28,7 @@ func (v InlineValue) Type() string { return "Inline" } func (v NodeValue) Type() string { return "Node" } // Nodes -type Node[H HashOut] interface { +type Node[H hashdb.HashOut] interface { Type() string } @@ -53,7 +53,7 @@ func (n Leaf) Type() string { return "Leaf" } func (n NibbledBranch) Type() string { return "NibbledBranch" } // NodeOwned is a trie node -type NodeOwned[H HashOut] interface { +type NodeOwned[H hashdb.HashOut] interface { Type() string } @@ -61,12 +61,12 @@ type ( // NodeEmptyNode represents an empty node NodeOwnedEmpty struct{} // NodeLeaf represents a leaf node - NodeOwnedLeaf[H HashOut] struct { + NodeOwnedLeaf[H hashdb.HashOut] struct { PartialKey nibble.NibbleSlice Value ValueOwned[H] } // NodeNibbledBranch represents a branch node - NodeOwnedNibbledBranch[H HashOut] struct { + NodeOwnedNibbledBranch[H hashdb.HashOut] struct { PartialKey nibble.NibbleSlice EncodedChildren [nibble.NibbleLength]NodeHandleOwned[H] Value ValueOwned[H] @@ -78,18 +78,18 @@ func (n NodeOwnedLeaf[H]) Type() string { return "Leaf" } func (n NodeOwnedNibbledBranch[H]) Type() string { return "NibbledBranch" } // Value is a trie node value -type ValueOwned[H HashOut] interface { +type ValueOwned[H hashdb.HashOut] interface { Type() string AsValue() Value } type ( // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value - InlineValueOwned[H HashOut] struct { + InlineValueOwned[H hashdb.HashOut] struct { bytes []byte hash H } // HashedNodeValue is a trie node pointer to a hashed node - NodeValueOwned[H HashOut] struct { + NodeValueOwned[H hashdb.HashOut] struct { hash H } ) @@ -100,19 +100,19 @@ func (v InlineValueOwned[H]) AsValue() Value { } func (v NodeValueOwned[H]) Type() string { return "Node" } func (v NodeValueOwned[H]) AsValue() Value { - return NodeValue{Bytes: v.hash.ToBytes()} + return NodeValue{Bytes: v.hash.Bytes()} } // NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandleOwned[H HashOut] interface { +type NodeHandleOwned[H hashdb.HashOut] interface { Type() string AsChildReference(codec NodeCodec[H]) ChildReference[H] } type ( - NodeHandleOwnedHash[H HashOut] struct { + NodeHandleOwnedHash[H hashdb.HashOut] struct { Hash H } - NodeHandleOwnedInline[H HashOut] struct { + NodeHandleOwnedInline[H hashdb.HashOut] struct { Node NodeOwned[H] } ) @@ -146,7 +146,7 @@ type ( func (h Hash) Type() string { return "Hash" } func (h Inline) Type() string { return "Inline" } -func DecodeHash[H HashOut](data []byte, hasher hashdb.Hasher[H]) *H { +func DecodeHash[H hashdb.HashOut](data []byte, hasher hashdb.Hasher[H]) *H { if len(data) != hasher.Length() { return nil } diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index 1a56ac41d8..412a2fbdcf 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -10,15 +10,15 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) -type ChildReference[H HashOut] interface { +type ChildReference[H hashdb.HashOut] interface { Type() string } type ( - ChildReferenceHash[H HashOut] struct { + ChildReferenceHash[H hashdb.HashOut] struct { hash H } - ChildReferenceInline[H HashOut] struct { + ChildReferenceInline[H hashdb.HashOut] struct { hash H length uint } @@ -27,12 +27,7 @@ type ( func (c ChildReferenceHash[H]) Type() string { return "Hash" } func (c ChildReferenceInline[H]) Type() string { return "Inline" } -type HashOut interface { - comparable - ToBytes() []byte -} - -type NodeCodec[H HashOut] interface { +type NodeCodec[H hashdb.HashOut] interface { HashedNullNode() H Hasher() hashdb.Hasher[H] EmptyNode() []byte @@ -46,7 +41,7 @@ type NodeCodec[H HashOut] interface { Decode(data []byte) (Node[H], error) } -func EncodeNodeOwned[H HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { +func EncodeNodeOwned[H hashdb.HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { switch n := node.(type) { case NodeOwnedEmpty: return codec.EmptyNode() diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go index bfe50c9cd6..d5a3b6d476 100644 --- a/pkg/trie/triedb/recorder.go +++ b/pkg/trie/triedb/recorder.go @@ -6,6 +6,7 @@ package triedb import ( "fmt" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" ) @@ -17,25 +18,25 @@ const ( RecordedForKeyNone ) -type TrieRecorder[Out node.HashOut] interface { +type TrieRecorder[Out hashdb.HashOut] interface { record(access TrieAccess[Out]) trieNodesRecordedForKey(key []byte) RecordedForKey } // TrieAccess is used to report the trie access to the TrieRecorder -type TrieAccess[Out node.HashOut] interface { +type TrieAccess[Out hashdb.HashOut] interface { Type() string } type ( // TrieAccessNode means that the given node was accessed using its hash - TrieAccessNodeOwned[H node.HashOut] struct { + TrieAccessNodeOwned[H hashdb.HashOut] struct { hash H nodeOwned node.NodeOwned[H] } // TrieAccessEncodedNode means that the given encodedNode was accessed using its hash - TrieAccessEncodedNode[H node.HashOut] struct { + TrieAccessEncodedNode[H hashdb.HashOut] struct { hash H encodedNode []byte } @@ -43,7 +44,7 @@ type ( // TrieAccessValue means that the given value was accessed using its hash // fullKey is the key to access this value in the trie // Should map to RecordedForKeyValue when checking the recorder - TrieAccessValue[H node.HashOut] struct { + TrieAccessValue[H hashdb.HashOut] struct { hash H value []byte fullKey []byte @@ -52,13 +53,13 @@ type ( // TrieAccessInlineValue means that a value stored in an inlined node was accessed // The given fullKey is the key to access this value in the trie // Should map to RecordedForKeyValue when checking the recorder - TrieAccessInlineValue[H node.HashOut] struct { + TrieAccessInlineValue[H hashdb.HashOut] struct { fullKey []byte } // TrieAccessHash means that the hash of the value for a given fullKey was accessed // Should map to RecordedForKeyHash when checking the recorder - TrieAccessHash[H node.HashOut] struct { + TrieAccessHash[H hashdb.HashOut] struct { fullKey []byte } @@ -78,21 +79,21 @@ func (a TrieAccessNonExisting) Type() string { return "NotExisting" } // Recorder implementation -type Record[H node.HashOut] struct { +type Record[H hashdb.HashOut] struct { /// The hash of the node. Hash H /// The data representing the node. Data []byte } -type Recorder[H node.HashOut] struct { +type Recorder[H hashdb.HashOut] struct { nodes []Record[H] recorderKeys map[string]RecordedForKey // TODO: revisit this later, it should be a BTreeMap layout TrieLayout[H] } // NewRecorder creates a new Recorder which records all given nodes -func NewRecorder[H node.HashOut]() *Recorder[H] { +func NewRecorder[H hashdb.HashOut]() *Recorder[H] { return &Recorder[H]{ nodes: make([]Record[H], 0), recorderKeys: make(map[string]RecordedForKey), diff --git a/pkg/trie/triedb/recorder_test.go b/pkg/trie/triedb/recorder_test.go deleted file mode 100644 index 6688d19516..0000000000 --- a/pkg/trie/triedb/recorder_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package triedb - -import ( - "testing" -) - -func Test_Record(t *testing.T) { - //TODO: implement -} diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go index 7a2561adaa..0c526c64e1 100644 --- a/pkg/trie/triedb/trie.go +++ b/pkg/trie/triedb/trie.go @@ -3,7 +3,9 @@ package triedb -import "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +) type TrieValue interface { Type() string @@ -13,10 +15,10 @@ type ( InlineTrieValue struct { Bytes []byte } - NodeTrieValue[H node.HashOut] struct { + NodeTrieValue[H hashdb.HashOut] struct { Hash H } - NewNodeTrie[H node.HashOut] struct { + NewNodeTrie[H hashdb.HashOut] struct { Hash *H Bytes []byte } @@ -34,7 +36,7 @@ func NewTrieValueFromBytes[H HashOut](value []byte, threshold *uint) TrieValue { } } -type Trie[Hash node.HashOut] interface { +type Trie[Hash hashdb.HashOut] interface { Root() Hash IsEmpty() bool Contains(key []byte) (bool, error) @@ -45,7 +47,7 @@ type Trie[Hash node.HashOut] interface { //lookup_first_descendant } -type MutableTrie[Hash node.HashOut] interface { +type MutableTrie[Hash hashdb.HashOut] interface { insert(key []byte, value []byte) (*TrieValue, error) remove(key []byte) (*TrieValue, error) } diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index ed1c36c038..083eb11b6f 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -12,7 +12,7 @@ import ( "github.com/gammazero/deque" ) -type HashOut = node_types.HashOut +type HashOut = hashdb.HashOut type Children = [nibble.NibbleLength]NodeHandle // Node types in the Trie @@ -81,8 +81,8 @@ type NodeStorage[H HashOut] struct { freeIndices deque.Deque[uint] } -func NewEmptyNodeStorage[H HashOut]() *NodeStorage[H] { - return &NodeStorage[H]{ +func NewEmptyNodeStorage[H HashOut]() NodeStorage[H] { + return NodeStorage[H]{ nodes: make([]Stored[H], 0), } } @@ -111,26 +111,12 @@ type TrieDB[Out HashOut] struct { db hashdb.HashDB[Out, DBValue] root Out rootHandle NodeHandle - deathRow map[Out]nibble.Prefix + deathRow map[string]nibble.Prefix cache TrieCache[Out] recorder TrieRecorder[Out] layout TrieLayout[Out] } -func NewTrieDB[H HashOut]( - db hashdb.HashDB[H, DBValue], - root H, - cache TrieCache[H], - recorder TrieRecorder[H], -) *TrieDB[H] { - return &TrieDB[H]{ - db: db, - root: root, - cache: cache, - recorder: recorder, - } -} - type RemoveAtResult struct { handle StorageHandle changed bool @@ -223,10 +209,10 @@ func (tdb *TrieDB[H]) inspect( case PostInspectActionRestore[H]: result = InspectResult[H]{StoredCached[H]{s.Hash, action.node}, false} case PostInspectActionReplace[H]: - tdb.deathRow[s.Hash] = key.Left() + tdb.deathRow[s.Hash.ComparableKey()] = key.Left() result = InspectResult[H]{StoredNew[H]{action.node}, true} case PostInspectActionDelete: - tdb.deathRow[s.Hash] = key.Left() + tdb.deathRow[s.Hash.ComparableKey()] = key.Left() return nil, nil } } @@ -304,7 +290,7 @@ func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) case StoredNew[H]: childNode = s.Node case StoredCached[H]: - tdb.deathRow[s.Hash] = childPrefix + tdb.deathRow[s.Hash.ComparableKey()] = childPrefix childNode = s.Node } @@ -569,10 +555,10 @@ func (tdb *TrieDB[H]) replaceOldValue( switch n := storedValue.(type) { case NewNodeTrie[H]: if n.Hash != nil { - tdb.deathRow[*n.Hash] = prefix + tdb.deathRow[(*n.Hash).ComparableKey()] = prefix } case NodeTrieValue[H]: - tdb.deathRow[n.Hash] = prefix + tdb.deathRow[n.Hash.ComparableKey()] = prefix } *oldVal = storedValue } diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go index 9ddeec5bae..d944674a05 100644 --- a/pkg/trie/triedb/triedbbuilder.go +++ b/pkg/trie/triedb/triedbbuilder.go @@ -5,23 +5,25 @@ package triedb import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) type DBValue = []byte -type TrieDBBuilder[Out node.HashOut] struct { - db hashdb.HashDB[Out, DBValue] - root Out - cache TrieCache[Out] - recorder TrieRecorder[Out] +type TrieDBBuilder[H hashdb.HashOut] struct { + db hashdb.HashDB[H, DBValue] + root H + cache TrieCache[H] + recorder TrieRecorder[H] + layout TrieLayout[H] } -func NewTrieDBBuilder[Out node.HashOut]( - db hashdb.HashDB[Out, DBValue], - root Out, -) *TrieDBBuilder[Out] { - return &TrieDBBuilder[Out]{ +func NewTrieDBBuilder[H hashdb.HashOut]( + db hashdb.HashDB[H, DBValue], + root H, + layout TrieLayout[H], +) *TrieDBBuilder[H] { + return &TrieDBBuilder[H]{ db: db, root: root, cache: nil, @@ -29,16 +31,27 @@ func NewTrieDBBuilder[Out node.HashOut]( } } -func (tdbb *TrieDBBuilder[Out]) WithCache(cache TrieCache[Out]) *TrieDBBuilder[Out] { - tdbb.cache = cache - return tdbb +func (self *TrieDBBuilder[H]) WithCache(cache TrieCache[H]) *TrieDBBuilder[H] { + self.cache = cache + return self } -func (tdbb *TrieDBBuilder[Out]) WithRecorder(recorder TrieRecorder[Out]) *TrieDBBuilder[Out] { - tdbb.recorder = recorder - return tdbb +func (self *TrieDBBuilder[H]) WithRecorder(recorder TrieRecorder[H]) *TrieDBBuilder[H] { + self.recorder = recorder + return self } -func (tdbb *TrieDBBuilder[Out]) Build() *TrieDB[Out] { - return NewTrieDB(tdbb.db, tdbb.root, tdbb.cache, tdbb.recorder) +func (self *TrieDBBuilder[H]) Build() *TrieDB[H] { + rootHandle := Hash[H]{self.root} + + return &TrieDB[H]{ + db: self.db, + root: self.root, + cache: self.cache, + recorder: self.recorder, + storage: NewEmptyNodeStorage[H](), + deathRow: make(map[string]nibble.Prefix), + rootHandle: &rootHandle, + layout: self.layout, + } } From b18dc65c1c65edbeff131fd14fb69d15e277591f Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 15 Jan 2024 11:39:06 -0300 Subject: [PATCH 16/35] feat(pkg/trie): Implements memoryDB --- pkg/trie/memorydb/memorydb.go | 80 ++++++++++++++++--- .../keccak_hasher/keccak_hasher.go | 26 ++---- 2 files changed, 76 insertions(+), 30 deletions(-) diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index 9eaf7fef5e..141d5e6f27 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -4,6 +4,8 @@ package memorydb import ( + "bytes" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) @@ -18,6 +20,7 @@ type MemoryDB[H hashdb.HashOut] struct { hashedNullNode H nullNodeData []byte keyFunction KeyFunction[H] + hasher hashdb.Hasher[H] } func newFromNullNode[H hashdb.HashOut]( @@ -34,24 +37,81 @@ func newFromNullNode[H hashdb.HashOut]( } } -func (db *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { - panic("Implement me") +func (self *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { + if key.ComparableKey() == self.hashedNullNode.ComparableKey() { + return &self.nullNodeData + } + + key = self.keyFunction(key, prefix, self.hasher) + value, ok := self.data[key.ComparableKey()] + if ok && value.rc > 0 { + return &value.value + } + + return nil } -func (db *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { - panic("Implement me") +func (self *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { + if key.ComparableKey() == self.hashedNullNode.ComparableKey() { + return true + } + + key = self.keyFunction(key, prefix, self.hasher) + value, ok := self.data[key.ComparableKey()] + if ok && value.rc > 0 { + return true + } + + return false } -func (db *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { - panic("Implement me") +func (self *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { + if bytes.Equal(value, self.nullNodeData) { + return self.hashedNullNode + } + + key := self.keyFunction(self.hasher.Hash(value), prefix, self.hasher) + self.Emplace(key, prefix, value) + return key } -func (db *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { - panic("Implement me") +func (self *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { + if bytes.Equal(value, self.nullNodeData) { + return + } + + key = self.keyFunction(key, prefix, self.hasher) + + newEntry := MemoryDBValue{ + value: value, + rc: 1, + } + + currentEntry, ok := self.data[key.ComparableKey()] + if ok { + if currentEntry.rc <= 0 { + newEntry.value = value + } + newEntry.rc = currentEntry.rc + 1 + } + + self.data[key.ComparableKey()] = newEntry } -func (db *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { - panic("Implement me") +func (self *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { + if key.ComparableKey() == self.hashedNullNode.ComparableKey() { + return + } + + key = self.keyFunction(key, prefix, self.hasher) + + entry, ok := self.data[key.ComparableKey()] + if ok { + entry.rc-- + self.data[key.ComparableKey()] = entry + } else { + delete(self.data, key.ComparableKey()) + } } func NewMemoryDB[Hash hashdb.HashOut]( diff --git a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go index ffb1ae7cd3..a0fda0053c 100644 --- a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go +++ b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go @@ -1,36 +1,22 @@ package keccak_hasher import ( + "fmt" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "golang.org/x/crypto/sha3" ) const KeccakHasherLength = 32 -type KeccakHash struct { - bytes [KeccakHasherLength]byte -} - -func NewKeccakHash(bytes [KeccakHasherLength]byte) KeccakHash { - return KeccakHash{ - bytes: bytes, - } -} +type KeccakHash [KeccakHasherLength]byte func (k KeccakHash) Bytes() []byte { - return k.bytes[:] + return k[:] } func (k KeccakHash) ComparableKey() string { - return string(k.Bytes()) -} - -func KeccakHashFromBytes(b []byte) KeccakHash { - var newBytes [KeccakHasherLength]byte - copy(newBytes[:], b) - return KeccakHash{ - bytes: newBytes, - } + return fmt.Sprintf("%x", k) } var _ hashdb.HashOut = KeccakHash{} @@ -44,7 +30,7 @@ func (k KeccakHasher) Length() int { func (k KeccakHasher) FromBytes(in []byte) KeccakHash { var buf = [KeccakHasherLength]byte{} copy(buf[:], in) - return NewKeccakHash(buf) + return KeccakHash(buf) } func (k KeccakHasher) Hash(in []byte) KeccakHash { From 3f5d116b504a2d0b90a274198cdc2dab38ce1378 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 15 Jan 2024 12:15:59 -0300 Subject: [PATCH 17/35] fix(pkg/trie): fixes and add missing license --- pkg/trie/memorydb/keyfunction.go | 3 +++ pkg/trie/test_support/keccak_hasher/keccak_hasher.go | 3 +++ pkg/trie/triedb/triedb.go | 2 +- pkg/trie/triedb/triedbbuilder.go | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/trie/memorydb/keyfunction.go b/pkg/trie/memorydb/keyfunction.go index 47f686ff6e..05db9a1eb2 100644 --- a/pkg/trie/memorydb/keyfunction.go +++ b/pkg/trie/memorydb/keyfunction.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package memorydb import ( diff --git a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go index a0fda0053c..20ce958298 100644 --- a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go +++ b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package keccak_hasher import ( diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 083eb11b6f..cb7232584b 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -95,7 +95,7 @@ func (ns *NodeStorage[H]) alloc(stored Stored[H]) StorageHandle { } ns.nodes = append(ns.nodes, stored) - return uint(len(ns.nodes) - 1) + return StorageHandle(len(ns.nodes) - 1) } func (ns *NodeStorage[H]) destroy(handle StorageHandle) Stored[H] { diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go index d944674a05..5d630c2b32 100644 --- a/pkg/trie/triedb/triedbbuilder.go +++ b/pkg/trie/triedb/triedbbuilder.go @@ -28,6 +28,7 @@ func NewTrieDBBuilder[H hashdb.HashOut]( root: root, cache: nil, recorder: nil, + layout: layout, } } From 6addce755e1415bff76550646a453bbe661dd852 Mon Sep 17 00:00:00 2001 From: Diego Date: Mon, 15 Jan 2024 15:44:35 -0300 Subject: [PATCH 18/35] fix(pkg/trie): Fix lookup depth --- .../test_support/reference_trie/substrate.go | 68 +++++++++++++++++++ pkg/trie/tests/recorder_test.go | 49 +++++++++++++ pkg/trie/triedb/lookup.go | 2 +- 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 pkg/trie/test_support/reference_trie/substrate.go create mode 100644 pkg/trie/tests/recorder_test.go diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go new file mode 100644 index 0000000000..9091d85e7d --- /dev/null +++ b/pkg/trie/test_support/reference_trie/substrate.go @@ -0,0 +1,68 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package reference_trie + +import ( + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" + "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" + "github.com/ChainSafe/gossamer/pkg/trie/triedb" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" +) + +const FirstPrefix = 0b_00 << 6 +const EmptyTree = FirstPrefix | (0b_00 << 4) + +var hasher = keccak_hasher.NewKeccakHasher() + +type NodeCodec[H hashdb.HashOut] struct { + hasher hashdb.Hasher[H] +} + +func (self NodeCodec[H]) HashedNullNode() H { + return self.hasher.Hash(self.EmptyNode()) +} + +func (self NodeCodec[H]) Hasher() hashdb.Hasher[keccak_hasher.KeccakHash] { + return hasher +} + +func (self NodeCodec[H]) EmptyNode() []byte { + return []byte{EmptyTree} +} + +func (self NodeCodec[H]) LeafNode(partialKey nibble.NibbleSlice, numberNibble int, value node.Value) []byte { + panic("Implement me") +} + +func (self NodeCodec[H]) BranchNodeNibbled( + partialKey nibble.NibbleSlice, + numberNibble int, + children [16]node.ChildReference[H], + value *node.Value, +) []byte { + panic("Implement me") +} + +func (self NodeCodec[H]) Decode(data []byte) (node.Node[H], error) { + panic("Implement me") +} + +var _ node.NodeCodec[keccak_hasher.KeccakHash] = NodeCodec[keccak_hasher.KeccakHash]{} + +type LayoutV0[H hashdb.HashOut] struct{} + +func (l LayoutV0[H]) AllowEmpty() bool { + return true +} + +func (l LayoutV0[H]) MaxInlineValue() *uint { + return nil +} + +func (l LayoutV0[H]) Codec() node.NodeCodec[H] { + panic("Implement me") +} + +var V0Layout triedb.TrieLayout[hashdb.HashOut] = LayoutV0[hashdb.HashOut]{} diff --git a/pkg/trie/tests/recorder_test.go b/pkg/trie/tests/recorder_test.go new file mode 100644 index 0000000000..a204e3c029 --- /dev/null +++ b/pkg/trie/tests/recorder_test.go @@ -0,0 +1,49 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package tests + +import ( + "testing" + + "github.com/ChainSafe/gossamer/pkg/trie/memorydb" + "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" + "github.com/ChainSafe/gossamer/pkg/trie/test_support/reference_trie" + "github.com/ChainSafe/gossamer/pkg/trie/triedb" + "github.com/stretchr/testify/assert" +) + +type KeccakHash = keccak_hasher.KeccakHash + +var hasher = keccak_hasher.NewKeccakHasher() +var V0Layout = reference_trie.LayoutV0[KeccakHash]{} + +func Test_Record(t *testing.T) { + db := memorydb.NewMemoryDB[KeccakHash](keccak_hasher.NewKeccakHasher(), memorydb.HashKey[KeccakHash]) + + rootBytes := make([]byte, 32) + root := hasher.FromBytes(rootBytes) + + { + pairs := []struct { + key []byte + value []byte + }{ + {[]byte("dog"), []byte("cat")}, + {[]byte("lunch"), []byte("time")}, + {[]byte("notdog"), []byte("notcat")}, + {[]byte("hotdog"), []byte("hotcat")}, + {[]byte("letter"), []byte("confusion")}, + {[]byte("insert"), []byte("remove")}, + {[]byte("pirate"), []byte("aargh!")}, + {[]byte("yo ho ho"), []byte("and a bottle of rum")}, + } + + tdb := triedb.NewTrieDBBuilder[KeccakHash](db, root, V0Layout).Build() + + for _, pair := range pairs { + _, err := tdb.Insert(pair.key, pair.value) + assert.NoError(t, err) + } + } +} diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 5045db5761..6fb81c7ac2 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -115,8 +115,8 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er case node.Inline: nodeData = &n.Value } + depth++ } - depth++ } } From 4e604d28c91e3552a0fb6573810a50e9dbce2cd3 Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 16 Jan 2024 17:59:01 -0300 Subject: [PATCH 19/35] feat(pkg/trie): Adds memorydb tests --- pkg/trie/memorydb/memorydb.go | 105 ++++++++++++++++++++++++- pkg/trie/memorydb/memorydb_test.go | 121 +++++++++++++++++++++++++++++ pkg/trie/triedb/nibble/nibble.go | 2 + 3 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 pkg/trie/memorydb/memorydb_test.go diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go index 141d5e6f27..6fa304c2f6 100644 --- a/pkg/trie/memorydb/memorydb.go +++ b/pkg/trie/memorydb/memorydb.go @@ -12,7 +12,7 @@ import ( type MemoryDBValue struct { value []byte - rc int32 + rc int } type MemoryDB[H hashdb.HashOut] struct { @@ -34,9 +34,29 @@ func newFromNullNode[H hashdb.HashOut]( hashedNullNode: hasher.Hash(nullKey), nullNodeData: nullNodeData, keyFunction: keyFunction, + hasher: hasher, } } +// Raw returns the raw value for the given key +func (self *MemoryDB[H]) Raw(key H, prefix nibble.Prefix) *MemoryDBValue { + if key.ComparableKey() == self.hashedNullNode.ComparableKey() { + return &MemoryDBValue{ + value: self.nullNodeData, + rc: 1, + } + } + + key = self.keyFunction(key, prefix, self.hasher) + value, ok := self.data[key.ComparableKey()] + if ok { + return &value + } + + return nil +} + +// Get returns the value for the given key func (self *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { if key.ComparableKey() == self.hashedNullNode.ComparableKey() { return &self.nullNodeData @@ -51,6 +71,7 @@ func (self *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { return nil } +// Contains returns true if the key exists in the database func (self *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { if key.ComparableKey() == self.hashedNullNode.ComparableKey() { return true @@ -65,6 +86,7 @@ func (self *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { return false } +// Insert inserts a value into the database and returns the key func (self *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { if bytes.Equal(value, self.nullNodeData) { return self.hashedNullNode @@ -75,6 +97,7 @@ func (self *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { return key } +// Emplace inserts a value into the database an updates its reference count func (self *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { if bytes.Equal(value, self.nullNodeData) { return @@ -98,6 +121,7 @@ func (self *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { self.data[key.ComparableKey()] = newEntry } +// Remove removes reduces the reference count for that key by 1 or set -1 if the value does not exists func (self *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { if key.ComparableKey() == self.hashedNullNode.ComparableKey() { return @@ -110,10 +134,77 @@ func (self *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { entry.rc-- self.data[key.ComparableKey()] = entry } else { - delete(self.data, key.ComparableKey()) + self.data[key.ComparableKey()] = MemoryDBValue{ + value: nil, + rc: -1, + } + } +} + +// RemoveAndPurge removes an element and delete it from storage if reference count reaches 0. +// If the value was purged, return the old value. +func (self *MemoryDB[H]) RemoveAndPurge(key H, prefix nibble.Prefix) *[]byte { + if key.ComparableKey() == self.hashedNullNode.ComparableKey() { + return nil + } + + key = self.keyFunction(key, prefix, self.hasher) + + entry, ok := self.data[key.ComparableKey()] + if ok { + if entry.rc == 1 { + delete(self.data, key.ComparableKey()) + return &entry.value + } + entry.rc-- + self.data[key.ComparableKey()] = entry + return nil + } + + self.data[key.ComparableKey()] = MemoryDBValue{ + value: nil, + rc: -1, + } + + return nil +} + +// Purge purges all zero-referenced data from the database +func (self *MemoryDB[H]) Purge() { + for key, value := range self.data { + if value.rc == 0 { + delete(self.data, key) + } } } +// Drain returns the internal key-value map, clearing the current state +func (self *MemoryDB[H]) Drain() map[string]MemoryDBValue { + data := self.data + self.data = make(map[string]MemoryDBValue) + return data +} + +// Consolidate all the entries of `other` into `self` +func (self *MemoryDB[H]) Consolidate(other *MemoryDB[H]) { + for key, dbvalue := range other.Drain() { + entry, ok := self.data[key] + if ok { + if entry.rc < 0 { + entry.value = dbvalue.value + } + entry.rc += dbvalue.rc + self.data[key] = entry + } else { + self.data[key] = MemoryDBValue{ + value: dbvalue.value, + rc: dbvalue.rc, + } + } + } +} + +// NewMemoryDB creates a new memoryDB with a null node data func NewMemoryDB[Hash hashdb.HashOut]( hasher hashdb.Hasher[Hash], keyFunction KeyFunction[Hash], @@ -121,3 +212,13 @@ func NewMemoryDB[Hash hashdb.HashOut]( data := []byte{0x0} return newFromNullNode(data, data, hasher, keyFunction) } + +// NewMemoryDBWithRoot creates a new memoryDB with a null node data and returns the DB and the root +func NewMemoryDBWithRoot[Hash hashdb.HashOut]( + hasher hashdb.Hasher[Hash], + keyFunction KeyFunction[Hash], +) (*MemoryDB[Hash], Hash) { + db := NewMemoryDB(hasher, keyFunction) + root := db.hashedNullNode + return db, root +} diff --git a/pkg/trie/memorydb/memorydb_test.go b/pkg/trie/memorydb/memorydb_test.go new file mode 100644 index 0000000000..4aa03cfede --- /dev/null +++ b/pkg/trie/memorydb/memorydb_test.go @@ -0,0 +1,121 @@ +package memorydb + +import ( + "testing" + + "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" + "github.com/ChainSafe/gossamer/pkg/trie/test_support/reference_trie" + "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + "github.com/stretchr/testify/require" +) + +type KeccakHash = keccak_hasher.KeccakHash + +var hasher = keccak_hasher.NewKeccakHasher() +var V0Layout = reference_trie.LayoutV0[KeccakHash]{} + +var nullNode = []byte{0x0} +var emptyPrefix = nibble.EmptyPrefix + +func Test_New(t *testing.T) { + db := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) + hashedNullNode := hasher.Hash(nullNode) + require.Equal(t, hashedNullNode, db.Insert(emptyPrefix, nullNode)) + + db2, root := NewMemoryDBWithRoot[KeccakHash](hasher, HashKey[KeccakHash]) + require.True(t, db2.Contains(root, emptyPrefix)) + require.True(t, db.Contains(root, emptyPrefix)) +} + +func Test_Remove(t *testing.T) { + helloBytes := []byte("hello world!") + helloKey := hasher.Hash(helloBytes) + + t.Run("Remove purge insert purge", func(t *testing.T) { + m := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) + m.Remove(helloKey, emptyPrefix) + dbValue := m.Raw(helloKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, -1, dbValue.rc) + + m.Purge() + dbValue = m.Raw(helloKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, -1, dbValue.rc) + + m.Insert(emptyPrefix, helloBytes) + dbValue = m.Raw(helloKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, 0, dbValue.rc) + + m.Purge() + dbValue = m.Raw(helloKey, emptyPrefix) + require.Nil(t, dbValue) + }) + + t.Run("Remove and purge", func(t *testing.T) { + m := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) + res := m.RemoveAndPurge(helloKey, emptyPrefix) + require.Nil(t, res) + + dbValue := m.Raw(helloKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, -1, dbValue.rc) + + m.Insert(emptyPrefix, helloBytes) + m.Insert(emptyPrefix, helloBytes) + + dbValue = m.Raw(helloKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, 1, dbValue.rc) + + res = m.RemoveAndPurge(helloKey, emptyPrefix) + require.NotNil(t, res) + require.Equal(t, helloBytes, *res) + + dbValue = m.Raw(helloKey, emptyPrefix) + require.Nil(t, dbValue) + + res = m.RemoveAndPurge(helloKey, emptyPrefix) + require.Nil(t, res) + }) +} + +func Test_Consolidate(t *testing.T) { + main := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) + other := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) + + removeKey := other.Insert(emptyPrefix, []byte("doggo")) + main.Remove(removeKey, emptyPrefix) + + insertKey := other.Insert(emptyPrefix, []byte("arf")) + main.Emplace(insertKey, emptyPrefix, []byte("arf")) + + negativeRemoveKey := other.Insert(emptyPrefix, []byte("negative")) + other.Remove(negativeRemoveKey, emptyPrefix) // rc = 0 + other.Remove(negativeRemoveKey, emptyPrefix) // rc = -1 + main.Remove(negativeRemoveKey, emptyPrefix) // rc = -1 + + main.Consolidate(other) + + t.Run("removeKey with rc=0", func(t *testing.T) { + dbValue := main.Raw(removeKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, []byte("doggo"), dbValue.value) + require.Equal(t, 0, dbValue.rc) + }) + + t.Run("insertKey with rc=2", func(t *testing.T) { + dbValue := main.Raw(insertKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, []byte("arf"), dbValue.value) + require.Equal(t, 2, dbValue.rc) + }) + + t.Run("negativeRemoveKey with rc=-2", func(t *testing.T) { + dbValue := main.Raw(negativeRemoveKey, emptyPrefix) + require.NotNil(t, dbValue) + require.Equal(t, []byte("negative"), dbValue.value) + require.Equal(t, -2, dbValue.rc) + }) +} diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go index 024944cf5e..b17548c4b6 100644 --- a/pkg/trie/triedb/nibble/nibble.go +++ b/pkg/trie/triedb/nibble/nibble.go @@ -23,6 +23,8 @@ type Prefix struct { PaddedByte *byte } +var EmptyPrefix = Prefix{PartialKey: []byte{}, PaddedByte: nil} + func PadLeft(b byte) byte { padded := (b & ^PaddingBitmask) return padded From 70df09d436a4ce2035b0fbc55e106c3089b68a5a Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 17 Jan 2024 20:05:45 -0300 Subject: [PATCH 20/35] fix(pkg/trie): Node from encoded and cache lookup --- .../test_support/reference_trie/substrate.go | 16 ++- pkg/trie/tests/recorder_test.go | 42 ++++--- pkg/trie/triedb/layout.go | 4 +- pkg/trie/triedb/lookup.go | 4 +- pkg/trie/triedb/node/node.go | 13 ++- pkg/trie/triedb/triedb.go | 109 +++++++++++++++--- pkg/trie/triedb/triedbbuilder.go | 4 +- 7 files changed, 136 insertions(+), 56 deletions(-) diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go index 9091d85e7d..a7cc589239 100644 --- a/pkg/trie/test_support/reference_trie/substrate.go +++ b/pkg/trie/test_support/reference_trie/substrate.go @@ -24,8 +24,8 @@ func (self NodeCodec[H]) HashedNullNode() H { return self.hasher.Hash(self.EmptyNode()) } -func (self NodeCodec[H]) Hasher() hashdb.Hasher[keccak_hasher.KeccakHash] { - return hasher +func (self NodeCodec[H]) Hasher() hashdb.Hasher[H] { + return self.hasher } func (self NodeCodec[H]) EmptyNode() []byte { @@ -49,9 +49,15 @@ func (self NodeCodec[H]) Decode(data []byte) (node.Node[H], error) { panic("Implement me") } +func NewNodeCodecForKeccak() NodeCodec[keccak_hasher.KeccakHash] { + return NodeCodec[keccak_hasher.KeccakHash]{hasher} +} + var _ node.NodeCodec[keccak_hasher.KeccakHash] = NodeCodec[keccak_hasher.KeccakHash]{} -type LayoutV0[H hashdb.HashOut] struct{} +type LayoutV0[H hashdb.HashOut] struct { + codec node.NodeCodec[H] +} func (l LayoutV0[H]) AllowEmpty() bool { return true @@ -62,7 +68,7 @@ func (l LayoutV0[H]) MaxInlineValue() *uint { } func (l LayoutV0[H]) Codec() node.NodeCodec[H] { - panic("Implement me") + return l.codec } -var V0Layout triedb.TrieLayout[hashdb.HashOut] = LayoutV0[hashdb.HashOut]{} +var V0Layout triedb.TrieLayout[keccak_hasher.KeccakHash] = LayoutV0[keccak_hasher.KeccakHash]{NewNodeCodecForKeccak()} diff --git a/pkg/trie/tests/recorder_test.go b/pkg/trie/tests/recorder_test.go index a204e3c029..e50ad420fb 100644 --- a/pkg/trie/tests/recorder_test.go +++ b/pkg/trie/tests/recorder_test.go @@ -16,7 +16,7 @@ import ( type KeccakHash = keccak_hasher.KeccakHash var hasher = keccak_hasher.NewKeccakHasher() -var V0Layout = reference_trie.LayoutV0[KeccakHash]{} +var V0Layout = reference_trie.V0Layout func Test_Record(t *testing.T) { db := memorydb.NewMemoryDB[KeccakHash](keccak_hasher.NewKeccakHasher(), memorydb.HashKey[KeccakHash]) @@ -24,26 +24,24 @@ func Test_Record(t *testing.T) { rootBytes := make([]byte, 32) root := hasher.FromBytes(rootBytes) - { - pairs := []struct { - key []byte - value []byte - }{ - {[]byte("dog"), []byte("cat")}, - {[]byte("lunch"), []byte("time")}, - {[]byte("notdog"), []byte("notcat")}, - {[]byte("hotdog"), []byte("hotcat")}, - {[]byte("letter"), []byte("confusion")}, - {[]byte("insert"), []byte("remove")}, - {[]byte("pirate"), []byte("aargh!")}, - {[]byte("yo ho ho"), []byte("and a bottle of rum")}, - } - - tdb := triedb.NewTrieDBBuilder[KeccakHash](db, root, V0Layout).Build() - - for _, pair := range pairs { - _, err := tdb.Insert(pair.key, pair.value) - assert.NoError(t, err) - } + pairs := []struct { + key []byte + value []byte + }{ + {[]byte("dog"), []byte("cat")}, + /*{[]byte("lunch"), []byte("time")}, + {[]byte("notdog"), []byte("notcat")}, + {[]byte("hotdog"), []byte("hotcat")}, + {[]byte("letter"), []byte("confusion")}, + {[]byte("insert"), []byte("remove")}, + {[]byte("pirate"), []byte("aargh!")}, + {[]byte("yo ho ho"), []byte("and a bottle of rum")},*/ + } + + tdb := triedb.NewTrieDBBuilder[KeccakHash](db, root, V0Layout).Build() + + for _, pair := range pairs { + _, err := tdb.Insert(pair.key, pair.value) + assert.NoError(t, err) } } diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go index adbc886309..65c947294d 100644 --- a/pkg/trie/triedb/layout.go +++ b/pkg/trie/triedb/layout.go @@ -8,8 +8,8 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" ) -type TrieLayout[Out hashdb.HashOut] interface { +type TrieLayout[H hashdb.HashOut] interface { AllowEmpty() bool MaxInlineValue() *uint - Codec() node.NodeCodec[Out] + Codec() node.NodeCodec[H] } diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 6fb81c7ac2..67ca99f49d 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -107,13 +107,13 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er switch n := nextNode.(type) { case node.Hash: - nextHash := node.DecodeHash(n.Value, l.layout.Codec().Hasher()) + nextHash := node.DecodeHash(n.Data, l.layout.Codec().Hasher()) if nextHash == nil { return nil, InvalidHash } hash = *nextHash case node.Inline: - nodeData = &n.Value + nodeData = &n.Data } depth++ } diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 0de39bd0d4..ff376b549d 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -4,6 +4,8 @@ package node import ( + "errors" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) @@ -136,20 +138,19 @@ type NodeHandle interface { } type ( Hash struct { - Value []byte + Data []byte } Inline struct { - Value []byte + Data []byte } ) func (h Hash) Type() string { return "Hash" } func (h Inline) Type() string { return "Inline" } -func DecodeHash[H hashdb.HashOut](data []byte, hasher hashdb.Hasher[H]) *H { +func DecodeHash[H hashdb.HashOut](hasher hashdb.Hasher[H], data []byte) (H, error) { if len(data) != hasher.Length() { - return nil + return hasher.FromBytes([]byte{}), errors.New("decoding hash") } - hash := hasher.FromBytes(data) - return &hash + return hasher.FromBytes(data), nil } diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index cb7232584b..7e63d9f7f8 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -8,7 +8,7 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - node_types "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" + encoded_nodes "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" "github.com/gammazero/deque" ) @@ -130,23 +130,38 @@ func (tdb *TrieDB[H]) record( } } -func (tdb *TrieDB[H]) lookupAndCache( +func (tdb *TrieDB[H]) getAndCache( hash H, key nibble.Prefix, ) (StorageHandle, error) { + // We only check the `cache` for a node with `get_node` and don't insert + // the node if it wasn't there, because in substrate we only access the node while computing + // a new trie (aka some branch). We assume that this node isn't that important to have it being cached. var node Node[H] - nodeFromCache := tdb.cache.GetNode(hash) - if nodeFromCache != nil { - tdb.record(TrieAccessNodeOwned[H]{hash: hash, nodeOwned: *nodeFromCache}) - node = NodeFromNodeOwned(*nodeFromCache, tdb.storage) - } else { + hasCache := tdb.cache != nil + + if hasCache { + nodeFromCache := tdb.cache.GetNode(hash) + if nodeFromCache != nil { + tdb.record(TrieAccessNodeOwned[H]{hash: hash, nodeOwned: *nodeFromCache}) + node = NodeFromNodeOwned(*nodeFromCache, tdb.storage) + } + } + + if node == nil { nodeEncoded := tdb.db.Get(hash, key) if nodeEncoded == nil { return 0, ErrIncompleteDB } tdb.record(TrieAccessEncodedNode[H]{hash: hash, encodedNode: *nodeEncoded}) + + var err error + node, err = NodeFromEncoded(hash, *nodeEncoded, tdb.storage, tdb.layout) + if err != nil { + return 0, err + } } return tdb.storage.alloc(StoredCached[H]{Node: node, Hash: hash}), nil @@ -277,7 +292,7 @@ func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) case InMemory: stored = tdb.storage.destroy(c.Value) case Hash[H]: - handle, err := tdb.lookupAndCache(c.Value, childPrefix) + handle, err := tdb.getAndCache(c.Value, childPrefix) if err != nil { return nil, err } @@ -575,7 +590,7 @@ func (tdb *TrieDB[H]) removeAt( case InMemory: stored = tdb.storage.destroy(h.Value) case Hash[H]: - fromCache, err := tdb.lookupAndCache(h.Value, key.Left()) + fromCache, err := tdb.getAndCache(h.Value, key.Left()) if err != nil { return nil, err } @@ -636,7 +651,7 @@ func (tdb *TrieDB[H]) insertAt( case InMemory: storageHandle = h.Value case Hash[H]: - storageHandle, err = tdb.lookupAndCache(h.Value, key.Left()) + storageHandle, err = tdb.getAndCache(h.Value, key.Left()) if err != nil { return InsertAtResult{}, err } @@ -676,28 +691,47 @@ func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { return oldVal, nil } -func inlineOrHashOwned[H HashOut](child node_types.NodeHandleOwned[H], storage NodeStorage[H]) NodeHandle { +func inlineOrHashOwned[H HashOut](child encoded_nodes.NodeHandleOwned[H], storage NodeStorage[H]) NodeHandle { switch n := child.(type) { - case node_types.NodeHandleOwnedHash[H]: + case encoded_nodes.NodeHandleOwnedHash[H]: return Hash[H]{n.Hash} - case node_types.NodeHandleOwnedInline[H]: - child := NodeFromNodeOwned[H](n.Node, storage) + case encoded_nodes.NodeHandleOwnedInline[H]: + child := NodeFromNodeOwned(n.Node, storage) return InMemory{storage.alloc(StoredNew[H]{child})} default: panic("Invalid child") } } -func NodeFromNodeOwned[H HashOut](nodeOwned node_types.NodeOwned[H], storage NodeStorage[H]) Node[H] { +func inlineOrHash[H HashOut](parentHash H, child encoded_nodes.NodeHandle, storage NodeStorage[H], layout TrieLayout[H]) (NodeHandle, error) { + switch n := child.(type) { + case encoded_nodes.Hash: + hash, err := encoded_nodes.DecodeHash[H](layout.Codec().Hasher(), n.Data) + if err != nil { + return nil, errors.New("invalid hash") + } + return Hash[H]{hash}, nil + case encoded_nodes.Inline: + child, err := NodeFromEncoded(parentHash, n.Data, storage, layout) + if err != nil { + return nil, err + } + return InMemory{storage.alloc(StoredNew[H]{child})}, nil + default: + panic("Invalid child") + } +} + +func NodeFromNodeOwned[H HashOut](nodeOwned encoded_nodes.NodeOwned[H], storage NodeStorage[H]) Node[H] { switch node := nodeOwned.(type) { - case node_types.NodeOwnedEmpty: + case encoded_nodes.NodeOwnedEmpty: return Empty{} - case node_types.NodeOwnedLeaf[H]: + case encoded_nodes.NodeOwnedLeaf[H]: return Leaf{ encoded: node.PartialKey, value: node.Value, } - case node_types.NodeOwnedNibbledBranch[H]: + case encoded_nodes.NodeOwnedNibbledBranch[H]: child := func(i uint) NodeHandle { if node.EncodedChildren[i] != nil { return inlineOrHashOwned(node.EncodedChildren[i], storage) @@ -719,3 +753,42 @@ func NodeFromNodeOwned[H HashOut](nodeOwned node_types.NodeOwned[H], storage Nod panic("Invalid node") } } + +func NodeFromEncoded[H HashOut](nodeHash H, data []byte, storage NodeStorage[H], layout TrieLayout[H]) (Node[H], error) { + decodedNode, err := layout.Codec().Decode(data) + if err != nil { + return nil, errors.New("decoding node") + } + switch node := decodedNode.(type) { + case encoded_nodes.Empty: + return Empty{}, nil + case encoded_nodes.Leaf: + return Leaf{ + encoded: node.PartialKey, + value: node.Value, + }, nil + case encoded_nodes.NibbledBranch: + child := func(i uint) (NodeHandle, error) { + if node.Children[i] != nil { + return inlineOrHash[H](nodeHash, node.Children[i], storage, layout) + } + return nil, nil + } + + var children [16]NodeHandle + for i := uint(0); i < 16; i++ { + children[i], err = child(i) + if err != nil { + return nil, err + } + } + + return NibbledBranch{ + encoded: node.PartialKey, + children: children, + value: node.Value, + }, nil + default: + panic("Invalid node") + } +} diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go index 5d630c2b32..9b530484df 100644 --- a/pkg/trie/triedb/triedbbuilder.go +++ b/pkg/trie/triedb/triedbbuilder.go @@ -23,6 +23,8 @@ func NewTrieDBBuilder[H hashdb.HashOut]( root H, layout TrieLayout[H], ) *TrieDBBuilder[H] { + root = layout.Codec().HashedNullNode() + return &TrieDBBuilder[H]{ db: db, root: root, @@ -52,7 +54,7 @@ func (self *TrieDBBuilder[H]) Build() *TrieDB[H] { recorder: self.recorder, storage: NewEmptyNodeStorage[H](), deathRow: make(map[string]nibble.Prefix), - rootHandle: &rootHandle, + rootHandle: rootHandle, layout: self.layout, } } From 60c86ddbd64ee840d89c2a9e2586d720d411e236 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 02:27:51 -0300 Subject: [PATCH 21/35] refactor(pkg/trie): Split node handle from normal node --- pkg/trie/triedb/node/node.go | 52 -------------------------- pkg/trie/triedb/node/node_handle.go | 57 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 52 deletions(-) create mode 100644 pkg/trie/triedb/node/node_handle.go diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index ff376b549d..905f8fa3de 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -4,8 +4,6 @@ package node import ( - "errors" - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" ) @@ -104,53 +102,3 @@ func (v NodeValueOwned[H]) Type() string { return "Node" } func (v NodeValueOwned[H]) AsValue() Value { return NodeValue{Bytes: v.hash.Bytes()} } - -// NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandleOwned[H hashdb.HashOut] interface { - Type() string - AsChildReference(codec NodeCodec[H]) ChildReference[H] -} -type ( - NodeHandleOwnedHash[H hashdb.HashOut] struct { - Hash H - } - NodeHandleOwnedInline[H hashdb.HashOut] struct { - Node NodeOwned[H] - } -) - -func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } -func (h NodeHandleOwnedHash[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - return ChildReferenceHash[H]{hash: h.Hash} -} -func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } -func (h NodeHandleOwnedInline[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - encoded := EncodeNodeOwned(h.Node, codec) - if len(encoded) > codec.Hasher().Length() { - panic("Invalid inline node handle") - } - return ChildReferenceInline[H]{hash: codec.Hasher().FromBytes(encoded), length: uint(len(encoded))} -} - -// NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandle interface { - Type() string -} -type ( - Hash struct { - Data []byte - } - Inline struct { - Data []byte - } -) - -func (h Hash) Type() string { return "Hash" } -func (h Inline) Type() string { return "Inline" } - -func DecodeHash[H hashdb.HashOut](hasher hashdb.Hasher[H], data []byte) (H, error) { - if len(data) != hasher.Length() { - return hasher.FromBytes([]byte{}), errors.New("decoding hash") - } - return hasher.FromBytes(data), nil -} diff --git a/pkg/trie/triedb/node/node_handle.go b/pkg/trie/triedb/node/node_handle.go new file mode 100644 index 0000000000..f66c7d072c --- /dev/null +++ b/pkg/trie/triedb/node/node_handle.go @@ -0,0 +1,57 @@ +package node + +import ( + "errors" + + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" +) + +// NodeHandle is a reference to a trie node which may be stored within another trie node. +type NodeHandleOwned[H hashdb.HashOut] interface { + Type() string + AsChildReference(codec NodeCodec[H]) ChildReference[H] +} +type ( + NodeHandleOwnedHash[H hashdb.HashOut] struct { + Hash H + } + NodeHandleOwnedInline[H hashdb.HashOut] struct { + Node NodeOwned[H] + } +) + +func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } +func (h NodeHandleOwnedHash[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { + return ChildReferenceHash[H]{hash: h.Hash} +} +func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } +func (h NodeHandleOwnedInline[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { + encoded := EncodeNodeOwned(h.Node, codec) + if len(encoded) > codec.Hasher().Length() { + panic("Invalid inline node handle") + } + return ChildReferenceInline[H]{hash: codec.Hasher().FromBytes(encoded), length: uint(len(encoded))} +} + +// NodeHandle is a reference to a trie node which may be stored within another trie node. +type NodeHandle interface { + Type() string +} +type ( + Hash struct { + Data []byte + } + Inline struct { + Data []byte + } +) + +func (h Hash) Type() string { return "Hash" } +func (h Inline) Type() string { return "Inline" } + +func DecodeHash[H hashdb.HashOut](hasher hashdb.Hasher[H], data []byte) (H, error) { + if len(data) != hasher.Length() { + return hasher.FromBytes([]byte{}), errors.New("decoding hash") + } + return hasher.FromBytes(data), nil +} From 222980ca8590c2c841d74e15acca38a2cc02e997 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 02:58:58 -0300 Subject: [PATCH 22/35] feat(pkg/trie): Introduce node plan --- pkg/trie/triedb/node/node_plan.go | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 pkg/trie/triedb/node/node_plan.go diff --git a/pkg/trie/triedb/node/node_plan.go b/pkg/trie/triedb/node/node_plan.go new file mode 100644 index 0000000000..f72ed35e89 --- /dev/null +++ b/pkg/trie/triedb/node/node_plan.go @@ -0,0 +1,127 @@ +package node + +import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" + +type bytesRange struct { + start int + end int +} + +// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice +type NibbleSlicePlan struct { + bytes bytesRange + offset int +} + +func NewNibbleSlicePlan(bytes bytesRange, offset int) NibbleSlicePlan { + return NibbleSlicePlan{bytes, offset} +} + +func (self NibbleSlicePlan) Len() int { + return (self.bytes.end-self.bytes.start)*nibble.NibblePerByte - self.offset +} + +func (self NibbleSlicePlan) Build(data []byte) nibble.NibbleSlice { + return *nibble.NewNibbleSliceWithOffset(data[self.bytes.start:self.bytes.end], self.offset) +} + +// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie +type NodeHandlePlan interface { + Type() string + Build(data []byte) NodeHandle +} + +type ( + NodeHandlePlanHash struct { + bytes bytesRange + } + NodeHandlePlanInline struct { + bytes bytesRange + } +) + +func (NodeHandlePlanHash) Type() string { return "Hash" } +func (n NodeHandlePlanHash) Build(data []byte) NodeHandle { + return Hash{data[n.bytes.start:n.bytes.end]} +} +func (NodeHandlePlanInline) Type() string { return "Inline" } +func (n NodeHandlePlanInline) Build(data []byte) NodeHandle { + return Inline{data[n.bytes.start:n.bytes.end]} +} + +// Plan for value representation in `NodePlan`. +type ValuePlan interface { + Type() string + Build(data []byte) Value +} + +type ( + // Range for byte representation in encoded node. + ValuePlanInline struct { + bytes bytesRange + } + // Range for hash in encoded node and original + ValuePlanNode struct { + bytes bytesRange + } +) + +func (ValuePlanInline) Type() string { return "Inline" } +func (n ValuePlanInline) Build(data []byte) Value { + return InlineValue{data[n.bytes.start:n.bytes.end]} +} +func (ValuePlanNode) Type() string { return "Node" } +func (n ValuePlanNode) Build(data []byte) Value { + return NodeValue{data[n.bytes.start:n.bytes.end]} +} + +type NodePlan interface { + Type() string + Build(data []byte) Node +} + +type ( + // Null trie node; could be an empty root or an empty branch entry + NodePlanEmptyNode struct{} + // Leaf node, has a partial key plan and value + NodePlanLeafNode struct { + partial NibbleSlicePlan + value ValuePlan + } + // Branch node + NodePlanNibbledBranchNode struct { + partial NibbleSlicePlan + value ValuePlan + children [nibble.NibbleLength]NodeHandlePlan + } +) + +func (NodePlanEmptyNode) Type() string { return "Empty" } +func (self NodePlanEmptyNode) Build(data []byte) Node { + return Empty{} +} +func (NodePlanLeafNode) Type() string { return "Leaf" } +func (self NodePlanLeafNode) Build(data []byte) Node { + return Leaf{ + PartialKey: self.partial.Build(data), + Value: self.value.Build(data), + } +} +func (NodePlanNibbledBranchNode) Type() string { return "NibbledBranch" } +func (self NodePlanNibbledBranchNode) Build(data []byte) Node { + children := [nibble.NibbleLength]NodeHandle{} + for i := 0; i < nibble.NibbleLength; i++ { + if self.children[i] != nil { + children[i] = self.children[i].Build(data) + } + } + var value Value + if self.value != nil { + value = self.value.Build(data) + } + return NibbledBranch{ + PartialKey: self.partial.Build(data), + Children: children, + Value: value, + } +} From bf6dde0db4786e1d852061826a1dc93a139852bc Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 02:59:23 -0300 Subject: [PATCH 23/35] fix(pkg/trie): Fix lookup --- pkg/trie/triedb/lookup.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 67ca99f49d..a11fccf640 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -107,11 +107,10 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er switch n := nextNode.(type) { case node.Hash: - nextHash := node.DecodeHash(n.Data, l.layout.Codec().Hasher()) - if nextHash == nil { + hash, err = node.DecodeHash(l.layout.Codec().Hasher(), n.Data) + if err != nil { return nil, InvalidHash } - hash = *nextHash case node.Inline: nodeData = &n.Data } From 8f226cb6f05cf4c700f70ddf920d17cf2dee52a0 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 03:00:08 -0300 Subject: [PATCH 24/35] chore(pkg/trie): Removes unnecesary generic from Node --- pkg/trie/triedb/node/node.go | 2 +- pkg/trie/triedb/node/node_codec.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go index 905f8fa3de..cd88546279 100644 --- a/pkg/trie/triedb/node/node.go +++ b/pkg/trie/triedb/node/node.go @@ -28,7 +28,7 @@ func (v InlineValue) Type() string { return "Inline" } func (v NodeValue) Type() string { return "Node" } // Nodes -type Node[H hashdb.HashOut] interface { +type Node interface { Type() string } diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go index 412a2fbdcf..f50a5001b6 100644 --- a/pkg/trie/triedb/node/node_codec.go +++ b/pkg/trie/triedb/node/node_codec.go @@ -38,7 +38,7 @@ type NodeCodec[H hashdb.HashOut] interface { children [16]ChildReference[H], value *Value, ) []byte - Decode(data []byte) (Node[H], error) + Decode(data []byte) (Node, error) } func EncodeNodeOwned[H hashdb.HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { From b5408b6dc7256b87dcb00189b9cb47230365c3e7 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 03:09:59 -0300 Subject: [PATCH 25/35] fix(pkg/trie): Fix some impl for substrate NodeCodec --- pkg/trie/test_support/reference_trie/substrate.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go index a7cc589239..515694aefe 100644 --- a/pkg/trie/test_support/reference_trie/substrate.go +++ b/pkg/trie/test_support/reference_trie/substrate.go @@ -45,10 +45,18 @@ func (self NodeCodec[H]) BranchNodeNibbled( panic("Implement me") } -func (self NodeCodec[H]) Decode(data []byte) (node.Node[H], error) { +func (self NodeCodec[H]) decodePlan(data []byte) (node.NodePlan, error) { panic("Implement me") } +func (self NodeCodec[H]) Decode(data []byte) (node.Node, error) { + plan, err := self.decodePlan(data) + if err != nil { + return nil, err + } + return plan.Build(data), nil +} + func NewNodeCodecForKeccak() NodeCodec[keccak_hasher.KeccakHash] { return NodeCodec[keccak_hasher.KeccakHash]{hasher} } From fec33a071f73e0ac5ae7015a5d02f20b3184b0d9 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 03:11:09 -0300 Subject: [PATCH 26/35] fix(pkg/trie): Add missing license --- pkg/trie/memorydb/memorydb_test.go | 3 +++ pkg/trie/triedb/node/node_handle.go | 3 +++ pkg/trie/triedb/node/node_plan.go | 3 +++ 3 files changed, 9 insertions(+) diff --git a/pkg/trie/memorydb/memorydb_test.go b/pkg/trie/memorydb/memorydb_test.go index 4aa03cfede..122c4a06cc 100644 --- a/pkg/trie/memorydb/memorydb_test.go +++ b/pkg/trie/memorydb/memorydb_test.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package memorydb import ( diff --git a/pkg/trie/triedb/node/node_handle.go b/pkg/trie/triedb/node/node_handle.go index f66c7d072c..e6e68a4eed 100644 --- a/pkg/trie/triedb/node/node_handle.go +++ b/pkg/trie/triedb/node/node_handle.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package node import ( diff --git a/pkg/trie/triedb/node/node_plan.go b/pkg/trie/triedb/node/node_plan.go index f72ed35e89..b695f70950 100644 --- a/pkg/trie/triedb/node/node_plan.go +++ b/pkg/trie/triedb/node/node_plan.go @@ -1,3 +1,6 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + package node import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" From 64616abe31119fdfa32846f8a422f6ef97b505b0 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 04:03:10 -0300 Subject: [PATCH 27/35] feat(pkg/trie): Decode header and docs --- internal/trie/node/branch_encode.go | 2 +- .../test_support/reference_trie/substrate.go | 160 +++++++++++++++++- pkg/trie/triedb/node/node_plan.go | 30 ++-- 3 files changed, 171 insertions(+), 21 deletions(-) diff --git a/internal/trie/node/branch_encode.go b/internal/trie/node/branch_encode.go index d9b8e7b5a6..54a67033cb 100644 --- a/internal/trie/node/branch_encode.go +++ b/internal/trie/node/branch_encode.go @@ -41,7 +41,7 @@ var parallelEncodingRateLimit = make(chan struct{}, parallelLimit) // encodeChildrenOpportunisticParallel encodes children in parallel eventually. // Leaves are encoded in a blocking way, and branches are encoded in separate -// goroutines IF they are less than the parallelLimit number of goroutines already +// goroutines IF they are less than the parallelLimit number of goroutines al.y // running. This is designed to limit the total number of goroutines in order to // avoid using too much memory on the stack. func encodeChildrenOpportunisticParallel(children []*Node, maxInlineValue int, buffer io.Writer) (err error) { diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go index 515694aefe..bb70f7ac54 100644 --- a/pkg/trie/test_support/reference_trie/substrate.go +++ b/pkg/trie/test_support/reference_trie/substrate.go @@ -4,6 +4,9 @@ package reference_trie import ( + "errors" + "io" + "github.com/ChainSafe/gossamer/pkg/trie/hashdb" "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" "github.com/ChainSafe/gossamer/pkg/trie/triedb" @@ -11,31 +14,175 @@ import ( "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" ) -const FirstPrefix = 0b_00 << 6 -const EmptyTree = FirstPrefix | (0b_00 << 4) +const firstPrefix = 0b_00 << 6 +const leafPrefixMask = 0b_01 << 6 +const branchWithoutValueMask = 0b_10 << 6 +const branchWithValueMask = 0b_11 << 6 +const emptyTrie = firstPrefix | (0b_00 << 4) +const leafWithHashedValuePrefixMask = firstPrefix | (0b_1 << 5) +const branchWithHashedValuePrefixMask = firstPrefix | (0b_1 << 4) +const escapeCompactHeader = emptyTrie | 0b_00_01 var hasher = keccak_hasher.NewKeccakHasher() +type byteSliceInput struct { + data []byte + offset int +} + +func NewByteSliceInput(data []byte) byteSliceInput { + return byteSliceInput{data, 0} +} + +func (self byteSliceInput) Take(count int) (node.BytesRange, error) { + if self.offset+count > len(self.data) { + return node.BytesRange{}, errors.New("out of data") + } + + res := node.BytesRange{self.offset, self.offset + count} + self.offset += count + return res, nil +} + +type NodeHeader interface { + Type() string +} + +type ( + NullNodeHeader struct{} + BranchNodeHeader struct { + hasValue bool + nibbleCount int + } + LeafNodeHeader struct { + nibbleCount int + } + HashedValueBranchNodeHeader struct { + nibbleCount int + } + HashedValueLeaf struct { + nibbleCount int + } +) + +func (NullNodeHeader) Type() string { return "Null" } +func (BranchNodeHeader) Type() string { return "Branch" } +func (LeafNodeHeader) Type() string { return "Leaf" } +func (HashedValueBranchNodeHeader) Type() string { return "HashedValueBranch" } +func (HashedValueLeaf) Type() string { return "HashedValueLeaf" } + +func headerContainsHashedValues(header NodeHeader) bool { + switch header.(type) { + case HashedValueBranchNodeHeader, HashedValueLeaf: + return true + default: + return false + } +} + +// Decode nibble count from stream input and header byte +func decodeSize(first byte, input io.Reader, prefixMask int) (int, error) { + maxValue := byte(255) >> prefixMask + result := (first & maxValue) + if result < maxValue { + return int(result), nil + } + result -= 1 + for { + b := make([]byte, 1) + _, err := input.Read(b) + if err != nil { + return -1, err + } + n := int(b[0]) + if n < 255 { + return int(result) + n + 1, nil + } + result += 255 + } +} + +// DecodeHeader decodes a node header from a stream input +// TODO: revisit this code, I think we can improve it +func DecodeHeader(input io.Reader) (NodeHeader, error) { + b := make([]byte, 1) + _, err := input.Read(b) + if err != nil { + return nil, err + } + i := b[0] + + if i == emptyTrie { + return NullNodeHeader{}, nil + } + + mask := i & (0b11 << 6) + switch mask { + case leafPrefixMask: + size, err := decodeSize(i, input, 2) + if err != nil { + return nil, err + } + return LeafNodeHeader{size}, nil + case branchWithValueMask: + size, err := decodeSize(i, input, 2) + if err != nil { + return nil, err + } + return BranchNodeHeader{true, size}, nil + case branchWithoutValueMask: + size, err := decodeSize(i, input, 2) + if err != nil { + return nil, err + } + return BranchNodeHeader{false, size}, nil + case emptyTrie: + if i&(0b111<<5) == leafWithHashedValuePrefixMask { + size, err := decodeSize(i, input, 3) + if err != nil { + return nil, err + } + return HashedValueLeaf{size}, nil + } else if i&(0b1111<<4) == branchWithHashedValuePrefixMask { + size, err := decodeSize(i, input, 4) + if err != nil { + return nil, err + } + return HashedValueBranchNodeHeader{size}, nil + } else { + return nil, errors.New("invalid header") + } + default: + panic("unreachable") + } +} + +// NodeCodec is the node codec configuration used in substrate type NodeCodec[H hashdb.HashOut] struct { hasher hashdb.Hasher[H] } +// HashedNullNode returns the hash of an empty node func (self NodeCodec[H]) HashedNullNode() H { return self.hasher.Hash(self.EmptyNode()) } +// Hasher returns the hasher used for this codec func (self NodeCodec[H]) Hasher() hashdb.Hasher[H] { return self.hasher } +// EmptyNode returns an empty node func (self NodeCodec[H]) EmptyNode() []byte { - return []byte{EmptyTree} + return []byte{emptyTrie} } +// LeafNode encodes a leaf node func (self NodeCodec[H]) LeafNode(partialKey nibble.NibbleSlice, numberNibble int, value node.Value) []byte { panic("Implement me") } +// BranchNodeNibbled encodes a branch node func (self NodeCodec[H]) BranchNodeNibbled( partialKey nibble.NibbleSlice, numberNibble int, @@ -46,9 +193,12 @@ func (self NodeCodec[H]) BranchNodeNibbled( } func (self NodeCodec[H]) decodePlan(data []byte) (node.NodePlan, error) { - panic("Implement me") + //input := NewByteSliceInput(data) + + return nil, nil } +// Decode decodes bytes to a Node func (self NodeCodec[H]) Decode(data []byte) (node.Node, error) { plan, err := self.decodePlan(data) if err != nil { @@ -61,7 +211,7 @@ func NewNodeCodecForKeccak() NodeCodec[keccak_hasher.KeccakHash] { return NodeCodec[keccak_hasher.KeccakHash]{hasher} } -var _ node.NodeCodec[keccak_hasher.KeccakHash] = NodeCodec[keccak_hasher.KeccakHash]{} +var SubstrateNodeCodec node.NodeCodec[keccak_hasher.KeccakHash] = NodeCodec[keccak_hasher.KeccakHash]{} type LayoutV0[H hashdb.HashOut] struct { codec node.NodeCodec[H] diff --git a/pkg/trie/triedb/node/node_plan.go b/pkg/trie/triedb/node/node_plan.go index b695f70950..25f87f9647 100644 --- a/pkg/trie/triedb/node/node_plan.go +++ b/pkg/trie/triedb/node/node_plan.go @@ -5,27 +5,27 @@ package node import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -type bytesRange struct { - start int - end int +type BytesRange struct { + Start int + End int } // A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice type NibbleSlicePlan struct { - bytes bytesRange + bytes BytesRange offset int } -func NewNibbleSlicePlan(bytes bytesRange, offset int) NibbleSlicePlan { +func NewNibbleSlicePlan(bytes BytesRange, offset int) NibbleSlicePlan { return NibbleSlicePlan{bytes, offset} } func (self NibbleSlicePlan) Len() int { - return (self.bytes.end-self.bytes.start)*nibble.NibblePerByte - self.offset + return (self.bytes.End-self.bytes.Start)*nibble.NibblePerByte - self.offset } func (self NibbleSlicePlan) Build(data []byte) nibble.NibbleSlice { - return *nibble.NewNibbleSliceWithOffset(data[self.bytes.start:self.bytes.end], self.offset) + return *nibble.NewNibbleSliceWithOffset(data[self.bytes.Start:self.bytes.End], self.offset) } // A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie @@ -36,20 +36,20 @@ type NodeHandlePlan interface { type ( NodeHandlePlanHash struct { - bytes bytesRange + bytes BytesRange } NodeHandlePlanInline struct { - bytes bytesRange + bytes BytesRange } ) func (NodeHandlePlanHash) Type() string { return "Hash" } func (n NodeHandlePlanHash) Build(data []byte) NodeHandle { - return Hash{data[n.bytes.start:n.bytes.end]} + return Hash{data[n.bytes.Start:n.bytes.End]} } func (NodeHandlePlanInline) Type() string { return "Inline" } func (n NodeHandlePlanInline) Build(data []byte) NodeHandle { - return Inline{data[n.bytes.start:n.bytes.end]} + return Inline{data[n.bytes.Start:n.bytes.End]} } // Plan for value representation in `NodePlan`. @@ -61,21 +61,21 @@ type ValuePlan interface { type ( // Range for byte representation in encoded node. ValuePlanInline struct { - bytes bytesRange + bytes BytesRange } // Range for hash in encoded node and original ValuePlanNode struct { - bytes bytesRange + bytes BytesRange } ) func (ValuePlanInline) Type() string { return "Inline" } func (n ValuePlanInline) Build(data []byte) Value { - return InlineValue{data[n.bytes.start:n.bytes.end]} + return InlineValue{data[n.bytes.Start:n.bytes.End]} } func (ValuePlanNode) Type() string { return "Node" } func (n ValuePlanNode) Build(data []byte) Value { - return NodeValue{data[n.bytes.start:n.bytes.end]} + return NodeValue{data[n.bytes.Start:n.bytes.End]} } type NodePlan interface { From 0306d8df1e13096013cd29f1de403ef855c2b35e Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 18 Jan 2024 04:25:14 -0300 Subject: [PATCH 28/35] refactor(pkg/trie): Better node header decoding --- .../test_support/reference_trie/substrate.go | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go index bb70f7ac54..617714420c 100644 --- a/pkg/trie/test_support/reference_trie/substrate.go +++ b/pkg/trie/test_support/reference_trie/substrate.go @@ -103,7 +103,6 @@ func decodeSize(first byte, input io.Reader, prefixMask int) (int, error) { } // DecodeHeader decodes a node header from a stream input -// TODO: revisit this code, I think we can improve it func DecodeHeader(input io.Reader) (NodeHeader, error) { b := make([]byte, 1) _, err := input.Read(b) @@ -117,44 +116,40 @@ func DecodeHeader(input io.Reader) (NodeHeader, error) { } mask := i & (0b11 << 6) + + var ( + size int + node NodeHeader + ) + switch mask { case leafPrefixMask: - size, err := decodeSize(i, input, 2) - if err != nil { - return nil, err - } - return LeafNodeHeader{size}, nil + size, err = decodeSize(i, input, 2) + node = LeafNodeHeader{size} case branchWithValueMask: - size, err := decodeSize(i, input, 2) - if err != nil { - return nil, err - } - return BranchNodeHeader{true, size}, nil + size, err = decodeSize(i, input, 2) + node = BranchNodeHeader{true, size} case branchWithoutValueMask: - size, err := decodeSize(i, input, 2) - if err != nil { - return nil, err - } - return BranchNodeHeader{false, size}, nil + size, err = decodeSize(i, input, 2) + node = BranchNodeHeader{false, size} case emptyTrie: if i&(0b111<<5) == leafWithHashedValuePrefixMask { - size, err := decodeSize(i, input, 3) - if err != nil { - return nil, err - } - return HashedValueLeaf{size}, nil + size, err = decodeSize(i, input, 3) + node = HashedValueLeaf{size} } else if i&(0b1111<<4) == branchWithHashedValuePrefixMask { - size, err := decodeSize(i, input, 4) - if err != nil { - return nil, err - } - return HashedValueBranchNodeHeader{size}, nil + size, err = decodeSize(i, input, 4) + node = HashedValueBranchNodeHeader{size} } else { - return nil, errors.New("invalid header") + err = errors.New("invalid header") } default: panic("unreachable") } + + if err != nil { + return nil, err + } + return node, err } // NodeCodec is the node codec configuration used in substrate From a460cd1e697b811b3d214028243a98dc1a9e7489 Mon Sep 17 00:00:00 2001 From: Diego Date: Thu, 1 Feb 2024 01:31:13 -0300 Subject: [PATCH 29/35] fix(pkg/trie): fix lookup break for --- pkg/trie/triedb/lookup.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index a11fccf640..ccffea3219 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -63,6 +63,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er }) // Iterates children + childrens: for { // Decode node decodedNode, err := l.layout.Codec().Decode(*nodeData) @@ -111,6 +112,7 @@ func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, er if err != nil { return nil, InvalidHash } + break childrens case node.Inline: nodeData = &n.Data } From 4ac436d77db472ba7884bfe58233fefb936f961a Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:21:28 -0300 Subject: [PATCH 30/35] chore(pkg/trie): Add standalone trie pkg --- cmd/gossamer/commands/import_state.go | 2 +- dot/core/helpers_test.go | 2 +- dot/core/service.go | 2 +- dot/core/service_integration_test.go | 2 +- dot/core/service_test.go | 2 +- dot/digest/helpers_test.go | 2 +- dot/helpers_test.go | 2 +- dot/import.go | 2 +- dot/import_integration_test.go | 2 +- dot/node_integration_test.go | 2 +- dot/rpc/helpers_test.go | 2 +- dot/rpc/interfaces.go | 2 +- dot/rpc/modules/api.go | 2 +- dot/rpc/modules/author_integration_test.go | 2 +- dot/rpc/modules/chain_integration_test.go | 2 +- .../modules/childstate_integration_test.go | 2 +- dot/rpc/modules/childstate_test.go | 2 +- dot/rpc/modules/dev_integration_test.go | 2 +- dot/rpc/modules/helpers_test.go | 2 +- dot/rpc/modules/mocks/mocks.go | 2 +- dot/rpc/modules/mocks_test.go | 2 +- dot/rpc/modules/state.go | 2 +- dot/rpc/modules/state_integration_test.go | 2 +- dot/rpc/modules/state_test.go | 2 +- dot/rpc/modules/system_integration_test.go | 2 +- dot/state/base_test.go | 2 +- dot/state/block_data_test.go | 2 +- dot/state/block_finalisation_test.go | 2 +- dot/state/block_race_test.go | 2 +- dot/state/block_test.go | 2 +- dot/state/grandpa_test.go | 2 +- dot/state/helpers_test.go | 2 +- dot/state/initialize.go | 2 +- dot/state/mocks_database_test.go | 4 +- dot/state/mocks_generate_test.go | 2 +- dot/state/offline_pruner.go | 2 +- dot/state/service.go | 2 +- dot/state/service_integration_test.go | 2 +- dot/state/storage.go | 4 +- dot/state/storage_test.go | 4 +- dot/state/test_helpers.go | 2 +- dot/state/tries.go | 2 +- dot/state/tries_test.go | 4 +- dot/sync/chain_sync.go | 2 +- dot/sync/chain_sync_test.go | 2 +- dot/sync/message_integration_test.go | 2 +- dot/sync/syncer_integration_test.go | 2 +- dot/utils_integration_test.go | 2 +- lib/babe/helpers_test.go | 2 +- lib/grandpa/grandpa_integration_test.go | 2 +- lib/grandpa/helpers_integration_test.go | 2 +- lib/grandpa/helpers_unit_test.go | 2 +- lib/runtime/genesis.go | 2 +- lib/runtime/interfaces.go | 2 +- lib/runtime/storage/trie.go | 2 +- lib/runtime/storage/trie_test.go | 2 +- lib/runtime/wazero/imports.go | 4 +- lib/runtime/wazero/imports_test.go | 4 +- lib/runtime/wazero/instance.go | 2 +- lib/runtime/wazero/instance_test.go | 2 +- lib/runtime/wazero/test_helpers.go | 2 +- {lib => pkg}/trie/child_storage.go | 0 {lib => pkg}/trie/child_storage_test.go | 0 {internal => pkg}/trie/codec/nibbles.go | 0 {internal => pkg}/trie/codec/nibbles_test.go | 0 {lib => pkg}/trie/database.go | 6 +- {lib => pkg}/trie/database_test.go | 0 {lib => pkg}/trie/db/db.go | 0 {lib => pkg}/trie/db_getter_mocks_test.go | 4 +- {lib => pkg}/trie/genesis.go | 0 {lib => pkg}/trie/genesis_test.go | 2 +- pkg/trie/hashdb/hashdb.go | 25 - {lib => pkg}/trie/helpers_test.go | 4 +- {lib => pkg}/trie/interfaces.go | 2 +- {lib => pkg}/trie/layout.go | 0 {lib => pkg}/trie/layout_test.go | 0 {lib => pkg}/trie/mem_test.go | 0 pkg/trie/memorydb/keyfunction.go | 28 - pkg/trie/memorydb/memorydb.go | 224 ----- pkg/trie/memorydb/memorydb_test.go | 124 --- {lib => pkg}/trie/mocks_generate_test.go | 2 +- {lib => pkg}/trie/node.go | 2 +- {internal => pkg}/trie/node/README.md | 0 {internal => pkg}/trie/node/branch_encode.go | 0 .../trie/node/branch_encode_test.go | 0 {internal => pkg}/trie/node/buffer.go | 0 .../trie/node/buffer_mock_test.go | 2 +- {internal => pkg}/trie/node/children.go | 0 {internal => pkg}/trie/node/children_test.go | 0 {internal => pkg}/trie/node/copy.go | 0 {internal => pkg}/trie/node/copy_test.go | 0 {internal => pkg}/trie/node/decode.go | 0 {internal => pkg}/trie/node/decode_test.go | 0 {internal => pkg}/trie/node/dirty.go | 0 {internal => pkg}/trie/node/dirty_test.go | 0 {internal => pkg}/trie/node/encode.go | 2 +- .../trie/node/encode_decode_test.go | 0 {internal => pkg}/trie/node/encode_test.go | 0 {internal => pkg}/trie/node/hash.go | 2 +- {internal => pkg}/trie/node/hash_test.go | 0 {internal => pkg}/trie/node/header.go | 0 {internal => pkg}/trie/node/header_test.go | 0 {internal => pkg}/trie/node/helpers_test.go | 0 {internal => pkg}/trie/node/key.go | 2 +- {internal => pkg}/trie/node/key_test.go | 0 .../trie/node/mocks_generate_test.go | 0 {internal => pkg}/trie/node/node.go | 0 {internal => pkg}/trie/node/node_test.go | 0 .../trie/node/reader_mock_test.go | 0 {internal => pkg}/trie/node/subvalue.go | 0 {internal => pkg}/trie/node/subvalue_test.go | 0 {internal => pkg}/trie/node/types.go | 0 {internal => pkg}/trie/node/variants.go | 0 .../trie/node/writer_mock_test.go | 0 {internal => pkg}/trie/pools/pools.go | 0 {lib => pkg}/trie/print.go | 0 {lib => pkg}/trie/print_test.go | 0 .../trie/proof/database_mocks_test.go | 4 +- {lib => pkg}/trie/proof/generate.go | 10 +- {lib => pkg}/trie/proof/generate_test.go | 8 +- {lib => pkg}/trie/proof/helpers_test.go | 4 +- .../trie/proof/mocks_generate_test.go | 2 +- {lib => pkg}/trie/proof/proof_test.go | 4 +- {lib => pkg}/trie/proof/verify.go | 8 +- {lib => pkg}/trie/proof/verify_test.go | 6 +- .../keccak_hasher/keccak_hasher.go | 55 -- .../test_support/reference_trie/substrate.go | 227 ----- pkg/trie/tests/recorder_test.go | 47 -- {internal => pkg}/trie/tracking/deltas.go | 0 .../trie/tracking/deltas_test.go | 0 .../trie/tracking/helpers_test.go | 0 {internal => pkg}/trie/tracking/interfaces.go | 0 {lib => pkg}/trie/trie.go | 8 +- {lib => pkg}/trie/trie_endtoend_test.go | 2 +- {lib => pkg}/trie/trie_test.go | 6 +- pkg/trie/triedb/cache.go | 38 - pkg/trie/triedb/errors.go | 11 - pkg/trie/triedb/layout.go | 15 - pkg/trie/triedb/lookup.go | 138 --- pkg/trie/triedb/nibble/nibble.go | 123 --- pkg/trie/triedb/nibble/nibbleslice.go | 200 ----- pkg/trie/triedb/nibble/nibbleslice_test.go | 121 --- pkg/trie/triedb/node/node.go | 104 --- pkg/trie/triedb/node/node_codec.go | 63 -- pkg/trie/triedb/node/node_handle.go | 60 -- pkg/trie/triedb/node/node_plan.go | 130 --- pkg/trie/triedb/recorder.go | 144 ---- pkg/trie/triedb/trie.go | 53 -- pkg/trie/triedb/triedb.go | 794 ------------------ pkg/trie/triedb/triedbbuilder.go | 60 -- tests/rpc/rpc_05-state_test.go | 2 +- 151 files changed, 114 insertions(+), 2898 deletions(-) rename {lib => pkg}/trie/child_storage.go (100%) rename {lib => pkg}/trie/child_storage_test.go (100%) rename {internal => pkg}/trie/codec/nibbles.go (100%) rename {internal => pkg}/trie/codec/nibbles_test.go (100%) rename {lib => pkg}/trie/database.go (98%) rename {lib => pkg}/trie/database_test.go (100%) rename {lib => pkg}/trie/db/db.go (100%) rename {lib => pkg}/trie/db_getter_mocks_test.go (92%) rename {lib => pkg}/trie/genesis.go (100%) rename {lib => pkg}/trie/genesis_test.go (96%) delete mode 100644 pkg/trie/hashdb/hashdb.go rename {lib => pkg}/trie/helpers_test.go (96%) rename {lib => pkg}/trie/interfaces.go (92%) rename {lib => pkg}/trie/layout.go (100%) rename {lib => pkg}/trie/layout_test.go (100%) rename {lib => pkg}/trie/mem_test.go (100%) delete mode 100644 pkg/trie/memorydb/keyfunction.go delete mode 100644 pkg/trie/memorydb/memorydb.go delete mode 100644 pkg/trie/memorydb/memorydb_test.go rename {lib => pkg}/trie/mocks_generate_test.go (71%) rename {lib => pkg}/trie/node.go (78%) rename {internal => pkg}/trie/node/README.md (100%) rename {internal => pkg}/trie/node/branch_encode.go (100%) rename {internal => pkg}/trie/node/branch_encode_test.go (100%) rename {internal => pkg}/trie/node/buffer.go (100%) rename {internal => pkg}/trie/node/buffer_mock_test.go (96%) rename {internal => pkg}/trie/node/children.go (100%) rename {internal => pkg}/trie/node/children_test.go (100%) rename {internal => pkg}/trie/node/copy.go (100%) rename {internal => pkg}/trie/node/copy_test.go (100%) rename {internal => pkg}/trie/node/decode.go (100%) rename {internal => pkg}/trie/node/decode_test.go (100%) rename {internal => pkg}/trie/node/dirty.go (100%) rename {internal => pkg}/trie/node/dirty_test.go (100%) rename {internal => pkg}/trie/node/encode.go (97%) rename {internal => pkg}/trie/node/encode_decode_test.go (100%) rename {internal => pkg}/trie/node/encode_test.go (100%) rename {internal => pkg}/trie/node/hash.go (98%) rename {internal => pkg}/trie/node/hash_test.go (100%) rename {internal => pkg}/trie/node/header.go (100%) rename {internal => pkg}/trie/node/header_test.go (100%) rename {internal => pkg}/trie/node/helpers_test.go (100%) rename {internal => pkg}/trie/node/key.go (93%) rename {internal => pkg}/trie/node/key_test.go (100%) rename {internal => pkg}/trie/node/mocks_generate_test.go (100%) rename {internal => pkg}/trie/node/node.go (100%) rename {internal => pkg}/trie/node/node_test.go (100%) rename {internal => pkg}/trie/node/reader_mock_test.go (100%) rename {internal => pkg}/trie/node/subvalue.go (100%) rename {internal => pkg}/trie/node/subvalue_test.go (100%) rename {internal => pkg}/trie/node/types.go (100%) rename {internal => pkg}/trie/node/variants.go (100%) rename {internal => pkg}/trie/node/writer_mock_test.go (100%) rename {internal => pkg}/trie/pools/pools.go (100%) rename {lib => pkg}/trie/print.go (100%) rename {lib => pkg}/trie/print_test.go (100%) rename {lib => pkg}/trie/proof/database_mocks_test.go (92%) rename {lib => pkg}/trie/proof/generate.go (95%) rename {lib => pkg}/trie/proof/generate_test.go (99%) rename {lib => pkg}/trie/proof/helpers_test.go (96%) rename {lib => pkg}/trie/proof/mocks_generate_test.go (71%) rename {lib => pkg}/trie/proof/proof_test.go (99%) rename {lib => pkg}/trie/proof/verify.go (97%) rename {lib => pkg}/trie/proof/verify_test.go (99%) delete mode 100644 pkg/trie/test_support/keccak_hasher/keccak_hasher.go delete mode 100644 pkg/trie/test_support/reference_trie/substrate.go delete mode 100644 pkg/trie/tests/recorder_test.go rename {internal => pkg}/trie/tracking/deltas.go (100%) rename {internal => pkg}/trie/tracking/deltas_test.go (100%) rename {internal => pkg}/trie/tracking/helpers_test.go (100%) rename {internal => pkg}/trie/tracking/interfaces.go (100%) rename {lib => pkg}/trie/trie.go (99%) rename {lib => pkg}/trie/trie_endtoend_test.go (99%) rename {lib => pkg}/trie/trie_test.go (99%) delete mode 100644 pkg/trie/triedb/cache.go delete mode 100644 pkg/trie/triedb/errors.go delete mode 100644 pkg/trie/triedb/layout.go delete mode 100644 pkg/trie/triedb/lookup.go delete mode 100644 pkg/trie/triedb/nibble/nibble.go delete mode 100644 pkg/trie/triedb/nibble/nibbleslice.go delete mode 100644 pkg/trie/triedb/nibble/nibbleslice_test.go delete mode 100644 pkg/trie/triedb/node/node.go delete mode 100644 pkg/trie/triedb/node/node_codec.go delete mode 100644 pkg/trie/triedb/node/node_handle.go delete mode 100644 pkg/trie/triedb/node/node_plan.go delete mode 100644 pkg/trie/triedb/recorder.go delete mode 100644 pkg/trie/triedb/trie.go delete mode 100644 pkg/trie/triedb/triedb.go delete mode 100644 pkg/trie/triedb/triedbbuilder.go diff --git a/cmd/gossamer/commands/import_state.go b/cmd/gossamer/commands/import_state.go index 43d24a4588..d262a72488 100644 --- a/cmd/gossamer/commands/import_state.go +++ b/cmd/gossamer/commands/import_state.go @@ -7,8 +7,8 @@ import ( "fmt" "github.com/ChainSafe/gossamer/dot" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/spf13/cobra" ) diff --git a/dot/core/helpers_test.go b/dot/core/helpers_test.go index a011937672..3bac5b8272 100644 --- a/dot/core/helpers_test.go +++ b/dot/core/helpers_test.go @@ -20,9 +20,9 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/dot/core/service.go b/dot/core/service.go index a7dc445a3e..e630e8f26d 100644 --- a/dot/core/service.go +++ b/dot/core/service.go @@ -20,7 +20,7 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" cscale "github.com/centrifuge/go-substrate-rpc-client/v4/scale" ctypes "github.com/centrifuge/go-substrate-rpc-client/v4/types" diff --git a/dot/core/service_integration_test.go b/dot/core/service_integration_test.go index 4cca83e2d9..2078d2713e 100644 --- a/dot/core/service_integration_test.go +++ b/dot/core/service_integration_test.go @@ -24,9 +24,9 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) diff --git a/dot/core/service_test.go b/dot/core/service_test.go index c7fc9baa52..52a2df46af 100644 --- a/dot/core/service_test.go +++ b/dot/core/service_test.go @@ -23,8 +23,8 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" cscale "github.com/centrifuge/go-substrate-rpc-client/v4/scale" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" ctypes "github.com/centrifuge/go-substrate-rpc-client/v4/types" diff --git a/dot/digest/helpers_test.go b/dot/digest/helpers_test.go index 8637ccbaf2..05e82cd862 100644 --- a/dot/digest/helpers_test.go +++ b/dot/digest/helpers_test.go @@ -10,8 +10,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/helpers_test.go b/dot/helpers_test.go index a4539a9299..fa3e55809a 100644 --- a/dot/helpers_test.go +++ b/dot/helpers_test.go @@ -13,8 +13,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" runtime "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/import.go b/dot/import.go index 208d648765..5cdfb8ba04 100644 --- a/dot/import.go +++ b/dot/import.go @@ -13,8 +13,8 @@ import ( "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/internal/log" ) diff --git a/dot/import_integration_test.go b/dot/import_integration_test.go index 436e4c2204..54c0c7b5e5 100644 --- a/dot/import_integration_test.go +++ b/dot/import_integration_test.go @@ -13,8 +13,8 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/dot/node_integration_test.go b/dot/node_integration_test.go index ce0a2d999c..73f1339599 100644 --- a/dot/node_integration_test.go +++ b/dot/node_integration_test.go @@ -35,7 +35,7 @@ import ( "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gomock "go.uber.org/mock/gomock" diff --git a/dot/rpc/helpers_test.go b/dot/rpc/helpers_test.go index af7835518a..af8d15f5d7 100644 --- a/dot/rpc/helpers_test.go +++ b/dot/rpc/helpers_test.go @@ -10,8 +10,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/rpc/interfaces.go b/dot/rpc/interfaces.go index 0a868a0950..31bf5633c9 100644 --- a/dot/rpc/interfaces.go +++ b/dot/rpc/interfaces.go @@ -15,7 +15,7 @@ import ( "github.com/ChainSafe/gossamer/lib/grandpa" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) // StorageAPI is the interface for the storage state diff --git a/dot/rpc/modules/api.go b/dot/rpc/modules/api.go index dc7754a66f..e039df052b 100644 --- a/dot/rpc/modules/api.go +++ b/dot/rpc/modules/api.go @@ -13,7 +13,7 @@ import ( "github.com/ChainSafe/gossamer/lib/grandpa" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) // StorageAPI is the interface for the storage state diff --git a/dot/rpc/modules/author_integration_test.go b/dot/rpc/modules/author_integration_test.go index 33bd4e1c42..fbeef37cf9 100644 --- a/dot/rpc/modules/author_integration_test.go +++ b/dot/rpc/modules/author_integration_test.go @@ -31,8 +31,8 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" cscale "github.com/centrifuge/go-substrate-rpc-client/v4/scale" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" ctypes "github.com/centrifuge/go-substrate-rpc-client/v4/types" diff --git a/dot/rpc/modules/chain_integration_test.go b/dot/rpc/modules/chain_integration_test.go index de51c20c52..fb2d96b78b 100644 --- a/dot/rpc/modules/chain_integration_test.go +++ b/dot/rpc/modules/chain_integration_test.go @@ -16,8 +16,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "go.uber.org/mock/gomock" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" diff --git a/dot/rpc/modules/childstate_integration_test.go b/dot/rpc/modules/childstate_integration_test.go index 74c7ae53e0..7bacf23e96 100644 --- a/dot/rpc/modules/childstate_integration_test.go +++ b/dot/rpc/modules/childstate_integration_test.go @@ -13,7 +13,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/rpc/modules/childstate_test.go b/dot/rpc/modules/childstate_test.go index e56b776b3b..5d82c9b70b 100644 --- a/dot/rpc/modules/childstate_test.go +++ b/dot/rpc/modules/childstate_test.go @@ -11,7 +11,7 @@ import ( apimocks "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/lib/common" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "go.uber.org/mock/gomock" "github.com/stretchr/testify/assert" diff --git a/dot/rpc/modules/dev_integration_test.go b/dot/rpc/modules/dev_integration_test.go index 8ea64c66ee..293643a0c7 100644 --- a/dot/rpc/modules/dev_integration_test.go +++ b/dot/rpc/modules/dev_integration_test.go @@ -17,7 +17,7 @@ import ( "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) diff --git a/dot/rpc/modules/helpers_test.go b/dot/rpc/modules/helpers_test.go index bce60ba019..1b4e3e40a8 100644 --- a/dot/rpc/modules/helpers_test.go +++ b/dot/rpc/modules/helpers_test.go @@ -9,8 +9,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/rpc/modules/mocks/mocks.go b/dot/rpc/modules/mocks/mocks.go index b668da62f9..1b9c1629b2 100644 --- a/dot/rpc/modules/mocks/mocks.go +++ b/dot/rpc/modules/mocks/mocks.go @@ -20,7 +20,7 @@ import ( genesis "github.com/ChainSafe/gossamer/lib/genesis" runtime "github.com/ChainSafe/gossamer/lib/runtime" transaction "github.com/ChainSafe/gossamer/lib/transaction" - trie "github.com/ChainSafe/gossamer/lib/trie" + trie "github.com/ChainSafe/gossamer/pkg/trie" gomock "go.uber.org/mock/gomock" ) diff --git a/dot/rpc/modules/mocks_test.go b/dot/rpc/modules/mocks_test.go index 8e0ecb8c31..17c27be12d 100644 --- a/dot/rpc/modules/mocks_test.go +++ b/dot/rpc/modules/mocks_test.go @@ -17,7 +17,7 @@ import ( types "github.com/ChainSafe/gossamer/dot/types" common "github.com/ChainSafe/gossamer/lib/common" runtime "github.com/ChainSafe/gossamer/lib/runtime" - trie "github.com/ChainSafe/gossamer/lib/trie" + trie "github.com/ChainSafe/gossamer/pkg/trie" gomock "go.uber.org/mock/gomock" ) diff --git a/dot/rpc/modules/state.go b/dot/rpc/modules/state.go index 1cae7e3612..7fac92e561 100644 --- a/dot/rpc/modules/state.go +++ b/dot/rpc/modules/state.go @@ -11,8 +11,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" ) // StateGetReadProofRequest json fields diff --git a/dot/rpc/modules/state_integration_test.go b/dot/rpc/modules/state_integration_test.go index a37cb667e1..8a600c74b0 100644 --- a/dot/rpc/modules/state_integration_test.go +++ b/dot/rpc/modules/state_integration_test.go @@ -17,7 +17,7 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/dot/rpc/modules/state_test.go b/dot/rpc/modules/state_test.go index 491922804d..e86bf7abf3 100644 --- a/dot/rpc/modules/state_test.go +++ b/dot/rpc/modules/state_test.go @@ -22,7 +22,7 @@ import ( "testing" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/dot/rpc/modules/mocks" testdata "github.com/ChainSafe/gossamer/dot/rpc/modules/test_data" diff --git a/dot/rpc/modules/system_integration_test.go b/dot/rpc/modules/system_integration_test.go index 195624be4c..8ad072c74b 100644 --- a/dot/rpc/modules/system_integration_test.go +++ b/dot/rpc/modules/system_integration_test.go @@ -29,8 +29,8 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" ) var ( diff --git a/dot/state/base_test.go b/dot/state/base_test.go index 637ee44bb9..c08e478f04 100644 --- a/dot/state/base_test.go +++ b/dot/state/base_test.go @@ -8,7 +8,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/state/block_data_test.go b/dot/state/block_data_test.go index 370282850b..70b40a41f5 100644 --- a/dot/state/block_data_test.go +++ b/dot/state/block_data_test.go @@ -8,7 +8,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/state/block_finalisation_test.go b/dot/state/block_finalisation_test.go index 0834529bf0..578ce884da 100644 --- a/dot/state/block_finalisation_test.go +++ b/dot/state/block_finalisation_test.go @@ -8,7 +8,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/state/block_race_test.go b/dot/state/block_race_test.go index d840d1d426..e74dc36148 100644 --- a/dot/state/block_race_test.go +++ b/dot/state/block_race_test.go @@ -10,7 +10,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" diff --git a/dot/state/block_test.go b/dot/state/block_test.go index ced6cd4263..426f8f6a4e 100644 --- a/dot/state/block_test.go +++ b/dot/state/block_test.go @@ -12,8 +12,8 @@ import ( "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/blocktree" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" diff --git a/dot/state/grandpa_test.go b/dot/state/grandpa_test.go index 6e51cd4cc6..8018308c0b 100644 --- a/dot/state/grandpa_test.go +++ b/dot/state/grandpa_test.go @@ -15,7 +15,7 @@ import ( "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/gtank/merlin" "go.uber.org/mock/gomock" diff --git a/dot/state/helpers_test.go b/dot/state/helpers_test.go index 10029e9f01..283b5f2c0a 100644 --- a/dot/state/helpers_test.go +++ b/dot/state/helpers_test.go @@ -12,8 +12,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/state/initialize.go b/dot/state/initialize.go index 6be468907e..88c26e7744 100644 --- a/dot/state/initialize.go +++ b/dot/state/initialize.go @@ -14,7 +14,7 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) // Initialise initialises the genesis state of the DB using the given storage trie. diff --git a/dot/state/mocks_database_test.go b/dot/state/mocks_database_test.go index cd53b18af9..91978d00fd 100644 --- a/dot/state/mocks_database_test.go +++ b/dot/state/mocks_database_test.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: Database) +// Source: github.com/ChainSafe/gossamer/pkg/trie/db (interfaces: Database) // // Generated by this command: // -// mockgen -destination=mocks_database_test.go -package=state github.com/ChainSafe/gossamer/lib/trie/db Database +// mockgen -destination=mocks_database_test.go -package=state github.com/ChainSafe/gossamer/pkg/trie/db Database // // Package state is a generated GoMock package. diff --git a/dot/state/mocks_generate_test.go b/dot/state/mocks_generate_test.go index 10d9c46059..5d474062b6 100644 --- a/dot/state/mocks_generate_test.go +++ b/dot/state/mocks_generate_test.go @@ -7,4 +7,4 @@ package state //go:generate mockgen -destination=mocks_runtime_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/lib/runtime Instance //go:generate mockgen -destination=mock_gauge_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Gauge //go:generate mockgen -destination=mock_counter_test.go -package $GOPACKAGE github.com/prometheus/client_golang/prometheus Counter -//go:generate mockgen -destination=mocks_database_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db Database +//go:generate mockgen -destination=mocks_database_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/pkg/trie/db Database diff --git a/dot/state/offline_pruner.go b/dot/state/offline_pruner.go index 0d0e24568b..b39d138def 100644 --- a/dot/state/offline_pruner.go +++ b/dot/state/offline_pruner.go @@ -11,7 +11,7 @@ import ( "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) // OfflinePruner is a tool to prune the stale state with the help of diff --git a/dot/state/service.go b/dot/state/service.go index 7066664ee4..51691e6900 100644 --- a/dot/state/service.go +++ b/dot/state/service.go @@ -13,7 +13,7 @@ import ( "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/internal/metrics" "github.com/ChainSafe/gossamer/lib/blocktree" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) var logger = log.NewFromGlobal( diff --git a/dot/state/service_integration_test.go b/dot/state/service_integration_test.go index d733b90934..283d6c70d4 100644 --- a/dot/state/service_integration_test.go +++ b/dot/state/service_integration_test.go @@ -17,7 +17,7 @@ import ( "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" runtime "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" diff --git a/dot/state/storage.go b/dot/state/storage.go index ed8f8377a0..5410294733 100644 --- a/dot/state/storage.go +++ b/dot/state/storage.go @@ -13,8 +13,8 @@ import ( "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/proof" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/proof" ) // storagePrefix storage key prefix. diff --git a/dot/state/storage_test.go b/dot/state/storage_test.go index 5c66947c07..cad3431dfa 100644 --- a/dot/state/storage_test.go +++ b/dot/state/storage_test.go @@ -10,10 +10,10 @@ import ( "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" runtime "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/node" "go.uber.org/mock/gomock" "github.com/stretchr/testify/require" diff --git a/dot/state/test_helpers.go b/dot/state/test_helpers.go index 3109fede3d..c4fa421170 100644 --- a/dot/state/test_helpers.go +++ b/dot/state/test_helpers.go @@ -12,8 +12,8 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/state/tries.go b/dot/state/tries.go index d7c87bc625..ee0db88740 100644 --- a/dot/state/tries.go +++ b/dot/state/tries.go @@ -7,7 +7,7 @@ import ( "sync" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) diff --git a/dot/state/tries_test.go b/dot/state/tries_test.go index cff32bfac1..c434eaec9e 100644 --- a/dot/state/tries_test.go +++ b/dot/state/tries_test.go @@ -6,9 +6,9 @@ package state import ( "testing" - "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/node" "github.com/stretchr/testify/assert" "go.uber.org/mock/gomock" ) diff --git a/dot/sync/chain_sync.go b/dot/sync/chain_sync.go index eea6301a89..e700174f4e 100644 --- a/dot/sync/chain_sync.go +++ b/dot/sync/chain_sync.go @@ -25,7 +25,7 @@ import ( "github.com/ChainSafe/gossamer/internal/database" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/common/variadic" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) var _ ChainSync = (*chainSync)(nil) diff --git a/dot/sync/chain_sync_test.go b/dot/sync/chain_sync_test.go index 1964e0dc67..c64a42de57 100644 --- a/dot/sync/chain_sync_test.go +++ b/dot/sync/chain_sync_test.go @@ -17,8 +17,8 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/common/variadic" "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/dot/sync/message_integration_test.go b/dot/sync/message_integration_test.go index 7b4ff69529..300a1f580f 100644 --- a/dot/sync/message_integration_test.go +++ b/dot/sync/message_integration_test.go @@ -12,7 +12,7 @@ import ( "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common/variadic" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/dot/sync/syncer_integration_test.go b/dot/sync/syncer_integration_test.go index ac4ddf34e4..35dc29eba1 100644 --- a/dot/sync/syncer_integration_test.go +++ b/dot/sync/syncer_integration_test.go @@ -19,8 +19,8 @@ import ( runtime "github.com/ChainSafe/gossamer/lib/runtime" rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/dot/utils_integration_test.go b/dot/utils_integration_test.go index 27e0d7c738..b50ddd3b06 100644 --- a/dot/utils_integration_test.go +++ b/dot/utils_integration_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/lib/genesis" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/lib/babe/helpers_test.go b/lib/babe/helpers_test.go index f21eec6984..161110e77e 100644 --- a/lib/babe/helpers_test.go +++ b/lib/babe/helpers_test.go @@ -23,9 +23,9 @@ import ( rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) diff --git a/lib/grandpa/grandpa_integration_test.go b/lib/grandpa/grandpa_integration_test.go index 1771e585af..01950743d3 100644 --- a/lib/grandpa/grandpa_integration_test.go +++ b/lib/grandpa/grandpa_integration_test.go @@ -16,7 +16,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/lib/grandpa/helpers_integration_test.go b/lib/grandpa/helpers_integration_test.go index 0c80b8dce9..fb5b343825 100644 --- a/lib/grandpa/helpers_integration_test.go +++ b/lib/grandpa/helpers_integration_test.go @@ -20,8 +20,8 @@ import ( runtime "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/storage" wazero_runtime "github.com/ChainSafe/gossamer/lib/runtime/wazero" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" "github.com/stretchr/testify/assert" diff --git a/lib/grandpa/helpers_unit_test.go b/lib/grandpa/helpers_unit_test.go index 1c778e204d..16f480295a 100644 --- a/lib/grandpa/helpers_unit_test.go +++ b/lib/grandpa/helpers_unit_test.go @@ -10,7 +10,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/lib/runtime/genesis.go b/lib/runtime/genesis.go index 5912886bf1..27d93ff754 100644 --- a/lib/runtime/genesis.go +++ b/lib/runtime/genesis.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/ChainSafe/gossamer/lib/genesis" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) var ( diff --git a/lib/runtime/interfaces.go b/lib/runtime/interfaces.go index 2fb1f4799d..0abccf953c 100644 --- a/lib/runtime/interfaces.go +++ b/lib/runtime/interfaces.go @@ -6,7 +6,7 @@ package runtime import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" ) // Storage runtime interface. diff --git a/lib/runtime/storage/trie.go b/lib/runtime/storage/trie.go index 7b76779f5e..dfdc9275a8 100644 --- a/lib/runtime/storage/trie.go +++ b/lib/runtime/storage/trie.go @@ -11,7 +11,7 @@ import ( "sync" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "golang.org/x/exp/maps" ) diff --git a/lib/runtime/storage/trie_test.go b/lib/runtime/storage/trie_test.go index 01acbf0177..c79def9cce 100644 --- a/lib/runtime/storage/trie_test.go +++ b/lib/runtime/storage/trie_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) diff --git a/lib/runtime/wazero/imports.go b/lib/runtime/wazero/imports.go index 1fb3ae59fe..a0f797800a 100644 --- a/lib/runtime/wazero/imports.go +++ b/lib/runtime/wazero/imports.go @@ -21,9 +21,9 @@ import ( "github.com/ChainSafe/gossamer/lib/crypto/sr25519" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/proof" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/proof" "github.com/tetratelabs/wazero/api" ) diff --git a/lib/runtime/wazero/imports_test.go b/lib/runtime/wazero/imports_test.go index 5ecd82b41d..e7f955d8ab 100644 --- a/lib/runtime/wazero/imports_test.go +++ b/lib/runtime/wazero/imports_test.go @@ -24,9 +24,9 @@ import ( "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/proof" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/proof" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/runtime/wazero/instance.go b/lib/runtime/wazero/instance.go index 8a793691a9..6d62b23228 100644 --- a/lib/runtime/wazero/instance.go +++ b/lib/runtime/wazero/instance.go @@ -20,8 +20,8 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime/allocator" "github.com/ChainSafe/gossamer/lib/runtime/offchain" "github.com/ChainSafe/gossamer/lib/transaction" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/klauspost/compress/zstd" "github.com/tetratelabs/wazero" "github.com/tetratelabs/wazero/api" diff --git a/lib/runtime/wazero/instance_test.go b/lib/runtime/wazero/instance_test.go index bc42235f01..0316172f0f 100644 --- a/lib/runtime/wazero/instance_test.go +++ b/lib/runtime/wazero/instance_test.go @@ -21,9 +21,9 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/storage" "github.com/ChainSafe/gossamer/lib/runtime/wazero/testdata" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/centrifuge/go-substrate-rpc-client/v4/signature" "github.com/stretchr/testify/assert" diff --git a/lib/runtime/wazero/test_helpers.go b/lib/runtime/wazero/test_helpers.go index 067bd4173b..06725197b0 100644 --- a/lib/runtime/wazero/test_helpers.go +++ b/lib/runtime/wazero/test_helpers.go @@ -16,7 +16,7 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" "github.com/ChainSafe/gossamer/lib/runtime/mocks" "github.com/ChainSafe/gossamer/lib/runtime/storage" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" ) diff --git a/lib/trie/child_storage.go b/pkg/trie/child_storage.go similarity index 100% rename from lib/trie/child_storage.go rename to pkg/trie/child_storage.go diff --git a/lib/trie/child_storage_test.go b/pkg/trie/child_storage_test.go similarity index 100% rename from lib/trie/child_storage_test.go rename to pkg/trie/child_storage_test.go diff --git a/internal/trie/codec/nibbles.go b/pkg/trie/codec/nibbles.go similarity index 100% rename from internal/trie/codec/nibbles.go rename to pkg/trie/codec/nibbles.go diff --git a/internal/trie/codec/nibbles_test.go b/pkg/trie/codec/nibbles_test.go similarity index 100% rename from internal/trie/codec/nibbles_test.go rename to pkg/trie/codec/nibbles_test.go diff --git a/lib/trie/database.go b/pkg/trie/database.go similarity index 98% rename from lib/trie/database.go rename to pkg/trie/database.go index 667ccec5df..5a454e4d18 100644 --- a/lib/trie/database.go +++ b/pkg/trie/database.go @@ -8,10 +8,10 @@ import ( "fmt" "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/internal/trie/codec" - "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/codec" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" ) // NewBatcher creates a new database batch. diff --git a/lib/trie/database_test.go b/pkg/trie/database_test.go similarity index 100% rename from lib/trie/database_test.go rename to pkg/trie/database_test.go diff --git a/lib/trie/db/db.go b/pkg/trie/db/db.go similarity index 100% rename from lib/trie/db/db.go rename to pkg/trie/db/db.go diff --git a/lib/trie/db_getter_mocks_test.go b/pkg/trie/db_getter_mocks_test.go similarity index 92% rename from lib/trie/db_getter_mocks_test.go rename to pkg/trie/db_getter_mocks_test.go index caa0079c0c..1ace0c3dcb 100644 --- a/lib/trie/db_getter_mocks_test.go +++ b/pkg/trie/db_getter_mocks_test.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: DBGetter) +// Source: github.com/ChainSafe/gossamer/pkg/trie/db (interfaces: DBGetter) // // Generated by this command: // -// mockgen -destination=db_getter_mocks_test.go -package=trie github.com/ChainSafe/gossamer/lib/trie/db DBGetter +// mockgen -destination=db_getter_mocks_test.go -package=trie github.com/ChainSafe/gossamer/pkg/trie/db DBGetter // // Package trie is a generated GoMock package. diff --git a/lib/trie/genesis.go b/pkg/trie/genesis.go similarity index 100% rename from lib/trie/genesis.go rename to pkg/trie/genesis.go diff --git a/lib/trie/genesis_test.go b/pkg/trie/genesis_test.go similarity index 96% rename from lib/trie/genesis_test.go rename to pkg/trie/genesis_test.go index 48bff6e56b..b70e320b0d 100644 --- a/lib/trie/genesis_test.go +++ b/pkg/trie/genesis_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/pkg/trie/node" "github.com/stretchr/testify/assert" ) diff --git a/pkg/trie/hashdb/hashdb.go b/pkg/trie/hashdb/hashdb.go deleted file mode 100644 index 091ab990ea..0000000000 --- a/pkg/trie/hashdb/hashdb.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package hashdb - -import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - -type HashOut interface { - Bytes() []byte - ComparableKey() string -} - -type Hasher[Hash HashOut] interface { - Length() int - Hash(value []byte) Hash - FromBytes(value []byte) Hash -} - -type HashDB[Hash HashOut, T any] interface { - Get(key Hash, prefix nibble.Prefix) *T - Contains(key Hash, prefix nibble.Prefix) bool - Insert(prefix nibble.Prefix, value []byte) Hash - Emplace(key Hash, prefix nibble.Prefix, value T) - Remove(key Hash, prefix nibble.Prefix) -} diff --git a/lib/trie/helpers_test.go b/pkg/trie/helpers_test.go similarity index 96% rename from lib/trie/helpers_test.go rename to pkg/trie/helpers_test.go index cbae542394..1254a15b1b 100644 --- a/lib/trie/helpers_test.go +++ b/pkg/trie/helpers_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/pkg/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/tracking" "github.com/stretchr/testify/require" "golang.org/x/exp/maps" ) diff --git a/lib/trie/interfaces.go b/pkg/trie/interfaces.go similarity index 92% rename from lib/trie/interfaces.go rename to pkg/trie/interfaces.go index 452b46dbcc..8da58a28a3 100644 --- a/lib/trie/interfaces.go +++ b/pkg/trie/interfaces.go @@ -4,8 +4,8 @@ package trie import ( - "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/pkg/trie/tracking" ) // Deltas is the interface for the trie local deltas since diff --git a/lib/trie/layout.go b/pkg/trie/layout.go similarity index 100% rename from lib/trie/layout.go rename to pkg/trie/layout.go diff --git a/lib/trie/layout_test.go b/pkg/trie/layout_test.go similarity index 100% rename from lib/trie/layout_test.go rename to pkg/trie/layout_test.go diff --git a/lib/trie/mem_test.go b/pkg/trie/mem_test.go similarity index 100% rename from lib/trie/mem_test.go rename to pkg/trie/mem_test.go diff --git a/pkg/trie/memorydb/keyfunction.go b/pkg/trie/memorydb/keyfunction.go deleted file mode 100644 index 05db9a1eb2..0000000000 --- a/pkg/trie/memorydb/keyfunction.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package memorydb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -) - -type KeyFunction[H hashdb.HashOut] func(key H, prefix nibble.Prefix, hasher hashdb.Hasher[H]) H - -func HashKey[H hashdb.HashOut](key H, _ nibble.Prefix, _ hashdb.Hasher[H]) H { - return key -} - -func PrefixKey[H hashdb.HashOut](key H, prefix nibble.Prefix, hasher hashdb.Hasher[H]) H { - newLen := len(key.Bytes()) + len(prefix.PartialKey) + 1 - prefixedKey := make([]byte, 0, newLen) - - prefixedKey = append(prefixedKey, prefix.PartialKey...) - if prefix.PaddedByte != nil { - prefixedKey = append(prefixedKey, *prefix.PaddedByte) - } - prefixedKey = append(prefixedKey, key.Bytes()...) - - return hasher.FromBytes(prefixedKey) -} diff --git a/pkg/trie/memorydb/memorydb.go b/pkg/trie/memorydb/memorydb.go deleted file mode 100644 index 6fa304c2f6..0000000000 --- a/pkg/trie/memorydb/memorydb.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package memorydb - -import ( - "bytes" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -) - -type MemoryDBValue struct { - value []byte - rc int -} - -type MemoryDB[H hashdb.HashOut] struct { - data map[string]MemoryDBValue - hashedNullNode H - nullNodeData []byte - keyFunction KeyFunction[H] - hasher hashdb.Hasher[H] -} - -func newFromNullNode[H hashdb.HashOut]( - nullKey []byte, - nullNodeData []byte, - hasher hashdb.Hasher[H], - keyFunction KeyFunction[H], -) *MemoryDB[H] { - return &MemoryDB[H]{ - data: make(map[string]MemoryDBValue), - hashedNullNode: hasher.Hash(nullKey), - nullNodeData: nullNodeData, - keyFunction: keyFunction, - hasher: hasher, - } -} - -// Raw returns the raw value for the given key -func (self *MemoryDB[H]) Raw(key H, prefix nibble.Prefix) *MemoryDBValue { - if key.ComparableKey() == self.hashedNullNode.ComparableKey() { - return &MemoryDBValue{ - value: self.nullNodeData, - rc: 1, - } - } - - key = self.keyFunction(key, prefix, self.hasher) - value, ok := self.data[key.ComparableKey()] - if ok { - return &value - } - - return nil -} - -// Get returns the value for the given key -func (self *MemoryDB[H]) Get(key H, prefix nibble.Prefix) *[]byte { - if key.ComparableKey() == self.hashedNullNode.ComparableKey() { - return &self.nullNodeData - } - - key = self.keyFunction(key, prefix, self.hasher) - value, ok := self.data[key.ComparableKey()] - if ok && value.rc > 0 { - return &value.value - } - - return nil -} - -// Contains returns true if the key exists in the database -func (self *MemoryDB[H]) Contains(key H, prefix nibble.Prefix) bool { - if key.ComparableKey() == self.hashedNullNode.ComparableKey() { - return true - } - - key = self.keyFunction(key, prefix, self.hasher) - value, ok := self.data[key.ComparableKey()] - if ok && value.rc > 0 { - return true - } - - return false -} - -// Insert inserts a value into the database and returns the key -func (self *MemoryDB[H]) Insert(prefix nibble.Prefix, value []byte) H { - if bytes.Equal(value, self.nullNodeData) { - return self.hashedNullNode - } - - key := self.keyFunction(self.hasher.Hash(value), prefix, self.hasher) - self.Emplace(key, prefix, value) - return key -} - -// Emplace inserts a value into the database an updates its reference count -func (self *MemoryDB[H]) Emplace(key H, prefix nibble.Prefix, value []byte) { - if bytes.Equal(value, self.nullNodeData) { - return - } - - key = self.keyFunction(key, prefix, self.hasher) - - newEntry := MemoryDBValue{ - value: value, - rc: 1, - } - - currentEntry, ok := self.data[key.ComparableKey()] - if ok { - if currentEntry.rc <= 0 { - newEntry.value = value - } - newEntry.rc = currentEntry.rc + 1 - } - - self.data[key.ComparableKey()] = newEntry -} - -// Remove removes reduces the reference count for that key by 1 or set -1 if the value does not exists -func (self *MemoryDB[H]) Remove(key H, prefix nibble.Prefix) { - if key.ComparableKey() == self.hashedNullNode.ComparableKey() { - return - } - - key = self.keyFunction(key, prefix, self.hasher) - - entry, ok := self.data[key.ComparableKey()] - if ok { - entry.rc-- - self.data[key.ComparableKey()] = entry - } else { - self.data[key.ComparableKey()] = MemoryDBValue{ - value: nil, - rc: -1, - } - } -} - -// RemoveAndPurge removes an element and delete it from storage if reference count reaches 0. -// If the value was purged, return the old value. -func (self *MemoryDB[H]) RemoveAndPurge(key H, prefix nibble.Prefix) *[]byte { - if key.ComparableKey() == self.hashedNullNode.ComparableKey() { - return nil - } - - key = self.keyFunction(key, prefix, self.hasher) - - entry, ok := self.data[key.ComparableKey()] - if ok { - if entry.rc == 1 { - delete(self.data, key.ComparableKey()) - return &entry.value - } - entry.rc-- - self.data[key.ComparableKey()] = entry - return nil - } - - self.data[key.ComparableKey()] = MemoryDBValue{ - value: nil, - rc: -1, - } - - return nil -} - -// Purge purges all zero-referenced data from the database -func (self *MemoryDB[H]) Purge() { - for key, value := range self.data { - if value.rc == 0 { - delete(self.data, key) - } - } -} - -// Drain returns the internal key-value map, clearing the current state -func (self *MemoryDB[H]) Drain() map[string]MemoryDBValue { - data := self.data - self.data = make(map[string]MemoryDBValue) - return data -} - -// Consolidate all the entries of `other` into `self` -func (self *MemoryDB[H]) Consolidate(other *MemoryDB[H]) { - for key, dbvalue := range other.Drain() { - entry, ok := self.data[key] - if ok { - if entry.rc < 0 { - entry.value = dbvalue.value - } - entry.rc += dbvalue.rc - self.data[key] = entry - } else { - self.data[key] = MemoryDBValue{ - value: dbvalue.value, - rc: dbvalue.rc, - } - } - } -} - -// NewMemoryDB creates a new memoryDB with a null node data -func NewMemoryDB[Hash hashdb.HashOut]( - hasher hashdb.Hasher[Hash], - keyFunction KeyFunction[Hash], -) *MemoryDB[Hash] { - data := []byte{0x0} - return newFromNullNode(data, data, hasher, keyFunction) -} - -// NewMemoryDBWithRoot creates a new memoryDB with a null node data and returns the DB and the root -func NewMemoryDBWithRoot[Hash hashdb.HashOut]( - hasher hashdb.Hasher[Hash], - keyFunction KeyFunction[Hash], -) (*MemoryDB[Hash], Hash) { - db := NewMemoryDB(hasher, keyFunction) - root := db.hashedNullNode - return db, root -} diff --git a/pkg/trie/memorydb/memorydb_test.go b/pkg/trie/memorydb/memorydb_test.go deleted file mode 100644 index 122c4a06cc..0000000000 --- a/pkg/trie/memorydb/memorydb_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package memorydb - -import ( - "testing" - - "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" - "github.com/ChainSafe/gossamer/pkg/trie/test_support/reference_trie" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - "github.com/stretchr/testify/require" -) - -type KeccakHash = keccak_hasher.KeccakHash - -var hasher = keccak_hasher.NewKeccakHasher() -var V0Layout = reference_trie.LayoutV0[KeccakHash]{} - -var nullNode = []byte{0x0} -var emptyPrefix = nibble.EmptyPrefix - -func Test_New(t *testing.T) { - db := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) - hashedNullNode := hasher.Hash(nullNode) - require.Equal(t, hashedNullNode, db.Insert(emptyPrefix, nullNode)) - - db2, root := NewMemoryDBWithRoot[KeccakHash](hasher, HashKey[KeccakHash]) - require.True(t, db2.Contains(root, emptyPrefix)) - require.True(t, db.Contains(root, emptyPrefix)) -} - -func Test_Remove(t *testing.T) { - helloBytes := []byte("hello world!") - helloKey := hasher.Hash(helloBytes) - - t.Run("Remove purge insert purge", func(t *testing.T) { - m := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) - m.Remove(helloKey, emptyPrefix) - dbValue := m.Raw(helloKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, -1, dbValue.rc) - - m.Purge() - dbValue = m.Raw(helloKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, -1, dbValue.rc) - - m.Insert(emptyPrefix, helloBytes) - dbValue = m.Raw(helloKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, 0, dbValue.rc) - - m.Purge() - dbValue = m.Raw(helloKey, emptyPrefix) - require.Nil(t, dbValue) - }) - - t.Run("Remove and purge", func(t *testing.T) { - m := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) - res := m.RemoveAndPurge(helloKey, emptyPrefix) - require.Nil(t, res) - - dbValue := m.Raw(helloKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, -1, dbValue.rc) - - m.Insert(emptyPrefix, helloBytes) - m.Insert(emptyPrefix, helloBytes) - - dbValue = m.Raw(helloKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, 1, dbValue.rc) - - res = m.RemoveAndPurge(helloKey, emptyPrefix) - require.NotNil(t, res) - require.Equal(t, helloBytes, *res) - - dbValue = m.Raw(helloKey, emptyPrefix) - require.Nil(t, dbValue) - - res = m.RemoveAndPurge(helloKey, emptyPrefix) - require.Nil(t, res) - }) -} - -func Test_Consolidate(t *testing.T) { - main := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) - other := NewMemoryDB[KeccakHash](hasher, HashKey[KeccakHash]) - - removeKey := other.Insert(emptyPrefix, []byte("doggo")) - main.Remove(removeKey, emptyPrefix) - - insertKey := other.Insert(emptyPrefix, []byte("arf")) - main.Emplace(insertKey, emptyPrefix, []byte("arf")) - - negativeRemoveKey := other.Insert(emptyPrefix, []byte("negative")) - other.Remove(negativeRemoveKey, emptyPrefix) // rc = 0 - other.Remove(negativeRemoveKey, emptyPrefix) // rc = -1 - main.Remove(negativeRemoveKey, emptyPrefix) // rc = -1 - - main.Consolidate(other) - - t.Run("removeKey with rc=0", func(t *testing.T) { - dbValue := main.Raw(removeKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, []byte("doggo"), dbValue.value) - require.Equal(t, 0, dbValue.rc) - }) - - t.Run("insertKey with rc=2", func(t *testing.T) { - dbValue := main.Raw(insertKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, []byte("arf"), dbValue.value) - require.Equal(t, 2, dbValue.rc) - }) - - t.Run("negativeRemoveKey with rc=-2", func(t *testing.T) { - dbValue := main.Raw(negativeRemoveKey, emptyPrefix) - require.NotNil(t, dbValue) - require.Equal(t, []byte("negative"), dbValue.value) - require.Equal(t, -2, dbValue.rc) - }) -} diff --git a/lib/trie/mocks_generate_test.go b/pkg/trie/mocks_generate_test.go similarity index 71% rename from lib/trie/mocks_generate_test.go rename to pkg/trie/mocks_generate_test.go index f101c2239c..8d8db3d14c 100644 --- a/lib/trie/mocks_generate_test.go +++ b/pkg/trie/mocks_generate_test.go @@ -3,4 +3,4 @@ package trie -//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db DBGetter +//go:generate mockgen -destination=db_getter_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/pkg/trie/db DBGetter diff --git a/lib/trie/node.go b/pkg/trie/node.go similarity index 78% rename from lib/trie/node.go rename to pkg/trie/node.go index b16ffabb3e..da351be36a 100644 --- a/lib/trie/node.go +++ b/pkg/trie/node.go @@ -4,7 +4,7 @@ package trie import ( - "github.com/ChainSafe/gossamer/internal/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/node" ) // Node is a node in the trie and can be a leaf or a branch. diff --git a/internal/trie/node/README.md b/pkg/trie/node/README.md similarity index 100% rename from internal/trie/node/README.md rename to pkg/trie/node/README.md diff --git a/internal/trie/node/branch_encode.go b/pkg/trie/node/branch_encode.go similarity index 100% rename from internal/trie/node/branch_encode.go rename to pkg/trie/node/branch_encode.go diff --git a/internal/trie/node/branch_encode_test.go b/pkg/trie/node/branch_encode_test.go similarity index 100% rename from internal/trie/node/branch_encode_test.go rename to pkg/trie/node/branch_encode_test.go diff --git a/internal/trie/node/buffer.go b/pkg/trie/node/buffer.go similarity index 100% rename from internal/trie/node/buffer.go rename to pkg/trie/node/buffer.go diff --git a/internal/trie/node/buffer_mock_test.go b/pkg/trie/node/buffer_mock_test.go similarity index 96% rename from internal/trie/node/buffer_mock_test.go rename to pkg/trie/node/buffer_mock_test.go index 396f9bd50c..6e6f51b74e 100644 --- a/internal/trie/node/buffer_mock_test.go +++ b/pkg/trie/node/buffer_mock_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/internal/trie/node (interfaces: Buffer) +// Source: github.com/ChainSafe/gossamer/pkg/trie/node (interfaces: Buffer) // // Generated by this command: // diff --git a/internal/trie/node/children.go b/pkg/trie/node/children.go similarity index 100% rename from internal/trie/node/children.go rename to pkg/trie/node/children.go diff --git a/internal/trie/node/children_test.go b/pkg/trie/node/children_test.go similarity index 100% rename from internal/trie/node/children_test.go rename to pkg/trie/node/children_test.go diff --git a/internal/trie/node/copy.go b/pkg/trie/node/copy.go similarity index 100% rename from internal/trie/node/copy.go rename to pkg/trie/node/copy.go diff --git a/internal/trie/node/copy_test.go b/pkg/trie/node/copy_test.go similarity index 100% rename from internal/trie/node/copy_test.go rename to pkg/trie/node/copy_test.go diff --git a/internal/trie/node/decode.go b/pkg/trie/node/decode.go similarity index 100% rename from internal/trie/node/decode.go rename to pkg/trie/node/decode.go diff --git a/internal/trie/node/decode_test.go b/pkg/trie/node/decode_test.go similarity index 100% rename from internal/trie/node/decode_test.go rename to pkg/trie/node/decode_test.go diff --git a/internal/trie/node/dirty.go b/pkg/trie/node/dirty.go similarity index 100% rename from internal/trie/node/dirty.go rename to pkg/trie/node/dirty.go diff --git a/internal/trie/node/dirty_test.go b/pkg/trie/node/dirty_test.go similarity index 100% rename from internal/trie/node/dirty_test.go rename to pkg/trie/node/dirty_test.go diff --git a/internal/trie/node/encode.go b/pkg/trie/node/encode.go similarity index 97% rename from internal/trie/node/encode.go rename to pkg/trie/node/encode.go index 1e5810ea34..e7697344b5 100644 --- a/internal/trie/node/encode.go +++ b/pkg/trie/node/encode.go @@ -6,9 +6,9 @@ package node import ( "fmt" - "github.com/ChainSafe/gossamer/internal/trie/codec" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie/codec" ) // Encode encodes the node to the buffer given. diff --git a/internal/trie/node/encode_decode_test.go b/pkg/trie/node/encode_decode_test.go similarity index 100% rename from internal/trie/node/encode_decode_test.go rename to pkg/trie/node/encode_decode_test.go diff --git a/internal/trie/node/encode_test.go b/pkg/trie/node/encode_test.go similarity index 100% rename from internal/trie/node/encode_test.go rename to pkg/trie/node/encode_test.go diff --git a/internal/trie/node/hash.go b/pkg/trie/node/hash.go similarity index 98% rename from internal/trie/node/hash.go rename to pkg/trie/node/hash.go index 1dfbca098d..1fca6bfade 100644 --- a/internal/trie/node/hash.go +++ b/pkg/trie/node/hash.go @@ -9,7 +9,7 @@ import ( "hash" "io" - "github.com/ChainSafe/gossamer/internal/trie/pools" + "github.com/ChainSafe/gossamer/pkg/trie/pools" ) // MerkleValue writes the Merkle value from the encoding of a non-root diff --git a/internal/trie/node/hash_test.go b/pkg/trie/node/hash_test.go similarity index 100% rename from internal/trie/node/hash_test.go rename to pkg/trie/node/hash_test.go diff --git a/internal/trie/node/header.go b/pkg/trie/node/header.go similarity index 100% rename from internal/trie/node/header.go rename to pkg/trie/node/header.go diff --git a/internal/trie/node/header_test.go b/pkg/trie/node/header_test.go similarity index 100% rename from internal/trie/node/header_test.go rename to pkg/trie/node/header_test.go diff --git a/internal/trie/node/helpers_test.go b/pkg/trie/node/helpers_test.go similarity index 100% rename from internal/trie/node/helpers_test.go rename to pkg/trie/node/helpers_test.go diff --git a/internal/trie/node/key.go b/pkg/trie/node/key.go similarity index 93% rename from internal/trie/node/key.go rename to pkg/trie/node/key.go index 343a5d747d..2d8fe5c56c 100644 --- a/internal/trie/node/key.go +++ b/pkg/trie/node/key.go @@ -8,7 +8,7 @@ import ( "fmt" "io" - "github.com/ChainSafe/gossamer/internal/trie/codec" + "github.com/ChainSafe/gossamer/pkg/trie/codec" ) const maxPartialKeyLength = ^uint16(0) diff --git a/internal/trie/node/key_test.go b/pkg/trie/node/key_test.go similarity index 100% rename from internal/trie/node/key_test.go rename to pkg/trie/node/key_test.go diff --git a/internal/trie/node/mocks_generate_test.go b/pkg/trie/node/mocks_generate_test.go similarity index 100% rename from internal/trie/node/mocks_generate_test.go rename to pkg/trie/node/mocks_generate_test.go diff --git a/internal/trie/node/node.go b/pkg/trie/node/node.go similarity index 100% rename from internal/trie/node/node.go rename to pkg/trie/node/node.go diff --git a/internal/trie/node/node_test.go b/pkg/trie/node/node_test.go similarity index 100% rename from internal/trie/node/node_test.go rename to pkg/trie/node/node_test.go diff --git a/internal/trie/node/reader_mock_test.go b/pkg/trie/node/reader_mock_test.go similarity index 100% rename from internal/trie/node/reader_mock_test.go rename to pkg/trie/node/reader_mock_test.go diff --git a/internal/trie/node/subvalue.go b/pkg/trie/node/subvalue.go similarity index 100% rename from internal/trie/node/subvalue.go rename to pkg/trie/node/subvalue.go diff --git a/internal/trie/node/subvalue_test.go b/pkg/trie/node/subvalue_test.go similarity index 100% rename from internal/trie/node/subvalue_test.go rename to pkg/trie/node/subvalue_test.go diff --git a/internal/trie/node/types.go b/pkg/trie/node/types.go similarity index 100% rename from internal/trie/node/types.go rename to pkg/trie/node/types.go diff --git a/internal/trie/node/variants.go b/pkg/trie/node/variants.go similarity index 100% rename from internal/trie/node/variants.go rename to pkg/trie/node/variants.go diff --git a/internal/trie/node/writer_mock_test.go b/pkg/trie/node/writer_mock_test.go similarity index 100% rename from internal/trie/node/writer_mock_test.go rename to pkg/trie/node/writer_mock_test.go diff --git a/internal/trie/pools/pools.go b/pkg/trie/pools/pools.go similarity index 100% rename from internal/trie/pools/pools.go rename to pkg/trie/pools/pools.go diff --git a/lib/trie/print.go b/pkg/trie/print.go similarity index 100% rename from lib/trie/print.go rename to pkg/trie/print.go diff --git a/lib/trie/print_test.go b/pkg/trie/print_test.go similarity index 100% rename from lib/trie/print_test.go rename to pkg/trie/print_test.go diff --git a/lib/trie/proof/database_mocks_test.go b/pkg/trie/proof/database_mocks_test.go similarity index 92% rename from lib/trie/proof/database_mocks_test.go rename to pkg/trie/proof/database_mocks_test.go index ab48cca603..8743c92069 100644 --- a/lib/trie/proof/database_mocks_test.go +++ b/pkg/trie/proof/database_mocks_test.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/trie/db (interfaces: DBGetter) +// Source: github.com/ChainSafe/gossamer/pkg/trie/db (interfaces: DBGetter) // // Generated by this command: // -// mockgen -destination=database_mocks_test.go -package=proof github.com/ChainSafe/gossamer/lib/trie/db DBGetter +// mockgen -destination=database_mocks_test.go -package=proof github.com/ChainSafe/gossamer/pkg/trie/db DBGetter // // Package proof is a generated GoMock package. diff --git a/lib/trie/proof/generate.go b/pkg/trie/proof/generate.go similarity index 95% rename from lib/trie/proof/generate.go rename to pkg/trie/proof/generate.go index 0daca83e52..2c3ed4a981 100644 --- a/lib/trie/proof/generate.go +++ b/pkg/trie/proof/generate.go @@ -8,12 +8,12 @@ import ( "errors" "fmt" - "github.com/ChainSafe/gossamer/internal/trie/codec" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/internal/trie/pools" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/codec" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/pools" ) var ( diff --git a/lib/trie/proof/generate_test.go b/pkg/trie/proof/generate_test.go similarity index 99% rename from lib/trie/proof/generate_test.go rename to pkg/trie/proof/generate_test.go index 44e3c118d2..e05a7d2a55 100644 --- a/lib/trie/proof/generate_test.go +++ b/pkg/trie/proof/generate_test.go @@ -7,10 +7,10 @@ import ( "errors" "testing" - "github.com/ChainSafe/gossamer/internal/trie/codec" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/codec" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/lib/trie/proof/helpers_test.go b/pkg/trie/proof/helpers_test.go similarity index 96% rename from lib/trie/proof/helpers_test.go rename to pkg/trie/proof/helpers_test.go index 58ec6bb417..050c7094aa 100644 --- a/lib/trie/proof/helpers_test.go +++ b/pkg/trie/proof/helpers_test.go @@ -8,10 +8,10 @@ import ( "math/rand" "testing" - "github.com/ChainSafe/gossamer/internal/trie/node" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/node" "github.com/stretchr/testify/require" ) diff --git a/lib/trie/proof/mocks_generate_test.go b/pkg/trie/proof/mocks_generate_test.go similarity index 71% rename from lib/trie/proof/mocks_generate_test.go rename to pkg/trie/proof/mocks_generate_test.go index adde5e4e89..cf5c92d081 100644 --- a/lib/trie/proof/mocks_generate_test.go +++ b/pkg/trie/proof/mocks_generate_test.go @@ -3,4 +3,4 @@ package proof -//go:generate mockgen -destination=database_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/lib/trie/db DBGetter +//go:generate mockgen -destination=database_mocks_test.go -package=$GOPACKAGE github.com/ChainSafe/gossamer/pkg/trie/db DBGetter diff --git a/lib/trie/proof/proof_test.go b/pkg/trie/proof/proof_test.go similarity index 99% rename from lib/trie/proof/proof_test.go rename to pkg/trie/proof/proof_test.go index a45e4ffde3..e603311eee 100644 --- a/lib/trie/proof/proof_test.go +++ b/pkg/trie/proof/proof_test.go @@ -9,8 +9,8 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/db" "github.com/stretchr/testify/require" ) diff --git a/lib/trie/proof/verify.go b/pkg/trie/proof/verify.go similarity index 97% rename from lib/trie/proof/verify.go rename to pkg/trie/proof/verify.go index 97e439fdec..f913f731b4 100644 --- a/lib/trie/proof/verify.go +++ b/pkg/trie/proof/verify.go @@ -10,11 +10,11 @@ import ( "strings" "github.com/ChainSafe/gossamer/internal/log" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/internal/trie/pools" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/pools" ) var ( diff --git a/lib/trie/proof/verify_test.go b/pkg/trie/proof/verify_test.go similarity index 99% rename from lib/trie/proof/verify_test.go rename to pkg/trie/proof/verify_test.go index 78e0e5dd50..1b4e70411f 100644 --- a/lib/trie/proof/verify_test.go +++ b/pkg/trie/proof/verify_test.go @@ -6,9 +6,9 @@ package proof import ( "testing" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/lib/trie" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go b/pkg/trie/test_support/keccak_hasher/keccak_hasher.go deleted file mode 100644 index 20ce958298..0000000000 --- a/pkg/trie/test_support/keccak_hasher/keccak_hasher.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package keccak_hasher - -import ( - "fmt" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "golang.org/x/crypto/sha3" -) - -const KeccakHasherLength = 32 - -type KeccakHash [KeccakHasherLength]byte - -func (k KeccakHash) Bytes() []byte { - return k[:] -} - -func (k KeccakHash) ComparableKey() string { - return fmt.Sprintf("%x", k) -} - -var _ hashdb.HashOut = KeccakHash{} - -type KeccakHasher struct{} - -func (k KeccakHasher) Length() int { - return KeccakHasherLength -} - -func (k KeccakHasher) FromBytes(in []byte) KeccakHash { - var buf = [KeccakHasherLength]byte{} - copy(buf[:], in) - return KeccakHash(buf) -} - -func (k KeccakHasher) Hash(in []byte) KeccakHash { - h := sha3.NewLegacyKeccak256() - - _, err := h.Write(in) - if err != nil { - panic("Unexpected error hashing bytes") - } - - hash := h.Sum(nil) - return k.FromBytes(hash) -} - -func NewKeccakHasher() KeccakHasher { - return KeccakHasher{} -} - -var _ hashdb.Hasher[KeccakHash] = (*KeccakHasher)(nil) diff --git a/pkg/trie/test_support/reference_trie/substrate.go b/pkg/trie/test_support/reference_trie/substrate.go deleted file mode 100644 index 617714420c..0000000000 --- a/pkg/trie/test_support/reference_trie/substrate.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package reference_trie - -import ( - "errors" - "io" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" - "github.com/ChainSafe/gossamer/pkg/trie/triedb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" -) - -const firstPrefix = 0b_00 << 6 -const leafPrefixMask = 0b_01 << 6 -const branchWithoutValueMask = 0b_10 << 6 -const branchWithValueMask = 0b_11 << 6 -const emptyTrie = firstPrefix | (0b_00 << 4) -const leafWithHashedValuePrefixMask = firstPrefix | (0b_1 << 5) -const branchWithHashedValuePrefixMask = firstPrefix | (0b_1 << 4) -const escapeCompactHeader = emptyTrie | 0b_00_01 - -var hasher = keccak_hasher.NewKeccakHasher() - -type byteSliceInput struct { - data []byte - offset int -} - -func NewByteSliceInput(data []byte) byteSliceInput { - return byteSliceInput{data, 0} -} - -func (self byteSliceInput) Take(count int) (node.BytesRange, error) { - if self.offset+count > len(self.data) { - return node.BytesRange{}, errors.New("out of data") - } - - res := node.BytesRange{self.offset, self.offset + count} - self.offset += count - return res, nil -} - -type NodeHeader interface { - Type() string -} - -type ( - NullNodeHeader struct{} - BranchNodeHeader struct { - hasValue bool - nibbleCount int - } - LeafNodeHeader struct { - nibbleCount int - } - HashedValueBranchNodeHeader struct { - nibbleCount int - } - HashedValueLeaf struct { - nibbleCount int - } -) - -func (NullNodeHeader) Type() string { return "Null" } -func (BranchNodeHeader) Type() string { return "Branch" } -func (LeafNodeHeader) Type() string { return "Leaf" } -func (HashedValueBranchNodeHeader) Type() string { return "HashedValueBranch" } -func (HashedValueLeaf) Type() string { return "HashedValueLeaf" } - -func headerContainsHashedValues(header NodeHeader) bool { - switch header.(type) { - case HashedValueBranchNodeHeader, HashedValueLeaf: - return true - default: - return false - } -} - -// Decode nibble count from stream input and header byte -func decodeSize(first byte, input io.Reader, prefixMask int) (int, error) { - maxValue := byte(255) >> prefixMask - result := (first & maxValue) - if result < maxValue { - return int(result), nil - } - result -= 1 - for { - b := make([]byte, 1) - _, err := input.Read(b) - if err != nil { - return -1, err - } - n := int(b[0]) - if n < 255 { - return int(result) + n + 1, nil - } - result += 255 - } -} - -// DecodeHeader decodes a node header from a stream input -func DecodeHeader(input io.Reader) (NodeHeader, error) { - b := make([]byte, 1) - _, err := input.Read(b) - if err != nil { - return nil, err - } - i := b[0] - - if i == emptyTrie { - return NullNodeHeader{}, nil - } - - mask := i & (0b11 << 6) - - var ( - size int - node NodeHeader - ) - - switch mask { - case leafPrefixMask: - size, err = decodeSize(i, input, 2) - node = LeafNodeHeader{size} - case branchWithValueMask: - size, err = decodeSize(i, input, 2) - node = BranchNodeHeader{true, size} - case branchWithoutValueMask: - size, err = decodeSize(i, input, 2) - node = BranchNodeHeader{false, size} - case emptyTrie: - if i&(0b111<<5) == leafWithHashedValuePrefixMask { - size, err = decodeSize(i, input, 3) - node = HashedValueLeaf{size} - } else if i&(0b1111<<4) == branchWithHashedValuePrefixMask { - size, err = decodeSize(i, input, 4) - node = HashedValueBranchNodeHeader{size} - } else { - err = errors.New("invalid header") - } - default: - panic("unreachable") - } - - if err != nil { - return nil, err - } - return node, err -} - -// NodeCodec is the node codec configuration used in substrate -type NodeCodec[H hashdb.HashOut] struct { - hasher hashdb.Hasher[H] -} - -// HashedNullNode returns the hash of an empty node -func (self NodeCodec[H]) HashedNullNode() H { - return self.hasher.Hash(self.EmptyNode()) -} - -// Hasher returns the hasher used for this codec -func (self NodeCodec[H]) Hasher() hashdb.Hasher[H] { - return self.hasher -} - -// EmptyNode returns an empty node -func (self NodeCodec[H]) EmptyNode() []byte { - return []byte{emptyTrie} -} - -// LeafNode encodes a leaf node -func (self NodeCodec[H]) LeafNode(partialKey nibble.NibbleSlice, numberNibble int, value node.Value) []byte { - panic("Implement me") -} - -// BranchNodeNibbled encodes a branch node -func (self NodeCodec[H]) BranchNodeNibbled( - partialKey nibble.NibbleSlice, - numberNibble int, - children [16]node.ChildReference[H], - value *node.Value, -) []byte { - panic("Implement me") -} - -func (self NodeCodec[H]) decodePlan(data []byte) (node.NodePlan, error) { - //input := NewByteSliceInput(data) - - return nil, nil -} - -// Decode decodes bytes to a Node -func (self NodeCodec[H]) Decode(data []byte) (node.Node, error) { - plan, err := self.decodePlan(data) - if err != nil { - return nil, err - } - return plan.Build(data), nil -} - -func NewNodeCodecForKeccak() NodeCodec[keccak_hasher.KeccakHash] { - return NodeCodec[keccak_hasher.KeccakHash]{hasher} -} - -var SubstrateNodeCodec node.NodeCodec[keccak_hasher.KeccakHash] = NodeCodec[keccak_hasher.KeccakHash]{} - -type LayoutV0[H hashdb.HashOut] struct { - codec node.NodeCodec[H] -} - -func (l LayoutV0[H]) AllowEmpty() bool { - return true -} - -func (l LayoutV0[H]) MaxInlineValue() *uint { - return nil -} - -func (l LayoutV0[H]) Codec() node.NodeCodec[H] { - return l.codec -} - -var V0Layout triedb.TrieLayout[keccak_hasher.KeccakHash] = LayoutV0[keccak_hasher.KeccakHash]{NewNodeCodecForKeccak()} diff --git a/pkg/trie/tests/recorder_test.go b/pkg/trie/tests/recorder_test.go deleted file mode 100644 index e50ad420fb..0000000000 --- a/pkg/trie/tests/recorder_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package tests - -import ( - "testing" - - "github.com/ChainSafe/gossamer/pkg/trie/memorydb" - "github.com/ChainSafe/gossamer/pkg/trie/test_support/keccak_hasher" - "github.com/ChainSafe/gossamer/pkg/trie/test_support/reference_trie" - "github.com/ChainSafe/gossamer/pkg/trie/triedb" - "github.com/stretchr/testify/assert" -) - -type KeccakHash = keccak_hasher.KeccakHash - -var hasher = keccak_hasher.NewKeccakHasher() -var V0Layout = reference_trie.V0Layout - -func Test_Record(t *testing.T) { - db := memorydb.NewMemoryDB[KeccakHash](keccak_hasher.NewKeccakHasher(), memorydb.HashKey[KeccakHash]) - - rootBytes := make([]byte, 32) - root := hasher.FromBytes(rootBytes) - - pairs := []struct { - key []byte - value []byte - }{ - {[]byte("dog"), []byte("cat")}, - /*{[]byte("lunch"), []byte("time")}, - {[]byte("notdog"), []byte("notcat")}, - {[]byte("hotdog"), []byte("hotcat")}, - {[]byte("letter"), []byte("confusion")}, - {[]byte("insert"), []byte("remove")}, - {[]byte("pirate"), []byte("aargh!")}, - {[]byte("yo ho ho"), []byte("and a bottle of rum")},*/ - } - - tdb := triedb.NewTrieDBBuilder[KeccakHash](db, root, V0Layout).Build() - - for _, pair := range pairs { - _, err := tdb.Insert(pair.key, pair.value) - assert.NoError(t, err) - } -} diff --git a/internal/trie/tracking/deltas.go b/pkg/trie/tracking/deltas.go similarity index 100% rename from internal/trie/tracking/deltas.go rename to pkg/trie/tracking/deltas.go diff --git a/internal/trie/tracking/deltas_test.go b/pkg/trie/tracking/deltas_test.go similarity index 100% rename from internal/trie/tracking/deltas_test.go rename to pkg/trie/tracking/deltas_test.go diff --git a/internal/trie/tracking/helpers_test.go b/pkg/trie/tracking/helpers_test.go similarity index 100% rename from internal/trie/tracking/helpers_test.go rename to pkg/trie/tracking/helpers_test.go diff --git a/internal/trie/tracking/interfaces.go b/pkg/trie/tracking/interfaces.go similarity index 100% rename from internal/trie/tracking/interfaces.go rename to pkg/trie/tracking/interfaces.go diff --git a/lib/trie/trie.go b/pkg/trie/trie.go similarity index 99% rename from lib/trie/trie.go rename to pkg/trie/trie.go index 99152be798..da30b5de26 100644 --- a/lib/trie/trie.go +++ b/pkg/trie/trie.go @@ -8,11 +8,11 @@ import ( "fmt" "reflect" - "github.com/ChainSafe/gossamer/internal/trie/codec" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/codec" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/tracking" ) // EmptyHash is the empty trie hash. diff --git a/lib/trie/trie_endtoend_test.go b/pkg/trie/trie_endtoend_test.go similarity index 99% rename from lib/trie/trie_endtoend_test.go rename to pkg/trie/trie_endtoend_test.go index 1cccd342be..2c2046fa6c 100644 --- a/lib/trie/trie_endtoend_test.go +++ b/pkg/trie/trie_endtoend_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/require" "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/internal/trie/codec" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/pkg/trie/codec" ) const ( diff --git a/lib/trie/trie_test.go b/pkg/trie/trie_test.go similarity index 99% rename from lib/trie/trie_test.go rename to pkg/trie/trie_test.go index 7c78e3afb2..a4bce36c2f 100644 --- a/lib/trie/trie_test.go +++ b/pkg/trie/trie_test.go @@ -9,10 +9,10 @@ import ( "reflect" "testing" - "github.com/ChainSafe/gossamer/internal/trie/node" - "github.com/ChainSafe/gossamer/internal/trie/tracking" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/db" + "github.com/ChainSafe/gossamer/pkg/trie/node" + "github.com/ChainSafe/gossamer/pkg/trie/tracking" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" diff --git a/pkg/trie/triedb/cache.go b/pkg/trie/triedb/cache.go deleted file mode 100644 index 47ba81c9e8..0000000000 --- a/pkg/trie/triedb/cache.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" -) - -// CachedValue a value as cached by TrieCache -type CachedValue[H hashdb.HashOut] interface { - Type() string -} -type ( - // The value doesn't exists in ithe trie - NonExisting struct{} - // We cached the hash, because we did not yet accessed the data - ExistingHash[H comparable] struct { - hash H - } - // The value xists in the trie - Existing[H comparable] struct { - hash H // The hash of the value - data []byte // The actual data of the value - } -) - -func (v NonExisting) Type() string { return "NonExisting" } -func (v ExistingHash[H]) Type() string { return "ExistingHash" } -func (v Existing[H]) Type() string { return "Existing" } - -type TrieCache[Out hashdb.HashOut] interface { - LookupValueForKey(key []byte) *CachedValue[Out] - CacheValueForKey(key []byte, value CachedValue[Out]) - GetOrInsertNode(hash Out, fetchNode func() (node.NodeOwned[Out], error)) - GetNode(hash Out) *node.NodeOwned[Out] -} diff --git a/pkg/trie/triedb/errors.go b/pkg/trie/triedb/errors.go deleted file mode 100644 index e548fe9355..0000000000 --- a/pkg/trie/triedb/errors.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import "errors" - -var ErrInvalidStateRoot = errors.New("invalid state root") -var ErrIncompleteDB = errors.New("incomplete database") -var DecoderError = errors.New("corrupt trie item") -var InvalidHash = errors.New("hash is not value") diff --git a/pkg/trie/triedb/layout.go b/pkg/trie/triedb/layout.go deleted file mode 100644 index 65c947294d..0000000000 --- a/pkg/trie/triedb/layout.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" -) - -type TrieLayout[H hashdb.HashOut] interface { - AllowEmpty() bool - MaxInlineValue() *uint - Codec() node.NodeCodec[H] -} diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go deleted file mode 100644 index ccffea3219..0000000000 --- a/pkg/trie/triedb/lookup.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2022 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" -) - -var EmptyValue = []byte{} - -type Lookup[Hash hashdb.HashOut] struct { - db hashdb.HashDB[Hash, DBValue] - hash Hash - cache TrieCache[Hash] - recorder TrieRecorder[Hash] - layout TrieLayout[Hash] -} - -func NewLookup[H hashdb.HashOut]( - db hashdb.HashDB[H, DBValue], hash H, cache TrieCache[H], recorder TrieRecorder[H]) *Lookup[H] { - return &Lookup[H]{ - db: db, - hash: hash, - cache: cache, - recorder: recorder, - } -} - -func (l Lookup[H]) Lookup(nibbleKey *nibble.NibbleSlice) ([]byte, error) { - return l.lookupWithoutCache(nibbleKey) -} - -func (l Lookup[H]) record(access TrieAccess[H]) { - if l.recorder != nil { - l.recorder.record(access) - } -} - -func (l Lookup[H]) lookupWithoutCache(nibbleKey *nibble.NibbleSlice) ([]byte, error) { - partial := nibbleKey - hash := l.hash - keyNibbles := 0 - - depth := 0 - - for { - // Get node from DB - nodeData := l.db.Get(hash, nibbleKey.Mid(keyNibbles).Left()) - if nodeData == nil { - if depth == 0 { - return nil, ErrInvalidStateRoot - } else { - return nil, ErrIncompleteDB - } - } - - l.record(TrieAccessEncodedNode[H]{ - hash: hash, - encodedNode: *nodeData, - }) - - // Iterates children - childrens: - for { - // Decode node - decodedNode, err := l.layout.Codec().Decode(*nodeData) - if err != nil { - return nil, DecoderError - } - - var nextNode node.NodeHandle = nil - - switch node := decodedNode.(type) { - case node.Empty: - return EmptyValue, nil - case node.Leaf: - // If leaf and matches return value - if partial.Eq(node.PartialKey) { - return l.loadValue(node.Value, nibbleKey.OriginalDataAsPrefix()) - } - return EmptyValue, nil - case node.NibbledBranch: - slice := node.PartialKey - children := node.Children - value := node.Value - // Get next node - if !partial.StartsWith(slice) { - return EmptyValue, nil - } - - if partial.Len() == slice.Len() { - if value != nil { - return l.loadValue(value, nibbleKey.OriginalDataAsPrefix()) - } - } - - nextNode = children[partial.At(slice.Len())] - if nextNode == nil { - return EmptyValue, nil - } - - partial = partial.Mid(slice.Len() + 1) - keyNibbles += slice.Len() + 1 - } - - switch n := nextNode.(type) { - case node.Hash: - hash, err = node.DecodeHash(l.layout.Codec().Hasher(), n.Data) - if err != nil { - return nil, InvalidHash - } - break childrens - case node.Inline: - nodeData = &n.Data - } - depth++ - } - } -} - -func (l Lookup[H]) loadValue(value node.Value, prefix nibble.Prefix) ([]byte, error) { - switch v := value.(type) { - case node.InlineValue: - return v.Bytes, nil - case node.NodeValue: - hash := l.layout.Codec().Hasher().FromBytes(v.Bytes) - bytes := l.db.Get(hash, prefix) - if bytes == nil { - return nil, ErrIncompleteDB - } - return *bytes, nil - default: - panic("unknown value type") - } -} diff --git a/pkg/trie/triedb/nibble/nibble.go b/pkg/trie/triedb/nibble/nibble.go deleted file mode 100644 index b17548c4b6..0000000000 --- a/pkg/trie/triedb/nibble/nibble.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package nibble - -const NibblePerByte int = 2 -const PaddingBitmask byte = 0x0F -const BitPerNibble = 4 -const NibbleLength = 16 -const SplitLeftShift = 4 -const SplitRightShift = 4 - -// / A trie node prefix, it is the nibble path from the trie root -// / to the trie node. -// / For a node containing no partial key value it is the full key. -// / For a value node or node containing a partial key, it is the full key minus its node partial -// / nibbles (the node key can be split into prefix and node partial). -// / Therefore it is always the leftmost portion of the node key, so its internal representation -// / is a non expanded byte slice followed by a last padded byte representation. -// / The padded byte is an optional padded value. -type Prefix struct { - PartialKey []byte - PaddedByte *byte -} - -var EmptyPrefix = Prefix{PartialKey: []byte{}, PaddedByte: nil} - -func PadLeft(b byte) byte { - padded := (b & ^PaddingBitmask) - return padded -} - -func padRight(b byte) byte { - padded := (b & PaddingBitmask) - return padded -} - -func NumberPadding(i int) int { - return i % NibblePerByte -} - -func PushAtLeft(ix, v, into byte) byte { - if ix != 1 { - v = v << BitPerNibble - } - return into | v -} - -func ShiftKey(key *NibbleSlice, offset int) bool { - oldOffset := key.offset - key.offset = offset - - if oldOffset > offset { - // Shift left - kl := key.Len() - for i := 0; i < kl; i++ { - key.data[i] = key.data[i]<<2 | key.data[i+1]>>SplitLeftShift - } - key.data[kl-1] = key.data[kl-1] << SplitRightShift - return true - } else if oldOffset < offset { - // Shift right - key.data = append(key.data, 0) - for i := key.Len() - 1; i >= 1; i-- { - key.data[i] = key.data[i-1]<>SplitRightShift - } - key.data[0] = key.data[0] >> SplitRightShift - return true - } else { - return false - } -} - -// Count the biggest common depth between two left aligned packed nibble slice -func biggestDepth(v1, v2 []byte) int { - upperBound := minLength(v1, v2) - - for i := 0; i < upperBound; i++ { - if v1[i] != v2[i] { - return i*NibblePerByte + leftCommon(v1[i], v2[i]) - } - } - return upperBound * NibblePerByte -} - -// LeftCommon the number of common nibble between two left aligned bytes -func leftCommon(a, b byte) int { - if a == b { - return 2 - } - if PadLeft(a) == PadLeft(b) { - return 1 - } else { - return 0 - } -} - -func minLength(v1, v2 []byte) int { - if len(v1) < len(v2) { - return len(v1) - } - return len(v2) -} - -// CombineKeys combines two node keys representd by nibble slices into the first one -func CombineKeys(start *NibbleSlice, end NibbleSlice) { - if start.offset >= NibblePerByte || end.offset >= NibblePerByte { - panic("Cannot combine keys") - } - finalOffset := (start.offset + end.offset) % NibblePerByte - ShiftKey(start, finalOffset) - var st int - if end.offset > 0 { - startLen := start.Len() - start.data[startLen-1] = padRight(end.data[0]) - st = 1 - } else { - st = 0 - } - for i := st; i < end.Len(); i++ { - start.data = append(start.data, end.data[i]) - } -} diff --git a/pkg/trie/triedb/nibble/nibbleslice.go b/pkg/trie/triedb/nibble/nibbleslice.go deleted file mode 100644 index f8ec8012b5..0000000000 --- a/pkg/trie/triedb/nibble/nibbleslice.go +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2022 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package nibble - -// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. -// -// This is an immutable struct. No operations actually change it. -// -// # Example (TODO: translate to go) -// ```snippet -// use patricia_trie::nibbleslice::NibbleSlice; -// -// fn main() { -// let d1 = &[0x01u8, 0x23, 0x45]; -// let d2 = &[0x34u8, 0x50, 0x12]; -// let d3 = &[0x00u8, 0x12]; -// let n1 = NibbleSlice::new(d1); // 0,1,2,3,4,5 -// let n2 = NibbleSlice::new(d2); // 3,4,5,0,1,2 -// let n3 = NibbleSlice::new_offset(d3, 1); // 0,1,2 -// assert!(n1 > n3); // 0,1,2,... > 0,1,2 -// assert!(n1 < n2); // 0,... < 3,... -// assert!(n2.mid(3) == n3); // 0,1,2 == 0,1,2 -// assert!(n1.starts_with(&n3)); -// assert_eq!(n1.common_prefix(&n3), 3); -// assert_eq!(n2.mid(3).common_prefix(&n1), 3); -// } -// -// ``` -type NibbleSlice struct { - data []byte - offset int -} - -// NewNibbleSlice creates a new nibble slice from a byte slice -func NewNibbleSlice(data []byte) *NibbleSlice { - return &NibbleSlice{data, 0} -} - -// NewNibbleSliceWithOffset creates a new nibble slice from a byte slice with an offset -func NewNibbleSliceWithOffset(data []byte, offset int) *NibbleSlice { - return &NibbleSlice{data, offset} -} - -// Clone creates a deep copy of the nibble slice -func (ns *NibbleSlice) Clone() *NibbleSlice { - data := make([]byte, len(ns.data)) - copy(data, ns.data) - return &NibbleSlice{data, ns.offset} -} - -// ToStored is a helper function to create a node key from this nibble slice -func (ns *NibbleSlice) ToStored() NibbleSlice { - split := ns.offset / NibblePerByte - offset := ns.offset % NibblePerByte - return NibbleSlice{ - data: ns.data[split:], - offset: offset, - } -} - -// ToStoredRange is a helper function to create a node key from this `NibbleSlice` and for a given number of nibble -func (ns *NibbleSlice) ToStoredRange(nb int) NibbleSlice { - if nb > ns.Len() { - ns.ToStored() - } - if (ns.offset+nb)%NibblePerByte == 0 { - // aligned - start := ns.offset / NibblePerByte - end := (ns.offset + nb) / NibblePerByte - return NibbleSlice{ - data: ns.data[start:end], - offset: ns.offset % NibblePerByte, - } - } else { - // unaligned - start := ns.offset / NibblePerByte - end := (ns.offset + nb) / NibblePerByte - ea := ns.data[start : end+1] - eaOffset := ns.offset - nOffset := NumberPadding(nb) - result := NibbleSlice{ - ea, - eaOffset, - } - ShiftKey(&result, nOffset) - result.data = result.data[:len(result.data)-1] - return result - } -} - -// IsEmpty Return true if the slice contains no nibbles -func (ns *NibbleSlice) IsEmpty() bool { - return ns.Len() == 0 -} - -// Advance the view on the slice by `i` nibbles -func (ns *NibbleSlice) Advance(i int) { - if ns.Len() < i { - panic("Cannot advance more than the length of the slice") - } - ns.offset += i -} - -// Data returns the underlying byte slice -func (ns *NibbleSlice) Data() []byte { - return ns.data -} - -// Offset returns the offset of the nibble slice -func (ns *NibbleSlice) Offset() int { - return ns.offset -} - -// Mid returns a new nibble slice object which represents a view on to this slice (further) offset by `i` nibbles -func (ns *NibbleSlice) Mid(i int) *NibbleSlice { - return &NibbleSlice{ns.data, ns.offset + i} -} - -// Len returns the length of the nibble slice considering the offset -func (ns *NibbleSlice) Len() int { - return len(ns.data)*NibblePerByte - ns.offset -} - -// At returns the nibble at position `i` -func (ns *NibbleSlice) At(i int) byte { - ix := (ns.offset + i) / NibblePerByte - pad := (ns.offset + i) % NibblePerByte - b := ns.data[ix] - if pad == 1 { - return b & PaddingBitmask - } - return b >> BitPerNibble -} - -// StartsWith returns true if this nibble slice start with the same same nibbles contained in `other` -func (ns *NibbleSlice) StartsWith(other NibbleSlice) bool { - return ns.CommonPrefix(other) == other.Len() -} - -// Eq returns true if this nibble slice is equal to `other` -func (ns *NibbleSlice) Eq(other NibbleSlice) bool { - return ns.Len() == other.Len() && ns.StartsWith(other) -} - -// CommonPrefix return the amount of same nibbles at the beggining do we match with other -func (ns *NibbleSlice) CommonPrefix(other NibbleSlice) int { - selfAlign := ns.offset % NibblePerByte - otherAlign := other.offset % NibblePerByte - if selfAlign == otherAlign { - selfStart := ns.offset / NibblePerByte - otherStart := other.offset / NibblePerByte - first := 0 - if selfAlign != 0 { - if padRight(ns.data[selfStart]) != padRight(other.data[otherStart]) { - return 0 - } - selfStart++ - otherStart++ - first++ - } - return biggestDepth(ns.data[selfStart:], other.data[otherStart:]) + first - } - - s := minLength(ns.data, other.data) - i := 0 - for i < s { - if ns.At(i) != other.At(i) { - break - } - i++ - } - return i -} - -// Left returns left portion of `NibbleSlice` -// if the slice originates from a full key it will be the `Prefix of the node`. -func (ns *NibbleSlice) Left() Prefix { - split := ns.offset / NibblePerByte - ix := (ns.offset % NibblePerByte) - prefix := Prefix{ - PartialKey: ns.data[:split], - PaddedByte: nil, - } - if ix != 0 { - padded := padRight(ns.data[split]) - prefix.PaddedByte = &padded - } - - return prefix -} - -// OriginalDataAsPrefix gets `Prefix` representation of the inner data. -// This means the entire inner data will be returned as `Prefix`, ignoring any `offset`. -func (ns *NibbleSlice) OriginalDataAsPrefix() Prefix { - return Prefix{ - PartialKey: ns.data, - PaddedByte: nil, - } -} diff --git a/pkg/trie/triedb/nibble/nibbleslice_test.go b/pkg/trie/triedb/nibble/nibbleslice_test.go deleted file mode 100644 index 7cec57a8f4..0000000000 --- a/pkg/trie/triedb/nibble/nibbleslice_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package nibble - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -var testData = []byte{0x01, 0x23, 0x45} - -func Test_Basic(t *testing.T) { - t.Run("Nibble slice with offset 0", func(t *testing.T) { - n := NewNibbleSlice(testData) - - require.Equal(t, 6, n.Len()) - require.False(t, n.IsEmpty()) - - for i := 0; i < n.Len(); i++ { - require.Equal(t, byte(i), n.At(i)) - } - }) - - t.Run("Nibble slice with offset 6", func(t *testing.T) { - n := NewNibbleSliceWithOffset(testData, 6) - require.True(t, n.IsEmpty()) - }) - - t.Run("Nibble slice with offset 3", func(t *testing.T) { - n := NewNibbleSliceWithOffset(testData, 3) - require.Equal(t, 3, n.Len()) - for i := 0; i < n.Len(); i++ { - require.Equal(t, byte(i)+3, n.At(i)) - } - }) -} - -func Test_Mid(t *testing.T) { - n := NewNibbleSlice(testData) - t.Run("Mid 2", func(t *testing.T) { - m := n.Mid(2) - for i := 0; i < m.Len(); i++ { - require.Equal(t, byte(i)+2, m.At(i)) - } - }) - t.Run("Mid 3", func(t *testing.T) { - m := n.Mid(3) - for i := 0; i < m.Len(); i++ { - require.Equal(t, byte(i)+3, m.At(i)) - } - }) -} - -func Test_EncodedPre(t *testing.T) { - n := NewNibbleSlice(testData) - - t.Run("Mid 0", func(t *testing.T) { - m := n.Mid(0) - expected := NibbleSlice{ - data: []byte{0x01, 0x23, 0x45}, - offset: 0, - } - - require.Equal(t, expected, m.ToStored()) - }) - - t.Run("Mid 1", func(t *testing.T) { - m := n.Mid(1) - expected := NibbleSlice{ - data: []byte{0x01, 0x23, 0x45}, - offset: 1, - } - - require.Equal(t, expected, m.ToStored()) - }) - - t.Run("Mid 2", func(t *testing.T) { - m := n.Mid(2) - expected := NibbleSlice{ - data: []byte{0x23, 0x45}, - offset: 0, - } - - require.Equal(t, expected, m.ToStored()) - }) - - t.Run("Mid 3", func(t *testing.T) { - m := n.Mid(3) - expected := NibbleSlice{ - data: []byte{0x23, 0x45}, - offset: 1, - } - - require.Equal(t, expected, m.ToStored()) - }) -} - -func Test_Shared(t *testing.T) { - n := NewNibbleSlice(testData) - - other := []byte{0x01, 0x23, 0x01, 0x23, 0x45, 0x67} - m := NewNibbleSlice(other) - - require.Equal(t, 4, n.CommonPrefix(*m)) - require.Equal(t, 4, m.CommonPrefix(*n)) - require.Equal(t, 3, n.Mid(1).CommonPrefix(*m.Mid(1))) - require.Equal(t, 0, n.Mid(1).CommonPrefix(*m.Mid(2))) - require.Equal(t, 6, n.CommonPrefix(*m.Mid(4))) - require.True(t, m.Mid(4).StartsWith(*n)) -} - -func Test_Compare(t *testing.T) { - other := []byte{0x01, 0x23, 0x01, 0x23, 0x45} - n := NewNibbleSlice(testData) - m := NewNibbleSlice(other) - - require.False(t, n.Eq(*m)) - require.True(t, n.Eq(*m.Mid(4))) -} diff --git a/pkg/trie/triedb/node/node.go b/pkg/trie/triedb/node/node.go deleted file mode 100644 index cd88546279..0000000000 --- a/pkg/trie/triedb/node/node.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package node - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -) - -// Value -type Value interface { - Type() string -} - -type ( - // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value - InlineValue struct { - Bytes []byte - } - // HashedNodeValue is a trie node pointer to a hashed node - NodeValue struct { - Bytes []byte - } -) - -func (v InlineValue) Type() string { return "Inline" } -func (v NodeValue) Type() string { return "Node" } - -// Nodes -type Node interface { - Type() string -} - -type ( - // NodeEmptyNode represents an empty node - Empty struct{} - // NodeLeaf represents a leaf node - Leaf struct { - PartialKey nibble.NibbleSlice - Value Value - } - // NodeNibbledBranch represents a branch node - NibbledBranch struct { - PartialKey nibble.NibbleSlice - Children [nibble.NibbleLength]NodeHandle - Value Value - } -) - -func (n Empty) Type() string { return "Empty" } -func (n Leaf) Type() string { return "Leaf" } -func (n NibbledBranch) Type() string { return "NibbledBranch" } - -// NodeOwned is a trie node -type NodeOwned[H hashdb.HashOut] interface { - Type() string -} - -type ( - // NodeEmptyNode represents an empty node - NodeOwnedEmpty struct{} - // NodeLeaf represents a leaf node - NodeOwnedLeaf[H hashdb.HashOut] struct { - PartialKey nibble.NibbleSlice - Value ValueOwned[H] - } - // NodeNibbledBranch represents a branch node - NodeOwnedNibbledBranch[H hashdb.HashOut] struct { - PartialKey nibble.NibbleSlice - EncodedChildren [nibble.NibbleLength]NodeHandleOwned[H] - Value ValueOwned[H] - } -) - -func (n NodeOwnedEmpty) Type() string { return "Empty" } -func (n NodeOwnedLeaf[H]) Type() string { return "Leaf" } -func (n NodeOwnedNibbledBranch[H]) Type() string { return "NibbledBranch" } - -// Value is a trie node value -type ValueOwned[H hashdb.HashOut] interface { - Type() string - AsValue() Value -} -type ( - // InlineNodeValue if the value is inlined we can get the bytes and the hash of the value - InlineValueOwned[H hashdb.HashOut] struct { - bytes []byte - hash H - } - // HashedNodeValue is a trie node pointer to a hashed node - NodeValueOwned[H hashdb.HashOut] struct { - hash H - } -) - -func (v InlineValueOwned[H]) Type() string { return "Inline" } -func (v InlineValueOwned[H]) AsValue() Value { - return InlineValue{Bytes: v.bytes} -} -func (v NodeValueOwned[H]) Type() string { return "Node" } -func (v NodeValueOwned[H]) AsValue() Value { - return NodeValue{Bytes: v.hash.Bytes()} -} diff --git a/pkg/trie/triedb/node/node_codec.go b/pkg/trie/triedb/node/node_codec.go deleted file mode 100644 index f50a5001b6..0000000000 --- a/pkg/trie/triedb/node/node_codec.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package node - -import ( - "fmt" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -) - -type ChildReference[H hashdb.HashOut] interface { - Type() string -} - -type ( - ChildReferenceHash[H hashdb.HashOut] struct { - hash H - } - ChildReferenceInline[H hashdb.HashOut] struct { - hash H - length uint - } -) - -func (c ChildReferenceHash[H]) Type() string { return "Hash" } -func (c ChildReferenceInline[H]) Type() string { return "Inline" } - -type NodeCodec[H hashdb.HashOut] interface { - HashedNullNode() H - Hasher() hashdb.Hasher[H] - EmptyNode() []byte - LeafNode(partialKey nibble.NibbleSlice, numberNibble int, value Value) []byte - BranchNodeNibbled( - partialKey nibble.NibbleSlice, - numberNibble int, - children [16]ChildReference[H], - value *Value, - ) []byte - Decode(data []byte) (Node, error) -} - -func EncodeNodeOwned[H hashdb.HashOut](node NodeOwned[H], codec NodeCodec[H]) []byte { - switch n := node.(type) { - case NodeOwnedEmpty: - return codec.EmptyNode() - case NodeOwnedLeaf[H]: - return codec.LeafNode(n.PartialKey, n.PartialKey.Len(), n.Value) - case NodeOwnedNibbledBranch[H]: - var value = n.Value.AsValue() - - var children [16]ChildReference[H] - for i, c := range n.EncodedChildren { - if c != nil { - children[i] = c.AsChildReference(codec) - } - } - return codec.BranchNodeNibbled(n.PartialKey, n.PartialKey.Len(), children, &value) - default: - panic(fmt.Sprintf("unknown node type %s", n.Type())) - } -} diff --git a/pkg/trie/triedb/node/node_handle.go b/pkg/trie/triedb/node/node_handle.go deleted file mode 100644 index e6e68a4eed..0000000000 --- a/pkg/trie/triedb/node/node_handle.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package node - -import ( - "errors" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" -) - -// NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandleOwned[H hashdb.HashOut] interface { - Type() string - AsChildReference(codec NodeCodec[H]) ChildReference[H] -} -type ( - NodeHandleOwnedHash[H hashdb.HashOut] struct { - Hash H - } - NodeHandleOwnedInline[H hashdb.HashOut] struct { - Node NodeOwned[H] - } -) - -func (h NodeHandleOwnedHash[H]) Type() string { return "Hash" } -func (h NodeHandleOwnedHash[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - return ChildReferenceHash[H]{hash: h.Hash} -} -func (h NodeHandleOwnedInline[H]) Type() string { return "Inline" } -func (h NodeHandleOwnedInline[H]) AsChildReference(codec NodeCodec[H]) ChildReference[H] { - encoded := EncodeNodeOwned(h.Node, codec) - if len(encoded) > codec.Hasher().Length() { - panic("Invalid inline node handle") - } - return ChildReferenceInline[H]{hash: codec.Hasher().FromBytes(encoded), length: uint(len(encoded))} -} - -// NodeHandle is a reference to a trie node which may be stored within another trie node. -type NodeHandle interface { - Type() string -} -type ( - Hash struct { - Data []byte - } - Inline struct { - Data []byte - } -) - -func (h Hash) Type() string { return "Hash" } -func (h Inline) Type() string { return "Inline" } - -func DecodeHash[H hashdb.HashOut](hasher hashdb.Hasher[H], data []byte) (H, error) { - if len(data) != hasher.Length() { - return hasher.FromBytes([]byte{}), errors.New("decoding hash") - } - return hasher.FromBytes(data), nil -} diff --git a/pkg/trie/triedb/node/node_plan.go b/pkg/trie/triedb/node/node_plan.go deleted file mode 100644 index 25f87f9647..0000000000 --- a/pkg/trie/triedb/node/node_plan.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package node - -import "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - -type BytesRange struct { - Start int - End int -} - -// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice -type NibbleSlicePlan struct { - bytes BytesRange - offset int -} - -func NewNibbleSlicePlan(bytes BytesRange, offset int) NibbleSlicePlan { - return NibbleSlicePlan{bytes, offset} -} - -func (self NibbleSlicePlan) Len() int { - return (self.bytes.End-self.bytes.Start)*nibble.NibblePerByte - self.offset -} - -func (self NibbleSlicePlan) Build(data []byte) nibble.NibbleSlice { - return *nibble.NewNibbleSliceWithOffset(data[self.bytes.Start:self.bytes.End], self.offset) -} - -// A `NodeHandlePlan` is a decoding plan for constructing a `NodeHandle` from an encoded trie -type NodeHandlePlan interface { - Type() string - Build(data []byte) NodeHandle -} - -type ( - NodeHandlePlanHash struct { - bytes BytesRange - } - NodeHandlePlanInline struct { - bytes BytesRange - } -) - -func (NodeHandlePlanHash) Type() string { return "Hash" } -func (n NodeHandlePlanHash) Build(data []byte) NodeHandle { - return Hash{data[n.bytes.Start:n.bytes.End]} -} -func (NodeHandlePlanInline) Type() string { return "Inline" } -func (n NodeHandlePlanInline) Build(data []byte) NodeHandle { - return Inline{data[n.bytes.Start:n.bytes.End]} -} - -// Plan for value representation in `NodePlan`. -type ValuePlan interface { - Type() string - Build(data []byte) Value -} - -type ( - // Range for byte representation in encoded node. - ValuePlanInline struct { - bytes BytesRange - } - // Range for hash in encoded node and original - ValuePlanNode struct { - bytes BytesRange - } -) - -func (ValuePlanInline) Type() string { return "Inline" } -func (n ValuePlanInline) Build(data []byte) Value { - return InlineValue{data[n.bytes.Start:n.bytes.End]} -} -func (ValuePlanNode) Type() string { return "Node" } -func (n ValuePlanNode) Build(data []byte) Value { - return NodeValue{data[n.bytes.Start:n.bytes.End]} -} - -type NodePlan interface { - Type() string - Build(data []byte) Node -} - -type ( - // Null trie node; could be an empty root or an empty branch entry - NodePlanEmptyNode struct{} - // Leaf node, has a partial key plan and value - NodePlanLeafNode struct { - partial NibbleSlicePlan - value ValuePlan - } - // Branch node - NodePlanNibbledBranchNode struct { - partial NibbleSlicePlan - value ValuePlan - children [nibble.NibbleLength]NodeHandlePlan - } -) - -func (NodePlanEmptyNode) Type() string { return "Empty" } -func (self NodePlanEmptyNode) Build(data []byte) Node { - return Empty{} -} -func (NodePlanLeafNode) Type() string { return "Leaf" } -func (self NodePlanLeafNode) Build(data []byte) Node { - return Leaf{ - PartialKey: self.partial.Build(data), - Value: self.value.Build(data), - } -} -func (NodePlanNibbledBranchNode) Type() string { return "NibbledBranch" } -func (self NodePlanNibbledBranchNode) Build(data []byte) Node { - children := [nibble.NibbleLength]NodeHandle{} - for i := 0; i < nibble.NibbleLength; i++ { - if self.children[i] != nil { - children[i] = self.children[i].Build(data) - } - } - var value Value - if self.value != nil { - value = self.value.Build(data) - } - return NibbledBranch{ - PartialKey: self.partial.Build(data), - Children: children, - Value: value, - } -} diff --git a/pkg/trie/triedb/recorder.go b/pkg/trie/triedb/recorder.go deleted file mode 100644 index d5a3b6d476..0000000000 --- a/pkg/trie/triedb/recorder.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "fmt" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" -) - -type RecordedForKey uint8 - -const ( - RecordedForKeyValue RecordedForKey = iota - RecordedForKeyHash - RecordedForKeyNone -) - -type TrieRecorder[Out hashdb.HashOut] interface { - record(access TrieAccess[Out]) - trieNodesRecordedForKey(key []byte) RecordedForKey -} - -// TrieAccess is used to report the trie access to the TrieRecorder -type TrieAccess[Out hashdb.HashOut] interface { - Type() string -} - -type ( - // TrieAccessNode means that the given node was accessed using its hash - TrieAccessNodeOwned[H hashdb.HashOut] struct { - hash H - nodeOwned node.NodeOwned[H] - } - - // TrieAccessEncodedNode means that the given encodedNode was accessed using its hash - TrieAccessEncodedNode[H hashdb.HashOut] struct { - hash H - encodedNode []byte - } - - // TrieAccessValue means that the given value was accessed using its hash - // fullKey is the key to access this value in the trie - // Should map to RecordedForKeyValue when checking the recorder - TrieAccessValue[H hashdb.HashOut] struct { - hash H - value []byte - fullKey []byte - } - - // TrieAccessInlineValue means that a value stored in an inlined node was accessed - // The given fullKey is the key to access this value in the trie - // Should map to RecordedForKeyValue when checking the recorder - TrieAccessInlineValue[H hashdb.HashOut] struct { - fullKey []byte - } - - // TrieAccessHash means that the hash of the value for a given fullKey was accessed - // Should map to RecordedForKeyHash when checking the recorder - TrieAccessHash[H hashdb.HashOut] struct { - fullKey []byte - } - - // TrieAccessNonExisting means that the value/hash for fullKey was accessed, but it couldn't be found in the trie - // Should map to RecordedForKeyValue when checking the recorder - TrieAccessNonExisting struct { - fullKey []byte - } -) - -func (a TrieAccessNodeOwned[H]) Type() string { return "Node" } -func (a TrieAccessEncodedNode[H]) Type() string { return "EncodedNode" } -func (a TrieAccessValue[H]) Type() string { return "Value" } -func (a TrieAccessInlineValue[H]) Type() string { return "InlineValue" } -func (a TrieAccessHash[H]) Type() string { return "Hash" } -func (a TrieAccessNonExisting) Type() string { return "NotExisting" } - -// Recorder implementation - -type Record[H hashdb.HashOut] struct { - /// The hash of the node. - Hash H - /// The data representing the node. - Data []byte -} - -type Recorder[H hashdb.HashOut] struct { - nodes []Record[H] - recorderKeys map[string]RecordedForKey // TODO: revisit this later, it should be a BTreeMap - layout TrieLayout[H] -} - -// NewRecorder creates a new Recorder which records all given nodes -func NewRecorder[H hashdb.HashOut]() *Recorder[H] { - return &Recorder[H]{ - nodes: make([]Record[H], 0), - recorderKeys: make(map[string]RecordedForKey), - } -} - -// Drain drains all visited records -func (r *Recorder[H]) Drain() []Record[H] { - // Store temporal nodes - nodes := make([]Record[H], len(r.nodes)) - copy(nodes, r.nodes) - - // Clean up internal data and return the nodes - clear(r.nodes) - clear(r.recorderKeys) - - return nodes -} - -// Impl of TrieRecorder for Recorder -func (r *Recorder[H]) record(access TrieAccess[H]) { - switch access := access.(type) { - case TrieAccessEncodedNode[H]: - r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.encodedNode}) - case TrieAccessNodeOwned[H]: - r.nodes = append( - r.nodes, - Record[H]{ - Hash: access.hash, - Data: node.EncodeNodeOwned(access.nodeOwned, r.layout.Codec()), - }, - ) - case TrieAccessValue[H]: - r.nodes = append(r.nodes, Record[H]{Hash: access.hash, Data: access.value}) - r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue - case TrieAccessHash[H]: - if _, inserted := r.recorderKeys[string(access.fullKey)]; !inserted { - r.recorderKeys[string(access.fullKey)] = RecordedForKeyHash - } - case TrieAccessNonExisting: - // We handle the non existing value/hash like having recorded the value. - r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue - case TrieAccessInlineValue[H]: - r.recorderKeys[string(access.fullKey)] = RecordedForKeyValue - default: - panic(fmt.Sprintf("unknown access type %s", access.Type())) - } -} diff --git a/pkg/trie/triedb/trie.go b/pkg/trie/triedb/trie.go deleted file mode 100644 index 0c526c64e1..0000000000 --- a/pkg/trie/triedb/trie.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" -) - -type TrieValue interface { - Type() string -} - -type ( - InlineTrieValue struct { - Bytes []byte - } - NodeTrieValue[H hashdb.HashOut] struct { - Hash H - } - NewNodeTrie[H hashdb.HashOut] struct { - Hash *H - Bytes []byte - } -) - -func (v InlineTrieValue) Type() string { return "Inline" } -func (v NodeTrieValue[H]) Type() string { return "Node" } -func (v NewNodeTrie[H]) Type() string { return "NewNode" } - -func NewTrieValueFromBytes[H HashOut](value []byte, threshold *uint) TrieValue { - if threshold != nil && uint(len(value)) >= *threshold { - return NewNodeTrie[H]{nil, value} - } else { - return InlineTrieValue{Bytes: value} - } -} - -type Trie[Hash hashdb.HashOut] interface { - Root() Hash - IsEmpty() bool - Contains(key []byte) (bool, error) - GetHash(key []byte) (*Hash, error) - Get(key []byte) (*DBValue, error) - //TODO: - //get_with - //lookup_first_descendant -} - -type MutableTrie[Hash hashdb.HashOut] interface { - insert(key []byte, value []byte) (*TrieValue, error) - remove(key []byte) (*TrieValue, error) -} diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go deleted file mode 100644 index 7e63d9f7f8..0000000000 --- a/pkg/trie/triedb/triedb.go +++ /dev/null @@ -1,794 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "errors" - - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" - encoded_nodes "github.com/ChainSafe/gossamer/pkg/trie/triedb/node" - "github.com/gammazero/deque" -) - -type HashOut = hashdb.HashOut -type Children = [nibble.NibbleLength]NodeHandle - -// Node types in the Trie -type Node[H HashOut] interface { - Type() string -} - -type ( - // NodeEmptyNode represents an empty node - Empty struct{} - // NodeLeaf represents a leaf node - Leaf struct { - encoded nibble.NibbleSlice - value TrieValue - } - // NodeNibbledBranch represents a branch node - NibbledBranch struct { - encoded nibble.NibbleSlice - children Children - value TrieValue - } -) - -func (n Empty) Type() string { return "Empty" } -func (n Leaf) Type() string { return "Leaf" } -func (n NibbledBranch) Type() string { return "NibbledBranch" } - -type StorageHandle = uint -type NibbleFullKey = nibble.NibbleSlice - -type Stored[H HashOut] interface { - Type() string -} - -type ( - StoredNew[H HashOut] struct { - Node Node[H] - } - StoredCached[H HashOut] struct { - Hash H - Node Node[H] - } -) - -func (s StoredNew[Out]) Type() string { return "New" } -func (s StoredCached[Out]) Type() string { return "Cached" } - -type NodeHandle interface { - Type() string -} -type ( - Hash[H HashOut] struct { - Value H - } - InMemory struct { - Value StorageHandle - } -) - -func (h Hash[H]) Type() string { return "Hash" } -func (h InMemory) Type() string { return "InMemory" } - -// Compact storage for tree nodes -type NodeStorage[H HashOut] struct { - nodes []Stored[H] - freeIndices deque.Deque[uint] -} - -func NewEmptyNodeStorage[H HashOut]() NodeStorage[H] { - return NodeStorage[H]{ - nodes: make([]Stored[H], 0), - } -} - -func (ns *NodeStorage[H]) alloc(stored Stored[H]) StorageHandle { - if ns.freeIndices.Len() > 0 { - idx := ns.freeIndices.PopFront() - ns.nodes[idx] = stored - return idx - } - - ns.nodes = append(ns.nodes, stored) - return StorageHandle(len(ns.nodes) - 1) -} - -func (ns *NodeStorage[H]) destroy(handle StorageHandle) Stored[H] { - idx := handle - - ns.freeIndices.PushBack(idx) - ns.nodes[idx] = StoredNew[H]{Empty{}} - return ns.nodes[idx] -} - -type TrieDB[Out HashOut] struct { - storage NodeStorage[Out] - db hashdb.HashDB[Out, DBValue] - root Out - rootHandle NodeHandle - deathRow map[string]nibble.Prefix - cache TrieCache[Out] - recorder TrieRecorder[Out] - layout TrieLayout[Out] -} - -type RemoveAtResult struct { - handle StorageHandle - changed bool -} - -func (tdb *TrieDB[H]) record( - access TrieAccess[H], -) { - if tdb.recorder != nil { - tdb.recorder.record(access) - } -} - -func (tdb *TrieDB[H]) getAndCache( - hash H, - key nibble.Prefix, -) (StorageHandle, error) { - // We only check the `cache` for a node with `get_node` and don't insert - // the node if it wasn't there, because in substrate we only access the node while computing - // a new trie (aka some branch). We assume that this node isn't that important to have it being cached. - var node Node[H] - - hasCache := tdb.cache != nil - - if hasCache { - nodeFromCache := tdb.cache.GetNode(hash) - if nodeFromCache != nil { - tdb.record(TrieAccessNodeOwned[H]{hash: hash, nodeOwned: *nodeFromCache}) - node = NodeFromNodeOwned(*nodeFromCache, tdb.storage) - } - } - - if node == nil { - nodeEncoded := tdb.db.Get(hash, key) - if nodeEncoded == nil { - return 0, ErrIncompleteDB - } - - tdb.record(TrieAccessEncodedNode[H]{hash: hash, encodedNode: *nodeEncoded}) - - var err error - node, err = NodeFromEncoded(hash, *nodeEncoded, tdb.storage, tdb.layout) - if err != nil { - return 0, err - } - } - - return tdb.storage.alloc(StoredCached[H]{Node: node, Hash: hash}), nil -} - -type PostInspectAction interface { - Type() string -} - -type ( - PostInspectActionReplace[H HashOut] struct { - node Node[H] - } - PostInspectActionRestore[H HashOut] struct { - node Node[H] - } - PostInspectActionDelete struct{} -) - -func (r PostInspectActionReplace[H]) Type() string { return "Replace" } -func (r PostInspectActionRestore[H]) Type() string { return "Restore" } -func (r PostInspectActionDelete) Type() string { return "Delete" } - -type InspectResult[H HashOut] struct { - stored Stored[H] - changed bool -} - -func (tdb *TrieDB[H]) inspect( - stored Stored[H], - key NibbleFullKey, - inspector func( - node Node[H], - key NibbleFullKey, - ) (PostInspectAction, error), -) (*InspectResult[H], error) { - var result InspectResult[H] - - switch s := stored.(type) { - case StoredNew[H]: - execution, err := inspector(s.Node, key) - if err != nil { - return nil, err - } - switch action := execution.(type) { - case PostInspectActionRestore[H]: - result = InspectResult[H]{StoredNew[H]{action.node}, false} - case PostInspectActionReplace[H]: - result = InspectResult[H]{StoredNew[H]{action.node}, true} - case PostInspectActionDelete: - return nil, nil - } - case StoredCached[H]: - execution, err := inspector(s.Node, key) - if err != nil { - return nil, err - } - - switch action := execution.(type) { - case PostInspectActionRestore[H]: - result = InspectResult[H]{StoredCached[H]{s.Hash, action.node}, false} - case PostInspectActionReplace[H]: - tdb.deathRow[s.Hash.ComparableKey()] = key.Left() - result = InspectResult[H]{StoredNew[H]{action.node}, true} - case PostInspectActionDelete: - tdb.deathRow[s.Hash.ComparableKey()] = key.Left() - return nil, nil - } - } - - return &result, nil -} - -// Given a node which may be in an invalid state, fix it such that it is then in a valid -// state. -// -// invalid state means: -// - Branch node where there is only a single entry; -func (tdb *TrieDB[H]) fix(node Node[H], key nibble.NibbleSlice) (Node[H], error) { - switch n := node.(type) { - case NibbledBranch: - var ix byte - usedIndexes := 0 - for i := 0; i < 16; i++ { - if n.children[i] != nil { - ix = byte(i) - usedIndexes += 1 - } - } - if n.value == nil { - if usedIndexes == 0 { - panic("Branch with no subvalues. Something went wrong.") - } - if usedIndexes == 1 { - // only one onward node. use child instead - child := n.children[ix] - if child == nil { - return nil, errors.New("used_index only set if occupied") - } - key2 := key.Clone() - offset := len(n.encoded.Data())*nibble.NibblePerByte - n.encoded.Offset() - key2.Advance(offset) - prefix := key2.Left() - - start := prefix.PartialKey - var allocStart *[]byte - var prefixEnd *byte - if prefix.PaddedByte == nil { - allocStart = nil - pEnd := nibble.PushAtLeft(0, ix, 0) - prefixEnd = &pEnd - } else { - so := prefix.PartialKey - so = append(so, nibble.PadLeft(*prefix.PaddedByte)|ix) - allocStart = &so - prefixEnd = nil - } - childPrefix := nibble.Prefix{} - if allocStart != nil { - childPrefix.PartialKey = *allocStart - } else { - childPrefix.PartialKey = start - } - childPrefix.PaddedByte = prefixEnd - var stored Stored[H] - - switch c := child.(type) { - case InMemory: - stored = tdb.storage.destroy(c.Value) - case Hash[H]: - handle, err := tdb.getAndCache(c.Value, childPrefix) - if err != nil { - return nil, err - } - stored = tdb.storage.destroy(handle) - } - - var childNode Node[H] - - switch s := stored.(type) { - case StoredNew[H]: - childNode = s.Node - case StoredCached[H]: - tdb.deathRow[s.Hash.ComparableKey()] = childPrefix - childNode = s.Node - } - - switch cn := childNode.(type) { - case Leaf: - encNibble := n.encoded.Clone() - end := nibble.NewNibbleSliceWithOffset([]byte{ix}, nibble.NibblePerByte-1) - nibble.CombineKeys(encNibble, *end) - - end = nibble.NewNibbleSliceWithOffset(cn.encoded.Data(), cn.encoded.Offset()) - nibble.CombineKeys(encNibble, *end) - return Leaf{*encNibble, cn.value}, nil - case NibbledBranch: - encNibble := n.encoded.Clone() - end := nibble.NewNibbleSliceWithOffset([]byte{ix}, nibble.NibblePerByte-1) - nibble.CombineKeys(encNibble, *end) - end = nibble.NewNibbleSliceWithOffset(cn.encoded.Data(), cn.encoded.Offset()) - nibble.CombineKeys(encNibble, *end) - return NibbledBranch{*encNibble, cn.children, cn.value}, nil - default: - panic("Unreachable") - } - } - } - if n.value != nil { - if usedIndexes == 0 { - // Make a lift - // Fixing branch -> Leaf - return Leaf{n.encoded, n.value}, nil - } - } - // All is well, restoring branch - return NibbledBranch{n.encoded, n.children, n.value}, nil - default: - return node, nil - } -} - -func (tdb *TrieDB[H]) removeInspector( - node Node[H], - key NibbleFullKey, - oldVal *TrieValue, -) (PostInspectAction, error) { - switch n := node.(type) { - case Empty: - return PostInspectActionDelete{}, nil - case Leaf: - existingKey := n.encoded - if key.Eq(existingKey) { - // We found the node we want to delete, so we are going to remove it - keyVal := key.Clone() - keyVal.Advance(existingKey.Len()) - tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) - return PostInspectActionDelete{}, nil - } else { - // Leaf the node alone, restoring leaf wrong partial - return PostInspectActionRestore[H]{ - Leaf{n.encoded, n.value}, - }, nil - } - case NibbledBranch: - if key.IsEmpty() { - if n.value == nil { - return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, nil}}, nil - } - tdb.replaceOldValue(oldVal, n.value, key.Left()) - fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, nil}, key) - if err != nil { - return nil, err - } - return PostInspectActionReplace[H]{fixedNode}, nil - } - common := n.encoded.CommonPrefix(key) - existingLength := n.encoded.Len() - - if common == existingLength && common == key.Len() { - // Replace val - if n.value != nil { - keyVal := key.Clone() - keyVal.Advance(existingLength) - tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) - fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, nil}, key) - if err != nil { - return nil, err - } - return PostInspectActionReplace[H]{fixedNode}, nil - } - return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, nil}}, nil - } else if common < existingLength { - // Nothing to do here - return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, n.value}}, nil - } else { - // common == existing_length && common < partial.len() : check children - idx := key.At(common) - - child := n.children[idx] - if child != nil { - key.Advance(common + 1) - res, err := tdb.removeAt(child, key, oldVal) - if err != nil { - return nil, err - } - - if res != nil { - n.children[idx] = InMemory{res.handle} - branch := NibbledBranch{n.encoded, n.children, n.value} - if res.changed { - return PostInspectActionReplace[H]{branch}, nil - } else { - return PostInspectActionRestore[H]{branch}, nil - } - } - fixedNode, err := tdb.fix(NibbledBranch{n.encoded, n.children, n.value}, key) - if err != nil { - return nil, err - } - return PostInspectActionReplace[H]{fixedNode}, nil - } - return PostInspectActionRestore[H]{NibbledBranch{n.encoded, n.children, n.value}}, nil - } - default: - panic("Invalid node type") - } -} - -func (tdb *TrieDB[H]) insertInspector( - node Node[H], - key NibbleFullKey, - value []byte, - oldVal *TrieValue, -) (PostInspectAction, error) { - partial := key - - switch n := node.(type) { - case Empty: - value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) - return PostInspectActionReplace[H]{Leaf{partial, value}}, nil - case Leaf: - existingKey := n.encoded - common := partial.CommonPrefix(existingKey) - if common == existingKey.Len() && common == partial.Len() { - // Equivalent leaf replace - value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) - unchanged := n.value == value - keyVal := key.Clone() - keyVal.Advance(existingKey.Len()) - tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) - if unchanged { - return PostInspectActionRestore[H]{Leaf{n.encoded, n.value}}, nil - } - return PostInspectActionReplace[H]{Leaf{n.encoded, n.value}}, nil - } else if common < existingKey.Len() { - // one of us isn't empty: transmute to branch here - children := Children{} - idx := existingKey.At(common) - newLeaf := Leaf{*existingKey.Mid(common + 1), n.value} - children[idx] = InMemory{tdb.storage.alloc(StoredNew[H]{newLeaf})} - branch := NibbledBranch{partial.ToStoredRange(common), children, nil} - - branchAction, err := tdb.insertInspector(branch, key, value, oldVal) - if err != nil { - return nil, err - } - return PostInspectActionReplace[H]{branchAction}, nil - } else { - branch := NibbledBranch{ - existingKey.ToStored(), - Children{}, - n.value, - } - - branchAction, err := tdb.insertInspector(branch, key, value, oldVal) - if err != nil { - return nil, err - } - return PostInspectActionReplace[H]{branchAction}, nil - } - case NibbledBranch: - existingKey := n.encoded - common := partial.CommonPrefix(existingKey) - if common == existingKey.Len() && common == partial.Len() { - value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) - unchanged := n.value == value - branch := NibbledBranch{n.encoded, n.children, n.value} - - keyVal := key.Clone() - keyVal.Advance(existingKey.Len()) - tdb.replaceOldValue(oldVal, n.value, keyVal.Left()) - - if unchanged { - return PostInspectActionRestore[H]{branch}, nil - } - return PostInspectActionReplace[H]{branch}, nil - } else if common < existingKey.Len() { - nbranchPartial := existingKey.Mid(common + 1).ToStored() - low := NibbledBranch{nbranchPartial, n.children, n.value} - ix := existingKey.At(common) - children := Children{} - allocStorage := tdb.storage.alloc(StoredNew[H]{low}) - - children[ix] = InMemory{allocStorage} - value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) - - if partial.Len()-common == 0 { - return PostInspectActionReplace[H]{ - NibbledBranch{ - existingKey.ToStoredRange(common), - children, - value, - }, - }, nil - } else { - ix := partial.At(common) - storedLeaf := Leaf{partial.Mid(common + 1).ToStored(), value} - leaf := tdb.storage.alloc(StoredNew[H]{storedLeaf}) - - children[ix] = InMemory{leaf} - return PostInspectActionReplace[H]{ - NibbledBranch{ - existingKey.ToStoredRange(common), - children, - nil, - }, - }, nil - } - } else { - // Append after common == existing_key and partial > common - idx := partial.At(common) - key.Advance(common + 1) - child := n.children[idx] - if child != nil { - // Original had something there. recurse down into it. - res, err := tdb.insertAt(child, key, value, oldVal) - if err != nil { - return nil, err - } - n.children[idx] = InMemory{res.handle} - if !res.changed { - // The new node we composed didn't change. - // It means our branch is untouched too. - nBranch := NibbledBranch{n.encoded.ToStored(), n.children, n.value} - return PostInspectActionRestore[H]{nBranch}, nil - } - } else { - // Original had nothing there. compose a leaf. - value := NewTrieValueFromBytes[H](value, tdb.layout.MaxInlineValue()) - leaf := tdb.storage.alloc(StoredNew[H]{Leaf{key.ToStored(), value}}) - n.children[idx] = InMemory{leaf} - } - return PostInspectActionReplace[H]{NibbledBranch{existingKey.ToStored(), n.children, n.value}}, nil - } - default: - panic("Invalid node type") - } -} - -func (tdb *TrieDB[H]) replaceOldValue( - oldVal *TrieValue, - storedValue TrieValue, - prefix nibble.Prefix, -) { - switch n := storedValue.(type) { - case NewNodeTrie[H]: - if n.Hash != nil { - tdb.deathRow[(*n.Hash).ComparableKey()] = prefix - } - case NodeTrieValue[H]: - tdb.deathRow[n.Hash.ComparableKey()] = prefix - } - *oldVal = storedValue -} - -// Removes a node from the trie based on key -func (tdb *TrieDB[H]) removeAt( - handle NodeHandle, - key NibbleFullKey, - oldVal *TrieValue, -) (*RemoveAtResult, error) { - var stored Stored[H] - - switch h := handle.(type) { - case InMemory: - stored = tdb.storage.destroy(h.Value) - case Hash[H]: - fromCache, err := tdb.getAndCache(h.Value, key.Left()) - if err != nil { - return nil, err - } - stored = tdb.storage.destroy(fromCache) - } - - res, err := tdb.inspect(stored, key, func(node Node[H], key NibbleFullKey) (PostInspectAction, error) { - return tdb.removeInspector(node, key, oldVal) - }) - - if err != nil { - return nil, err - } - - return &RemoveAtResult{ - tdb.storage.alloc(res.stored), - res.changed, - }, nil -} - -func (tdb *TrieDB[H]) Remove(key []byte) (*TrieValue, error) { - rootHandle := tdb.rootHandle - keySlice := nibble.NewNibbleSlice(key) - var oldVal *TrieValue - - res, err := tdb.removeAt(rootHandle, *keySlice, oldVal) - - if err != nil { - return nil, err - } - - if res != nil { - tdb.rootHandle = InMemory{res.handle} - } else { - tdb.rootHandle = Hash[H]{tdb.layout.Codec().HashedNullNode()} - tdb.root = tdb.layout.Codec().HashedNullNode() - } - - return oldVal, nil -} - -type InsertAtResult struct { - handle StorageHandle - changed bool -} - -// / Insert a key-value pair into the trie, creating new nodes if necessary. -func (tdb *TrieDB[H]) insertAt( - handle NodeHandle, - key NibbleFullKey, - value []byte, - oldVal *TrieValue, -) (InsertAtResult, error) { - var storageHandle StorageHandle - var err error - - switch h := handle.(type) { - case InMemory: - storageHandle = h.Value - case Hash[H]: - storageHandle, err = tdb.getAndCache(h.Value, key.Left()) - if err != nil { - return InsertAtResult{}, err - } - } - - stored := tdb.storage.destroy(storageHandle) - - res, err := tdb.inspect(stored, key, func(node Node[H], key NibbleFullKey) (PostInspectAction, error) { - return tdb.insertInspector(node, key, value, oldVal) - }) - - if err != nil { - return InsertAtResult{}, err - } - - return InsertAtResult{ - tdb.storage.alloc(res.stored), - res.changed, - }, nil -} - -func (tdb *TrieDB[H]) Insert(key []byte, value []byte) (*TrieValue, error) { - if !tdb.layout.AllowEmpty() && len(value) == 0 { - return tdb.Remove(key) - } - - var oldVal *TrieValue - - insertRes, err := tdb.insertAt(tdb.rootHandle, *nibble.NewNibbleSlice(key), value, oldVal) - - if err != nil { - return nil, err - } - - tdb.rootHandle = InMemory{insertRes.handle} - - return oldVal, nil -} - -func inlineOrHashOwned[H HashOut](child encoded_nodes.NodeHandleOwned[H], storage NodeStorage[H]) NodeHandle { - switch n := child.(type) { - case encoded_nodes.NodeHandleOwnedHash[H]: - return Hash[H]{n.Hash} - case encoded_nodes.NodeHandleOwnedInline[H]: - child := NodeFromNodeOwned(n.Node, storage) - return InMemory{storage.alloc(StoredNew[H]{child})} - default: - panic("Invalid child") - } -} - -func inlineOrHash[H HashOut](parentHash H, child encoded_nodes.NodeHandle, storage NodeStorage[H], layout TrieLayout[H]) (NodeHandle, error) { - switch n := child.(type) { - case encoded_nodes.Hash: - hash, err := encoded_nodes.DecodeHash[H](layout.Codec().Hasher(), n.Data) - if err != nil { - return nil, errors.New("invalid hash") - } - return Hash[H]{hash}, nil - case encoded_nodes.Inline: - child, err := NodeFromEncoded(parentHash, n.Data, storage, layout) - if err != nil { - return nil, err - } - return InMemory{storage.alloc(StoredNew[H]{child})}, nil - default: - panic("Invalid child") - } -} - -func NodeFromNodeOwned[H HashOut](nodeOwned encoded_nodes.NodeOwned[H], storage NodeStorage[H]) Node[H] { - switch node := nodeOwned.(type) { - case encoded_nodes.NodeOwnedEmpty: - return Empty{} - case encoded_nodes.NodeOwnedLeaf[H]: - return Leaf{ - encoded: node.PartialKey, - value: node.Value, - } - case encoded_nodes.NodeOwnedNibbledBranch[H]: - child := func(i uint) NodeHandle { - if node.EncodedChildren[i] != nil { - return inlineOrHashOwned(node.EncodedChildren[i], storage) - } - return nil - } - - var children [16]NodeHandle - for i := uint(0); i < 16; i++ { - children[i] = child(i) - } - - return NibbledBranch{ - encoded: node.PartialKey, - children: children, - value: node.Value, - } - default: - panic("Invalid node") - } -} - -func NodeFromEncoded[H HashOut](nodeHash H, data []byte, storage NodeStorage[H], layout TrieLayout[H]) (Node[H], error) { - decodedNode, err := layout.Codec().Decode(data) - if err != nil { - return nil, errors.New("decoding node") - } - switch node := decodedNode.(type) { - case encoded_nodes.Empty: - return Empty{}, nil - case encoded_nodes.Leaf: - return Leaf{ - encoded: node.PartialKey, - value: node.Value, - }, nil - case encoded_nodes.NibbledBranch: - child := func(i uint) (NodeHandle, error) { - if node.Children[i] != nil { - return inlineOrHash[H](nodeHash, node.Children[i], storage, layout) - } - return nil, nil - } - - var children [16]NodeHandle - for i := uint(0); i < 16; i++ { - children[i], err = child(i) - if err != nil { - return nil, err - } - } - - return NibbledBranch{ - encoded: node.PartialKey, - children: children, - value: node.Value, - }, nil - default: - panic("Invalid node") - } -} diff --git a/pkg/trie/triedb/triedbbuilder.go b/pkg/trie/triedb/triedbbuilder.go deleted file mode 100644 index 9b530484df..0000000000 --- a/pkg/trie/triedb/triedbbuilder.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package triedb - -import ( - "github.com/ChainSafe/gossamer/pkg/trie/hashdb" - "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibble" -) - -type DBValue = []byte - -type TrieDBBuilder[H hashdb.HashOut] struct { - db hashdb.HashDB[H, DBValue] - root H - cache TrieCache[H] - recorder TrieRecorder[H] - layout TrieLayout[H] -} - -func NewTrieDBBuilder[H hashdb.HashOut]( - db hashdb.HashDB[H, DBValue], - root H, - layout TrieLayout[H], -) *TrieDBBuilder[H] { - root = layout.Codec().HashedNullNode() - - return &TrieDBBuilder[H]{ - db: db, - root: root, - cache: nil, - recorder: nil, - layout: layout, - } -} - -func (self *TrieDBBuilder[H]) WithCache(cache TrieCache[H]) *TrieDBBuilder[H] { - self.cache = cache - return self -} - -func (self *TrieDBBuilder[H]) WithRecorder(recorder TrieRecorder[H]) *TrieDBBuilder[H] { - self.recorder = recorder - return self -} - -func (self *TrieDBBuilder[H]) Build() *TrieDB[H] { - rootHandle := Hash[H]{self.root} - - return &TrieDB[H]{ - db: self.db, - root: self.root, - cache: self.cache, - recorder: self.recorder, - storage: NewEmptyNodeStorage[H](), - deathRow: make(map[string]nibble.Prefix), - rootHandle: rootHandle, - layout: self.layout, - } -} diff --git a/tests/rpc/rpc_05-state_test.go b/tests/rpc/rpc_05-state_test.go index 9d3573cce8..c37f794077 100644 --- a/tests/rpc/rpc_05-state_test.go +++ b/tests/rpc/rpc_05-state_test.go @@ -12,9 +12,9 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/runtime" - "github.com/ChainSafe/gossamer/lib/trie" libutils "github.com/ChainSafe/gossamer/lib/utils" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/tests/utils/config" "github.com/ChainSafe/gossamer/tests/utils/node" "github.com/ChainSafe/gossamer/tests/utils/rpc" From 8d55a15514caee183692de705c74c478409e3cd8 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:35:32 -0300 Subject: [PATCH 31/35] fix(pkg/trie): imports --- scripts/trie_state_script.go | 2 +- scripts/trie_state_script_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/trie_state_script.go b/scripts/trie_state_script.go index 0348134c54..3117ce93c3 100644 --- a/scripts/trie_state_script.go +++ b/scripts/trie_state_script.go @@ -12,8 +12,8 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/tests/utils/rpc" ) diff --git a/scripts/trie_state_script_test.go b/scripts/trie_state_script_test.go index 06bbedf654..11703b7077 100644 --- a/scripts/trie_state_script_test.go +++ b/scripts/trie_state_script_test.go @@ -9,7 +9,7 @@ import ( "github.com/ChainSafe/gossamer/dot/rpc/modules" "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/trie" + "github.com/ChainSafe/gossamer/pkg/trie" "github.com/stretchr/testify/require" ) From 09e8bba8cef2908fd3d6216e7b064620b7aee655 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:41:58 -0300 Subject: [PATCH 32/35] fix(ci): Update tests path --- .github/workflows/unit-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 315acc738d..d2c79f3101 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -82,9 +82,9 @@ jobs: - name: Trie memory test run: | - sed -i 's/const skip = true/const skip = false/g' ./lib/trie/mem_test.go - go test -run ^Test_Trie_MemoryUsage$ ./lib/trie - sed -i 's/const skip = false/const skip = true/g' ./lib/trie/mem_test.go + sed -i 's/const skip = true/const skip = false/g' ./pkg/trie/mem_test.go + go test -run ^Test_Trie_MemoryUsage$ ./pkg/trie + sed -i 's/const skip = false/const skip = true/g' ./pkg/trie/mem_test.go - name: Test - Race run: make test-using-race-detector From 17350953e652ca2ffb32fce69ca679ab8d01f988 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:46:12 -0300 Subject: [PATCH 33/35] chore(pkg/trie): fix comment --- pkg/trie/node/branch_encode.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/trie/node/branch_encode.go b/pkg/trie/node/branch_encode.go index 54a67033cb..1e64c7c35b 100644 --- a/pkg/trie/node/branch_encode.go +++ b/pkg/trie/node/branch_encode.go @@ -41,9 +41,9 @@ var parallelEncodingRateLimit = make(chan struct{}, parallelLimit) // encodeChildrenOpportunisticParallel encodes children in parallel eventually. // Leaves are encoded in a blocking way, and branches are encoded in separate -// goroutines IF they are less than the parallelLimit number of goroutines al.y -// running. This is designed to limit the total number of goroutines in order to -// avoid using too much memory on the stack. +// goroutines IF they are less than the parallelLimit number of goroutines +// already running. This is designed to limit the total number of goroutines in +// order to avoid using too much memory on the stack. func encodeChildrenOpportunisticParallel(children []*Node, maxInlineValue int, buffer io.Writer) (err error) { // Buffered channels since children might be encoded in this // goroutine or another one. From cb6247d347b15e7d7f604ae628c3530cabbaba94 Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:47:01 -0300 Subject: [PATCH 34/35] chore(deps): Remove unnecesary dependencies --- go.mod | 1 - go.sum | 3 --- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index dac5e2c456..1d549678b6 100644 --- a/go.mod +++ b/go.mod @@ -74,7 +74,6 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gammazero/deque v0.2.1 github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 817e9aee26..8b428d72ae 100644 --- a/go.sum +++ b/go.sum @@ -147,10 +147,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ethereum/go-ethereum v1.13.12 h1:iDr9UM2JWkngBHGovRJEQn4Kor7mT4gt9rUZqB5M29Y= github.com/ethereum/go-ethereum v1.13.12/go.mod h1:hKL2Qcj1OvStXNSEDbucexqnEt1Wh4Cz329XsjAalZY= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -679,7 +677,6 @@ golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= From 4eadc2fc94099a784096ada36ce4716eae0b036b Mon Sep 17 00:00:00 2001 From: Diego Date: Wed, 14 Feb 2024 10:50:21 -0300 Subject: [PATCH 35/35] fix(ci): Update fuzz test path --- .github/workflows/fuzz.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index c3d4d6a3d6..cdf5093286 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -41,4 +41,4 @@ jobs: restore-keys: ${{ runner.os }}-go-mod - name: Fuzz trie - run: go test -run Fuzz_Trie_PutAndGet_Single -fuzz=Fuzz_Trie_PutAndGet_Single -fuzztime=5m github.com/ChainSafe/gossamer/lib/trie + run: go test -run Fuzz_Trie_PutAndGet_Single -fuzz=Fuzz_Trie_PutAndGet_Single -fuzztime=5m github.com/ChainSafe/gossamer/pkg/trie