Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimize Insertion in Deposit Trie #4299

Merged
merged 21 commits into from
Dec 17, 2019
Merged

Optimize Insertion in Deposit Trie #4299

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
86933c2
current changes
nisdas Dec 16, 2019
9c666be
change algorithm for tree insert
rauljordan Dec 16, 2019
c009a5c
almost done with getting this to pass
rauljordan Dec 16, 2019
b30da5b
unit test passes
rauljordan Dec 16, 2019
7a01d2a
tests now pass
rauljordan Dec 16, 2019
6f46974
fix in repo
rauljordan Dec 16, 2019
b9fe45e
Merge branch 'master' into optimizeDepositLogs
rauljordan Dec 16, 2019
1effa14
fix build
rauljordan Dec 16, 2019
aadbe7c
Merge branch 'optimizeDepositLogs' of github.com:prysmaticlabs/prysm …
rauljordan Dec 16, 2019
b8d3b53
remove tautology
rauljordan Dec 16, 2019
2f06285
fix tautology
rauljordan Dec 16, 2019
77b2594
fix up sparsity
rauljordan Dec 16, 2019
6d41dda
Merge branch 'master' into optimizeDepositLogs
rauljordan Dec 16, 2019
a237d47
further fixes
rauljordan Dec 16, 2019
70840ad
Merge branch 'optimizeDepositLogs' of github.com:prysmaticlabs/prysm …
rauljordan Dec 16, 2019
f3b7ca4
Update shared/trieutil/sparse_merkle.go
rauljordan Dec 16, 2019
0ab8e8f
comments
rauljordan Dec 16, 2019
ce0a918
Merge branch 'optimizeDepositLogs' of github.com:prysmaticlabs/prysm …
rauljordan Dec 16, 2019
fe2ed98
add bench for optimized
rauljordan Dec 17, 2019
b144321
gaz
rauljordan Dec 17, 2019
9f65ea6
Merge refs/heads/master into optimizeDepositLogs
prylabs-bulldozer[bot] Dec 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion beacon-chain/core/state/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func GenesisBeaconState(deposits []*ethpb.Deposit, genesisTime uint64, eth1Data
}
leaves = append(leaves, hash[:])
}
var trie *trieutil.MerkleTrie
var trie *trieutil.SparseMerkleTrie
if len(leaves) > 0 {
trie, err = trieutil.GenerateTrieFromItems(leaves, int(params.BeaconConfig().DepositContractTreeDepth))
if err != nil {
Expand Down
9 changes: 3 additions & 6 deletions beacon-chain/powchain/log_processing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ func (s *Service) ProcessDepositLog(ctx context.Context, depositLog gethTypes.Lo

// We then decode the deposit input in order to create a deposit object
// we can store in our persistent DB.
validData := true
depositData := &ethpb.Deposit_Data{
Amount: bytesutil.FromBytes8(amount),
PublicKey: pubkey,
Expand All @@ -123,9 +122,7 @@ func (s *Service) ProcessDepositLog(ctx context.Context, depositLog gethTypes.Lo
return errors.Wrap(err, "Unable to determine hashed value of deposit")
}

if err := s.depositTrie.InsertIntoTrie(depositHash[:], int(index)); err != nil {
return errors.Wrap(err, "Unable to insert deposit into trie")
}
s.depositTrie.InsertIntoTrie(depositHash[:], int(index))

proof, err := s.depositTrie.MerkleProof(int(index))
if err != nil {
Expand All @@ -138,7 +135,7 @@ func (s *Service) ProcessDepositLog(ctx context.Context, depositLog gethTypes.Lo
}

// Make sure duplicates are rejected pre-chainstart.
if !s.chainStarted && validData {
if !s.chainStarted {
var pubkey = fmt.Sprintf("#%x", depositData.PublicKey)
if s.depositCache.PubkeyInChainstart(ctx, pubkey) {
log.Warnf("Pubkey %#x has already been submitted for chainstart", pubkey)
Expand All @@ -150,7 +147,7 @@ func (s *Service) ProcessDepositLog(ctx context.Context, depositLog gethTypes.Lo

// We always store all historical deposits in the DB.
s.depositCache.InsertDeposit(ctx, deposit, big.NewInt(int64(depositLog.BlockNumber)), int(index), s.depositTrie.Root())

validData := true
if !s.chainStarted {
s.chainStartDeposits = append(s.chainStartDeposits, deposit)
root := s.depositTrie.Root()
Expand Down
4 changes: 2 additions & 2 deletions beacon-chain/powchain/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ type Service struct {
blockCache *blockCache // cache to store block hash/block height.
depositContractCaller *contracts.DepositContractCaller
depositRoot []byte
depositTrie *trieutil.MerkleTrie
depositTrie *trieutil.SparseMerkleTrie
chainStartDeposits []*ethpb.Deposit
chainStarted bool
chainStartBlockNumber *big.Int
Expand Down Expand Up @@ -255,7 +255,7 @@ func (s *Service) DepositRoot() [32]byte {

// DepositTrie returns the sparse Merkle trie used for storing
// deposits from the ETH1.0 deposit contract.
func (s *Service) DepositTrie() *trieutil.MerkleTrie {
func (s *Service) DepositTrie() *trieutil.SparseMerkleTrie {
return s.depositTrie
}

Expand Down
4 changes: 2 additions & 2 deletions beacon-chain/powchain/testing/faulty_mock.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions beacon-chain/powchain/testing/mock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ func (m *POWChain) Eth2GenesisPowchainInfo() (uint64, *big.Int) {
}

// DepositTrie --
func (m *POWChain) DepositTrie() *trieutil.MerkleTrie {
return &trieutil.MerkleTrie{}
func (m *POWChain) DepositTrie() *trieutil.SparseMerkleTrie {
return &trieutil.SparseMerkleTrie{}
}

// BlockExists --
Expand Down
2 changes: 1 addition & 1 deletion beacon-chain/rpc/proposer/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func (ps *Server) defaultEth1DataResponse(ctx context.Context, currentHeight *bi
}, nil
}

func constructMerkleProof(trie *trieutil.MerkleTrie, index int, deposit *ethpb.Deposit) (*ethpb.Deposit, error) {
func constructMerkleProof(trie *trieutil.SparseMerkleTrie, index int, deposit *ethpb.Deposit) (*ethpb.Deposit, error) {
proof, err := trie.MerkleProof(index)
if err != nil {
return nil, errors.Wrapf(err, "could not generate merkle proof for deposit at index %d", index)
Expand Down
30 changes: 6 additions & 24 deletions beacon-chain/rpc/proposer/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,10 +314,7 @@ func TestPendingDeposits_OutsideEth1FollowWindow(t *testing.T) {
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, dp.Block, dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down Expand Up @@ -467,10 +464,7 @@ func TestPendingDeposits_FollowsCorrectEth1Block(t *testing.T) {
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, dp.Block, dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down Expand Up @@ -584,10 +578,7 @@ func TestPendingDeposits_CantReturnBelowStateEth1DepositIndex(t *testing.T) {
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, big.NewInt(int64(dp.Index)), dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down Expand Up @@ -693,10 +684,7 @@ func TestPendingDeposits_CantReturnMoreThanMax(t *testing.T) {
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, big.NewInt(int64(dp.Index)), dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down Expand Up @@ -800,10 +788,7 @@ func TestPendingDeposits_CantReturnMoreDepositCount(t *testing.T) {
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, big.NewInt(int64(dp.Index)), dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down Expand Up @@ -1166,10 +1151,7 @@ func TestDeposits_ReturnsEmptyList_IfLatestEth1DataEqGenesisEth1Block(t *testing
t.Fatalf("Unable to determine hashed value of deposit %v", err)
}

if err := depositTrie.InsertIntoTrie(depositHash[:], int(dp.Index)); err != nil {
t.Fatalf("Unable to insert deposit into trie %v", err)
}

depositTrie.InsertIntoTrie(depositHash[:], dp.Index)
depositCache.InsertDeposit(ctx, dp.Deposit, big.NewInt(int64(dp.Index)), dp.Index, depositTrie.Root())
}
for _, dp := range recentDeposits {
Expand Down
17 changes: 5 additions & 12 deletions contracts/deposit-contract/deposit_tree_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/prysmaticlabs/go-ssz"

rauljordan marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Revert new line please

depositcontract "github.com/prysmaticlabs/prysm/contracts/deposit-contract"
"github.com/prysmaticlabs/prysm/shared/interop"
"github.com/prysmaticlabs/prysm/shared/params"
Expand All @@ -32,7 +33,7 @@ func TestDepositTrieRoot_OK(t *testing.T) {
t.Errorf("Local deposit trie root and contract deposit trie root are not equal. Expected %#x , Got %#x", depRoot, localTrie.Root())
}

privKeys, pubKeys, err := interop.DeterministicallyGenerateKeys(0 /*startIndex*/, 101)
privKeys, pubKeys, err := interop.DeterministicallyGenerateKeys(0 /*startIndex*/, 10)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why?

if err != nil {
t.Fatal(err)
}
Expand All @@ -43,7 +44,7 @@ func TestDepositTrieRoot_OK(t *testing.T) {

testAcc.TxOpts.Value = depositcontract.Amount32Eth()

for i := 0; i < 100; i++ {
for i := 0; i < 5; i++ {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why lower this?

data := depositDataItems[i]
dataRoot := [32]byte{}
copy(dataRoot[:], depositDataRoots[i])
Expand All @@ -58,16 +59,11 @@ func TestDepositTrieRoot_OK(t *testing.T) {
t.Fatal(err)
}

err = localTrie.InsertIntoTrie(item[:], i)
if err != nil {
t.Error(err)
}

localTrie.InsertIntoTrie(item[:], i)
depRoot, err = testAcc.Contract.GetDepositRoot(&bind.CallOpts{})
if err != nil {
t.Fatal(err)
}

if depRoot != localTrie.HashTreeRoot() {
t.Errorf("Local deposit trie root and contract deposit trie root are not equal for index %d. Expected %#x , Got %#x", i, depRoot, localTrie.Root())
}
Expand Down Expand Up @@ -122,10 +118,7 @@ func TestDepositTrieRoot_Fail(t *testing.T) {
t.Fatal(err)
}

err = localTrie.InsertIntoTrie(item[:], i)
if err != nil {
t.Error(err)
}
localTrie.InsertIntoTrie(item[:], i)

depRoot, err = testAcc.Contract.GetDepositRoot(&bind.CallOpts{})
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions shared/interop/generate_genesis_state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func GenerateGenesisState(genesisTime, numValidators uint64) (*pb.BeaconState, [
}

// GenerateDepositsFromData a list of deposit items by creating proofs for each of them from a sparse Merkle trie.
func GenerateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trieutil.MerkleTrie) ([]*ethpb.Deposit, error) {
func GenerateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trieutil.SparseMerkleTrie) ([]*ethpb.Deposit, error) {
deposits := make([]*ethpb.Deposit, len(depositDataItems))
results, err := mputil.Scatter(len(depositDataItems), func(offset int, entries int, _ *sync.RWMutex) (interface{}, error) {
return generateDepositsFromData(depositDataItems[offset:offset+entries], offset, trie)
Expand All @@ -79,7 +79,7 @@ func GenerateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, trie *trie
}

// generateDepositsFromData a list of deposit items by creating proofs for each of them from a sparse Merkle trie.
func generateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, offset int, trie *trieutil.MerkleTrie) ([]*ethpb.Deposit, error) {
func generateDepositsFromData(depositDataItems []*ethpb.Deposit_Data, offset int, trie *trieutil.SparseMerkleTrie) ([]*ethpb.Deposit, error) {
deposits := make([]*ethpb.Deposit, len(depositDataItems))
for i, item := range depositDataItems {
proof, err := trie.MerkleProof(i + offset)
Expand Down
10 changes: 4 additions & 6 deletions shared/testutil/deposits.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ var lock sync.Mutex
// Caches
var cachedDeposits []*ethpb.Deposit
var privKeys []*bls.SecretKey
var trie *trieutil.MerkleTrie
var trie *trieutil.SparseMerkleTrie

// DeterministicDepositsAndKeys returns the entered amount of deposits and secret keys.
// The deposits are configured such that for deposit n the validator
Expand Down Expand Up @@ -81,9 +81,7 @@ func DeterministicDepositsAndKeys(numDeposits uint64) ([]*ethpb.Deposit, []*bls.
return nil, nil, errors.Wrap(err, "could not tree hash deposit data")
}

if err := trie.InsertIntoTrie(hashedDeposit[:], int(numExisting+i)); err != nil {
return nil, nil, errors.Wrap(err, "could not tree hash deposit data")
}
trie.InsertIntoTrie(hashedDeposit[:], int(numExisting+i))
}
}

Expand All @@ -105,7 +103,7 @@ func DeterministicDepositsAndKeys(numDeposits uint64) ([]*ethpb.Deposit, []*bls.

// DeterministicDepositTrie returns a merkle trie of the requested size from the
// deterministic deposits.
func DeterministicDepositTrie(size int) (*trieutil.MerkleTrie, [][32]byte, error) {
func DeterministicDepositTrie(size int) (*trieutil.SparseMerkleTrie, [][32]byte, error) {
items := trie.Items()
if size > len(items) {
return nil, [][32]byte{}, errors.New("requested a larger tree than amount of deposits")
Expand Down Expand Up @@ -162,7 +160,7 @@ func DeterministicGenesisState(t testing.TB, numValidators uint64) (*pb.BeaconSt
}

// DepositTrieFromDeposits takes an array of deposits and returns the deposit trie.
func DepositTrieFromDeposits(deposits []*ethpb.Deposit) (*trieutil.MerkleTrie, [][32]byte, error) {
func DepositTrieFromDeposits(deposits []*ethpb.Deposit) (*trieutil.SparseMerkleTrie, [][32]byte, error) {
encodedDeposits := make([][]byte, len(deposits))
for i := 0; i < len(encodedDeposits); i++ {
hashedDeposit, err := ssz.HashTreeRoot(deposits[i].Data)
Expand Down
4 changes: 2 additions & 2 deletions shared/trieutil/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"merkle_trie.go",
"helpers.go",
"sparse_merkle.go",
"zerohashes.go",
],
Expand All @@ -20,7 +20,7 @@ go_test(
name = "go_default_test",
size = "small",
srcs = [
"merkle_trie_test.go",
"helpers_test.go",
"sparse_merkle_test.go",
],
embed = [":go_default_library"],
Expand Down
0 shared/trieutil/merkle_trie.go → shared/trieutil/helpers.go
View file
Open in desktop
File renamed without changes.
0 shared/trieutil/merkle_trie_test.go → shared/trieutil/helpers_test.go
View file
Open in desktop
File renamed without changes.
Loading