From 24710ab1c54687086894300b41e206d4cf9071ad Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 25 Mar 2020 14:50:38 -0700 Subject: [PATCH 1/6] Add HighestSlotStatesBelow --- beacon-chain/db/kv/state.go | 93 ++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/beacon-chain/db/kv/state.go b/beacon-chain/db/kv/state.go index 65197c16062e..5d88a35d8b90 100644 --- a/beacon-chain/db/kv/state.go +++ b/beacon-chain/db/kv/state.go @@ -339,39 +339,40 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er if err != nil { return err } - highestSlot := highestIndex - 1 - highestSlot = int(math.Max(0, float64(highestSlot))) - f := filters.NewFilter().SetStartSlot(uint64(highestSlot)).SetEndSlot(uint64(highestSlot)) + states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) - keys, err := getBlockRootsByFilter(ctx, tx, f) - if err != nil { - return err - } + return err + }) + if err != nil { + return nil, err + } - if len(keys) == 0 { - return errors.New("could not get one block root to get state") - } + if len(states) == 0 { + return nil, errors.New("could not get one state") + } - stateBkt := tx.Bucket(stateBucket) - for i := range keys { - enc := stateBkt.Get(keys[i][:]) - if enc == nil { - continue - } - pbState, err := createState(enc) - if err != nil { - return err - } - s, err := state.InitializeFromProtoUnsafe(pbState) - if err != nil { - return err - } - states = append(states, s) + return states, nil +} + +// HighestSlotStates returns the states with the highest slot below the input slot +// from the db. Ideally there should just be one state per slot, but given validator +// can double propose, a single slot could have multiple block roots and +// reuslts states. This returns a list of states. +func (k *Store) HighestSlotStatesBelow(ctx context.Context, slot uint64) ([]*state.BeaconState, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotStatesBelow") + defer span.End() + var states []*state.BeaconState + err := k.db.View(func(tx *bolt.Tx) error { + slotBkt := tx.Bucket(slotsHasObjectBucket) + savedSlots := slotBkt.Get(savedStateSlotsKey) + highestIndex, err := bytesutil.HighestBitIndexAt(savedSlots, int(slot)) + if err != nil { + return err } + states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) return err }) - if err != nil { return nil, err } @@ -383,6 +384,46 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er return states, nil } +// statesAtSlotBitfieldIndex retrieves the states in DB given the input index. The index represents +// the position of the slot bitfield the saved state maps to. +func (k *Store) statesAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*state.BeaconState, error) { + ctx, span := trace.StartSpan(ctx, "BeaconDB.statesAtSlotBitfieldIndex") + defer span.End() + + highestSlot := index - 1 + highestSlot = uint64(math.Max(0, float64(highestSlot))) + f := filters.NewFilter().SetStartSlot(highestSlot).SetEndSlot(highestSlot) + + keys, err := getBlockRootsByFilter(ctx, tx, f) + if err != nil { + return nil, err + } + + if len(keys) == 0 { + return nil, errors.New("could not get one block root to get state") + } + + stateBkt := tx.Bucket(stateBucket) + states := make([]*state.BeaconState, 0, len(keys)) + for i := range keys { + enc := stateBkt.Get(keys[i][:]) + if enc == nil { + continue + } + pbState, err := createState(enc) + if err != nil { + return nil, err + } + s, err := state.InitializeFromProtoUnsafe(pbState) + if err != nil { + return nil, err + } + states = append(states, s) + } + + return states, err +} + // setStateSlotBitField sets the state slot bit in DB. // This helps to track which slot has a saved state in db. func (k *Store) setStateSlotBitField(ctx context.Context, tx *bolt.Tx, slot uint64) error { From ffb2d1eeb5f2c730e6b9ab7b3af60d3c160a78dd Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 25 Mar 2020 14:51:00 -0700 Subject: [PATCH 2/6] Tests for HighestSlotStatesBelow --- beacon-chain/db/kv/state_test.go | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/beacon-chain/db/kv/state_test.go b/beacon-chain/db/kv/state_test.go index a65ebac492bf..d43489561e78 100644 --- a/beacon-chain/db/kv/state_test.go +++ b/beacon-chain/db/kv/state_test.go @@ -368,3 +368,82 @@ func TestStore_SaveDeleteState_CanGetHighest(t *testing.T) { t.Errorf("Did not retrieve saved state: %v != %v", highest, s1) } } + +func TestStore_SaveDeleteState_CanGetHighestBelow(t *testing.T) { + db := setupDB(t) + defer teardownDB(t, db) + + s0 := &pb.BeaconState{Slot: 1} + b := ðpb.SignedBeaconBlock{Block: ðpb.BeaconBlock{Slot: 1}} + r, _ := ssz.HashTreeRoot(b.Block) + if err := db.SaveBlock(context.Background(), b); err != nil { + t.Fatal(err) + } + st, err := state.InitializeFromProto(s0) + if err != nil { + t.Fatal(err) + } + if err := db.SaveState(context.Background(), st, r); err != nil { + t.Fatal(err) + } + + s1 := &pb.BeaconState{Slot: 100} + b = ðpb.SignedBeaconBlock{Block: ðpb.BeaconBlock{Slot: 100}} + r1, _ := ssz.HashTreeRoot(b.Block) + if err := db.SaveBlock(context.Background(), b); err != nil { + t.Fatal(err) + } + st, err = state.InitializeFromProto(s1) + if err != nil { + t.Fatal(err) + } + if err := db.SaveState(context.Background(), st, r1); err != nil { + t.Fatal(err) + } + + highest, err := db.HighestSlotStates(context.Background()) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(highest[0].InnerStateUnsafe(), s1) { + t.Errorf("Did not retrieve saved state: %v != %v", highest, s1) + } + + s2 := &pb.BeaconState{Slot: 1000} + b = ðpb.SignedBeaconBlock{Block: ðpb.BeaconBlock{Slot: 1000}} + r2, _ := ssz.HashTreeRoot(b.Block) + if err := db.SaveBlock(context.Background(), b); err != nil { + t.Fatal(err) + } + st, err = state.InitializeFromProto(s2) + if err != nil { + t.Fatal(err) + } + if err := db.SaveState(context.Background(), st, r2); err != nil { + t.Fatal(err) + } + + highest, err = db.HighestSlotStatesBelow(context.Background(), 2) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(highest[0].InnerStateUnsafe(), s0) { + t.Errorf("Did not retrieve saved state: %v != %v", highest, s0) + } + + highest, err = db.HighestSlotStatesBelow(context.Background(), 101) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(highest[0].InnerStateUnsafe(), s1) { + t.Errorf("Did not retrieve saved state: %v != %v", highest, s1) + } + + highest, err = db.HighestSlotStatesBelow(context.Background(), 1001) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(highest[0].InnerStateUnsafe(), s2) { + t.Errorf("Did not retrieve saved state: %v != %v", highest, s2) + } +} From cb755ce88cf629c814f7eb98cc47efddaca629dd Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 25 Mar 2020 14:51:04 -0700 Subject: [PATCH 3/6] Typos --- beacon-chain/db/kv/blocks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon-chain/db/kv/blocks.go b/beacon-chain/db/kv/blocks.go index 60a67c4991a8..9f871aa28118 100644 --- a/beacon-chain/db/kv/blocks.go +++ b/beacon-chain/db/kv/blocks.go @@ -350,7 +350,7 @@ func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*eth return blocks, err } -// blocksAtSlotBitfieldIndex retrieves the block in DB given the input index. The index represents +// blocksAtSlotBitfieldIndex retrieves the blocks in DB given the input index. The index represents // the position of the slot bitfield the saved block maps to. func (k *Store) blocksAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*ethpb.SignedBeaconBlock, error) { ctx, span := trace.StartSpan(ctx, "BeaconDB.blocksAtSlotBitfieldIndex") From f57df7118f04f623894f7bc5be8b4d8ace04efea Mon Sep 17 00:00:00 2001 From: terence tsao Date: Wed, 25 Mar 2020 14:58:11 -0700 Subject: [PATCH 4/6] Comment --- beacon-chain/db/kv/blocks.go | 2 +- beacon-chain/db/kv/state.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon-chain/db/kv/blocks.go b/beacon-chain/db/kv/blocks.go index 9f871aa28118..e8c47be85da5 100644 --- a/beacon-chain/db/kv/blocks.go +++ b/beacon-chain/db/kv/blocks.go @@ -329,7 +329,7 @@ func (k *Store) HighestSlotBlocks(ctx context.Context) ([]*ethpb.SignedBeaconBlo // HighestSlotBlocksBelow returns the block with the highest slot below the input slot from the db. func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*ethpb.SignedBeaconBlock, error) { - ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotBlockAt") + ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotBlocksBelow") defer span.End() blocks := make([]*ethpb.SignedBeaconBlock, 0) diff --git a/beacon-chain/db/kv/state.go b/beacon-chain/db/kv/state.go index 5d88a35d8b90..074444367af7 100644 --- a/beacon-chain/db/kv/state.go +++ b/beacon-chain/db/kv/state.go @@ -354,7 +354,7 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er return states, nil } -// HighestSlotStates returns the states with the highest slot below the input slot +// HighestSlotStatesBelow returns the states with the highest slot below the input slot // from the db. Ideally there should just be one state per slot, but given validator // can double propose, a single slot could have multiple block roots and // reuslts states. This returns a list of states. From fbb76672000c25d0bd708ca98e2ee3f1c4122c55 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 26 Mar 2020 07:13:53 -0700 Subject: [PATCH 5/6] Quick fix --- beacon-chain/db/kv/state.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beacon-chain/db/kv/state.go b/beacon-chain/db/kv/state.go index 074444367af7..5b9f3c619c91 100644 --- a/beacon-chain/db/kv/state.go +++ b/beacon-chain/db/kv/state.go @@ -339,7 +339,7 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er if err != nil { return err } - states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) + states, err = k.statesAtSlotBitfieldIndex(ctx, tx, highestIndex) return err }) @@ -369,7 +369,7 @@ func (k *Store) HighestSlotStatesBelow(ctx context.Context, slot uint64) ([]*sta if err != nil { return err } - states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) + states, err = k.statesAtSlotBitfieldIndex(ctx, tx, highestIndex) return err }) @@ -386,13 +386,13 @@ func (k *Store) HighestSlotStatesBelow(ctx context.Context, slot uint64) ([]*sta // statesAtSlotBitfieldIndex retrieves the states in DB given the input index. The index represents // the position of the slot bitfield the saved state maps to. -func (k *Store) statesAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*state.BeaconState, error) { +func (k *Store) statesAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index int) ([]*state.BeaconState, error) { ctx, span := trace.StartSpan(ctx, "BeaconDB.statesAtSlotBitfieldIndex") defer span.End() highestSlot := index - 1 - highestSlot = uint64(math.Max(0, float64(highestSlot))) - f := filters.NewFilter().SetStartSlot(highestSlot).SetEndSlot(highestSlot) + highestSlot = int(math.Max(0, float64(highestSlot))) + f := filters.NewFilter().SetStartSlot(uint64(highestSlot)).SetEndSlot(uint64(highestSlot)) keys, err := getBlockRootsByFilter(ctx, tx, f) if err != nil { From 6e467e6f539448f29e2ff3beeb9d45f49ff13b41 Mon Sep 17 00:00:00 2001 From: terence tsao Date: Thu, 26 Mar 2020 07:49:21 -0700 Subject: [PATCH 6/6] Prevent underflow foreal, thanks nishant! --- beacon-chain/db/kv/blocks.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/beacon-chain/db/kv/blocks.go b/beacon-chain/db/kv/blocks.go index e8c47be85da5..900658346cff 100644 --- a/beacon-chain/db/kv/blocks.go +++ b/beacon-chain/db/kv/blocks.go @@ -317,7 +317,7 @@ func (k *Store) HighestSlotBlocks(ctx context.Context) ([]*ethpb.SignedBeaconBlo return err } - blocks, err = k.blocksAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) + blocks, err = k.blocksAtSlotBitfieldIndex(ctx, tx, highestIndex) if err != nil { return err } @@ -340,7 +340,7 @@ func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*eth if err != nil { return err } - blocks, err = k.blocksAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex)) + blocks, err = k.blocksAtSlotBitfieldIndex(ctx, tx, highestIndex) if err != nil { return err } @@ -352,13 +352,13 @@ func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*eth // blocksAtSlotBitfieldIndex retrieves the blocks in DB given the input index. The index represents // the position of the slot bitfield the saved block maps to. -func (k *Store) blocksAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*ethpb.SignedBeaconBlock, error) { +func (k *Store) blocksAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index int) ([]*ethpb.SignedBeaconBlock, error) { ctx, span := trace.StartSpan(ctx, "BeaconDB.blocksAtSlotBitfieldIndex") defer span.End() highestSlot := index - 1 - highestSlot = uint64(math.Max(0, float64(highestSlot))) - f := filters.NewFilter().SetStartSlot(highestSlot).SetEndSlot(highestSlot) + highestSlot = int(math.Max(0, float64(highestSlot))) + f := filters.NewFilter().SetStartSlot(uint64(highestSlot)).SetEndSlot(uint64(highestSlot)) keys, err := getBlockRootsByFilter(ctx, tx, f) if err != nil {