Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add HighestSlotStatesBelow DB getter #5213

Merged
merged 8 commits into from
Mar 26, 2020
Merged

Add HighestSlotStatesBelow DB getter #5213

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
24710ab
Add HighestSlotStatesBelow
terencechain Mar 25, 2020
ffb2d1e
Tests for HighestSlotStatesBelow
terencechain Mar 25, 2020
cb755ce
Typos
terencechain Mar 25, 2020
f57df71
Comment
terencechain Mar 25, 2020
5844d02
Merge refs/heads/master into states-slots-saved-at
prylabs-bulldozer[bot] Mar 26, 2020
fbb7667
Quick fix
terencechain Mar 26, 2020
6504547
Merge branch 'states-slots-saved-at' of github.com:prysmaticlabs/prys…
terencechain Mar 26, 2020
6e467e6
Prevent underflow foreal, thanks nishant!
terencechain Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions beacon-chain/db/kv/blocks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ func (k *Store) HighestSlotBlocks(ctx context.Context) ([]*ethpb.SignedBeaconBlo

// HighestSlotBlocksBelow returns the block with the highest slot below the input slot from the db.
func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*ethpb.SignedBeaconBlock, error) {
ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotBlockAt")
ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotBlocksBelow")
defer span.End()

blocks := make([]*ethpb.SignedBeaconBlock, 0)
Expand All @@ -350,7 +350,7 @@ func (k *Store) HighestSlotBlocksBelow(ctx context.Context, slot uint64) ([]*eth
return blocks, err
}

// blocksAtSlotBitfieldIndex retrieves the block in DB given the input index. The index represents
// blocksAtSlotBitfieldIndex retrieves the blocks in DB given the input index. The index represents
// the position of the slot bitfield the saved block maps to.
func (k *Store) blocksAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*ethpb.SignedBeaconBlock, error) {
ctx, span := trace.StartSpan(ctx, "BeaconDB.blocksAtSlotBitfieldIndex")
Expand Down
93 changes: 67 additions & 26 deletions beacon-chain/db/kv/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,39 +339,40 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er
if err != nil {
return err
}
highestSlot := highestIndex - 1
highestSlot = int(math.Max(0, float64(highestSlot)))
f := filters.NewFilter().SetStartSlot(uint64(highestSlot)).SetEndSlot(uint64(highestSlot))
states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex))

keys, err := getBlockRootsByFilter(ctx, tx, f)
if err != nil {
return err
}
return err
})
if err != nil {
return nil, err
}

if len(keys) == 0 {
return errors.New("could not get one block root to get state")
}
if len(states) == 0 {
return nil, errors.New("could not get one state")
}

stateBkt := tx.Bucket(stateBucket)
for i := range keys {
enc := stateBkt.Get(keys[i][:])
if enc == nil {
continue
}
pbState, err := createState(enc)
if err != nil {
return err
}
s, err := state.InitializeFromProtoUnsafe(pbState)
if err != nil {
return err
}
states = append(states, s)
return states, nil
}

// HighestSlotStatesBelow returns the states with the highest slot below the input slot
// from the db. Ideally there should just be one state per slot, but given validator
// can double propose, a single slot could have multiple block roots and
// reuslts states. This returns a list of states.
func (k *Store) HighestSlotStatesBelow(ctx context.Context, slot uint64) ([]*state.BeaconState, error) {
ctx, span := trace.StartSpan(ctx, "BeaconDB.HighestSlotStatesBelow")
defer span.End()
var states []*state.BeaconState
err := k.db.View(func(tx *bolt.Tx) error {
slotBkt := tx.Bucket(slotsHasObjectBucket)
savedSlots := slotBkt.Get(savedStateSlotsKey)
highestIndex, err := bytesutil.HighestBitIndexAt(savedSlots, int(slot))
if err != nil {
return err
}
states, err = k.statesAtSlotBitfieldIndex(ctx, tx, uint64(highestIndex))

return err
})

if err != nil {
return nil, err
}
Expand All @@ -383,6 +384,46 @@ func (k *Store) HighestSlotStates(ctx context.Context) ([]*state.BeaconState, er
return states, nil
}

// statesAtSlotBitfieldIndex retrieves the states in DB given the input index. The index represents
// the position of the slot bitfield the saved state maps to.
func (k *Store) statesAtSlotBitfieldIndex(ctx context.Context, tx *bolt.Tx, index uint64) ([]*state.BeaconState, error) {
ctx, span := trace.StartSpan(ctx, "BeaconDB.statesAtSlotBitfieldIndex")
defer span.End()

highestSlot := index - 1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what if index is 0, this overflows

Copy link
Member Author

@terencechain terencechain Mar 26, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

addressed the line below :)

highestSlot = int(math.Max(0, float64(highestSlot)))

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

wouldn't

math.Max(0, float64(highestSlot))

return the larger number which will be MaxFloat64 ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice catch! It's now taking int instead of uint64, also verified via a test. Thank you :)

highestSlot = uint64(math.Max(0, float64(highestSlot)))
f := filters.NewFilter().SetStartSlot(highestSlot).SetEndSlot(highestSlot)

keys, err := getBlockRootsByFilter(ctx, tx, f)
if err != nil {
return nil, err
}

if len(keys) == 0 {
return nil, errors.New("could not get one block root to get state")
}

stateBkt := tx.Bucket(stateBucket)
states := make([]*state.BeaconState, 0, len(keys))
for i := range keys {
enc := stateBkt.Get(keys[i][:])
if enc == nil {
continue
}
pbState, err := createState(enc)
if err != nil {
return nil, err
}
s, err := state.InitializeFromProtoUnsafe(pbState)
if err != nil {
return nil, err
}
states = append(states, s)
}

return states, err
}

// setStateSlotBitField sets the state slot bit in DB.
// This helps to track which slot has a saved state in db.
func (k *Store) setStateSlotBitField(ctx context.Context, tx *bolt.Tx, slot uint64) error {
Expand Down
79 changes: 79 additions & 0 deletions beacon-chain/db/kv/state_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,3 +368,82 @@ func TestStore_SaveDeleteState_CanGetHighest(t *testing.T) {
t.Errorf("Did not retrieve saved state: %v != %v", highest, s1)
}
}

func TestStore_SaveDeleteState_CanGetHighestBelow(t *testing.T) {
db := setupDB(t)
defer teardownDB(t, db)

s0 := &pb.BeaconState{Slot: 1}
b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
r, _ := ssz.HashTreeRoot(b.Block)
if err := db.SaveBlock(context.Background(), b); err != nil {
t.Fatal(err)
}
st, err := state.InitializeFromProto(s0)
if err != nil {
t.Fatal(err)
}
if err := db.SaveState(context.Background(), st, r); err != nil {
t.Fatal(err)
}

s1 := &pb.BeaconState{Slot: 100}
b = &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 100}}
r1, _ := ssz.HashTreeRoot(b.Block)
if err := db.SaveBlock(context.Background(), b); err != nil {
t.Fatal(err)
}
st, err = state.InitializeFromProto(s1)
if err != nil {
t.Fatal(err)
}
if err := db.SaveState(context.Background(), st, r1); err != nil {
t.Fatal(err)
}

highest, err := db.HighestSlotStates(context.Background())
if err != nil {
t.Fatal(err)
}
if !proto.Equal(highest[0].InnerStateUnsafe(), s1) {
t.Errorf("Did not retrieve saved state: %v != %v", highest, s1)
}

s2 := &pb.BeaconState{Slot: 1000}
b = &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1000}}
r2, _ := ssz.HashTreeRoot(b.Block)
if err := db.SaveBlock(context.Background(), b); err != nil {
t.Fatal(err)
}
st, err = state.InitializeFromProto(s2)
if err != nil {
t.Fatal(err)
}
if err := db.SaveState(context.Background(), st, r2); err != nil {
t.Fatal(err)
}

highest, err = db.HighestSlotStatesBelow(context.Background(), 2)
if err != nil {
t.Fatal(err)
}
if !proto.Equal(highest[0].InnerStateUnsafe(), s0) {
t.Errorf("Did not retrieve saved state: %v != %v", highest, s0)
}

highest, err = db.HighestSlotStatesBelow(context.Background(), 101)
if err != nil {
t.Fatal(err)
}
if !proto.Equal(highest[0].InnerStateUnsafe(), s1) {
t.Errorf("Did not retrieve saved state: %v != %v", highest, s1)
}

highest, err = db.HighestSlotStatesBelow(context.Background(), 1001)
if err != nil {
t.Fatal(err)
}
if !proto.Equal(highest[0].InnerStateUnsafe(), s2) {
t.Errorf("Did not retrieve saved state: %v != %v", highest, s2)
}
}