Invert --disable-new-state-mgmt to --enable-new-state-mgmt (#5567)

* Invert the flag
* Merge branch 'master' into invert-disable-new-state-mgmt
* Merge refs/heads/master into invert-disable-new-state-mgmt
* Add new-state-mgmt to e2e flag
* Merge branch 'invert-disable-new-state-mgmt' of github.com:prysmaticlabs/prysm into invert-disable-new-state-mgmt
* Merge refs/heads/master into invert-disable-new-state-mgmt

beacon-chain/blockchain/head.go

@@ -59,7 +59,7 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// If the head state is not available, just return nil.
 	// There's nothing to cache
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		if !s.stateGen.StateSummaryExists(ctx, headRoot) {
 			return nil
 		}
@@ -81,7 +81,7 @@ func (s *Service) saveHead(ctx context.Context, headRoot [32]byte) error {
 
 	// Get the new head state from cached state or DB.
 	var newHeadState *state.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		newHeadState, err = s.stateGen.StateByRoot(ctx, headRoot)
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head state in DB")
@@ -121,7 +121,7 @@ func (s *Service) saveHeadNoDB(ctx context.Context, b *ethpb.SignedBeaconBlock,
 
 	var headState *state.BeaconState
 	var err error
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		headState, err = s.stateGen.StateByRoot(ctx, r)
 		if err != nil {
 			return errors.Wrap(err, "could not retrieve head state in DB")

beacon-chain/blockchain/process_attestation_helpers.go

@@ -32,7 +32,7 @@ func (s *Service) getAttPreState(ctx context.Context, c *ethpb.Checkpoint) (*sta
 	}
 
 	var baseState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		if !s.stateGen.HasState(ctx, bytesutil.ToBytes32(c.Root)) {
 			if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
 				return nil, errors.Wrap(err, "could not save initial sync blocks")
@@ -134,7 +134,7 @@ func (s *Service) verifyAttestation(ctx context.Context, baseState *stateTrie.Be
 			// different seeds.
 			var aState *stateTrie.BeaconState
 			var err error
-			if !featureconfig.Get().DisableNewStateMgmt {
+			if featureconfig.Get().NewStateMgmt {
 				if !s.stateGen.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) {
 					if err := s.beaconDB.SaveBlocks(ctx, s.getInitSyncBlocks()); err != nil {
 						return nil, errors.Wrap(err, "could not save initial sync blocks")

beacon-chain/blockchain/process_attestation_test.go

@@ -114,7 +114,7 @@ func TestStore_OnAttestation(t *testing.T) {
 			a:             &ethpb.Attestation{Data: &ethpb.AttestationData{Target: &ethpb.Checkpoint{Root: BlkWithOutStateRoot[:]}}},
 			s:             &pb.BeaconState{},
 			wantErr:       true,
-			wantErrString: "could not get pre state for slot 0: could not get ancestor state",
+			wantErrString: "pre state of target block 0 does not exist",
 		},
 		{
 			name: "process attestation doesn't match current epoch",

beacon-chain/blockchain/process_block.go

@@ -92,7 +92,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 		return nil, errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
 			return nil, errors.Wrap(err, "could not save state")
 		}
@@ -122,7 +122,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 			return nil, errors.Wrap(err, "could not save finalized checkpoint")
 		}
 
-		if featureconfig.Get().DisableNewStateMgmt {
+		if !featureconfig.Get().NewStateMgmt {
 			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
 			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
 			if endSlot > startSlot {
@@ -147,7 +147,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock)
 			return nil, errors.Wrap(err, "could not save new justified")
 		}
 
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
 			fBlock, err := s.beaconDB.Block(ctx, fRoot)
 			if err != nil {
@@ -233,7 +233,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		return errors.Wrapf(err, "could not insert block %d to fork choice store", b.Slot)
 	}
 
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		if err := s.stateGen.SaveState(ctx, root, postState); err != nil {
 			return errors.Wrap(err, "could not save state")
 		}
@@ -268,7 +268,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 
 	// Update finalized check point. Prune the block cache and helper caches on every new finalized epoch.
 	if postState.FinalizedCheckpointEpoch() > s.finalizedCheckpt.Epoch {
-		if featureconfig.Get().DisableNewStateMgmt {
+		if !featureconfig.Get().NewStateMgmt {
 			startSlot := helpers.StartSlot(s.prevFinalizedCheckpt.Epoch)
 			endSlot := helpers.StartSlot(s.finalizedCheckpt.Epoch)
 			if endSlot > startSlot {
@@ -301,7 +301,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 			return errors.Wrap(err, "could not save new justified")
 		}
 
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			fRoot := bytesutil.ToBytes32(postState.FinalizedCheckpoint().Root)
 			fBlock, err := s.beaconDB.Block(ctx, fRoot)
 			if err != nil {
@@ -313,7 +313,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 		}
 	}
 
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		numOfStates := len(s.boundaryRoots)
 		if numOfStates > initialSyncCacheSize {
 			if err = s.persistCachedStates(ctx, numOfStates); err != nil {
@@ -338,7 +338,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed
 			return err
 		}
 
-		if featureconfig.Get().DisableNewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
+		if !featureconfig.Get().NewStateMgmt && helpers.IsEpochStart(postState.Slot()) {
 			if err := s.beaconDB.SaveState(ctx, postState, root); err != nil {
 				return errors.Wrap(err, "could not save state")
 			}

beacon-chain/blockchain/process_block_helpers.go

@@ -65,7 +65,7 @@ func (s *Service) verifyBlkPreState(ctx context.Context, b *ethpb.BeaconBlock) (
 	ctx, span := trace.StartSpan(ctx, "chainService.verifyBlkPreState")
 	defer span.End()
 
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		parentRoot := bytesutil.ToBytes32(b.ParentRoot)
 		// Loosen the check to HasBlock because state summary gets saved in batches
 		// during initial syncing. There's no risk given a state summary object is just a
@@ -283,7 +283,7 @@ func (s *Service) updateJustified(ctx context.Context, state *stateTrie.BeaconSt
 		s.justifiedCheckpt = cpt
 	}
 
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		justifiedRoot := bytesutil.ToBytes32(cpt.Root)
 
 		justifiedState := s.initSyncState[justifiedRoot]

beacon-chain/blockchain/process_block_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/roughtime"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
@@ -88,7 +89,7 @@ func TestStore_OnBlock(t *testing.T) {
 			name:          "parent block root does not have a state",
 			blk:           &ethpb.BeaconBlock{},
 			s:             st.Copy(),
-			wantErrString: "could not reconstruct parent state",
+			wantErrString: "provided block root does not have block saved in the db",
 		},
 		{
 			name:          "block is from the feature",
@@ -300,6 +301,11 @@ func TestShouldUpdateJustified_ReturnFalse(t *testing.T) {
 }
 
 func TestCachedPreState_CanGetFromStateSummary(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: true,
+	}
+	featureconfig.Init(config)
+
 	ctx := context.Background()
 	db := testDB.SetupDB(t)
 	defer testDB.TeardownDB(t, db)

beacon-chain/blockchain/receive_attestation.go

@@ -91,7 +91,7 @@ func (s *Service) processAttestation(subscribedToStateEvents chan struct{}) {
 			atts := s.attPool.ForkchoiceAttestations()
 			for _, a := range atts {
 				var hasState bool
-				if !featureconfig.Get().DisableNewStateMgmt {
+				if featureconfig.Get().NewStateMgmt {
 					hasState = s.stateGen.StateSummaryExists(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot))
 				} else {
 					hasState = s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.BeaconBlockRoot)) && s.beaconDB.HasState(ctx, bytesutil.ToBytes32(a.Data.Target.Root))

beacon-chain/blockchain/service.go

@@ -140,7 +140,7 @@ func (s *Service) Start() {
 	}
 
 	if beaconState == nil {
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			beaconState, err = s.stateGen.StateByRoot(ctx, bytesutil.ToBytes32(cp.Root))
 			if err != nil {
 				log.Fatalf("Could not fetch beacon state by root: %v", err)
@@ -181,7 +181,7 @@ func (s *Service) Start() {
 		s.prevFinalizedCheckpt = stateTrie.CopyCheckpoint(finalizedCheckpoint)
 		s.resumeForkChoice(justifiedCheckpoint, finalizedCheckpoint)
 
-		if featureconfig.Get().DisableNewStateMgmt {
+		if !featureconfig.Get().NewStateMgmt {
 			if finalizedCheckpoint.Epoch > 1 {
 				if err := s.pruneGarbageState(ctx, helpers.StartSlot(finalizedCheckpoint.Epoch)-params.BeaconConfig().SlotsPerEpoch); err != nil {
 					log.WithError(err).Warn("Could not prune old states")
@@ -327,7 +327,7 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState *stateTrie.B
 	if err := s.beaconDB.SaveBlock(ctx, genesisBlk); err != nil {
 		return errors.Wrap(err, "could not save genesis block")
 	}
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		if err := s.stateGen.SaveState(ctx, genesisBlkRoot, genesisState); err != nil {
 			return errors.Wrap(err, "could not save genesis state")
 		}
@@ -415,7 +415,7 @@ func (s *Service) initializeChainInfo(ctx context.Context) error {
 	}
 	finalizedRoot := bytesutil.ToBytes32(finalized.Root)
 	var finalizedState *stateTrie.BeaconState
-	if !featureconfig.Get().DisableNewStateMgmt {
+	if featureconfig.Get().NewStateMgmt {
 		finalizedRoot = s.beaconDB.LastArchivedIndexRoot(ctx)
 		finalizedState, err = s.stateGen.Resume(ctx)
 		if err != nil {

beacon-chain/db/kv/blocks.go

@@ -263,7 +263,7 @@ func (k *Store) SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
 	ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveHeadBlockRoot")
 	defer span.End()
 	return k.db.Update(func(tx *bolt.Tx) error {
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			hasStateSummaryInCache := k.stateSummaryCache.Has(blockRoot)
 			hasStateSummaryInDB := tx.Bucket(stateSummaryBucket).Get(blockRoot[:]) != nil
 			hasStateInDB := tx.Bucket(stateBucket).Get(blockRoot[:]) != nil

beacon-chain/db/kv/check_historical_state.go

@@ -13,7 +13,7 @@ var historicalStateDeletedKey = []byte("historical-states-deleted")
 
 // HistoricalStatesDeleted verifies historical states exist in DB.
 func (kv *Store) HistoricalStatesDeleted(ctx context.Context) error {
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		return kv.db.Update(func(tx *bolt.Tx) error {
 			bkt := tx.Bucket(newStateServiceCompatibleBucket)
 			return bkt.Put(historicalStateDeletedKey, []byte{0x01})

beacon-chain/db/kv/checkpoint.go

@@ -65,7 +65,7 @@ func (k *Store) SaveJustifiedCheckpoint(ctx context.Context, checkpoint *ethpb.C
 	}
 	return k.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(checkpointBucket)
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			hasStateSummaryInDB := tx.Bucket(stateSummaryBucket).Get(checkpoint.Root) != nil
 			hasStateSummaryInCache := k.stateSummaryCache.Has(bytesutil.ToBytes32(checkpoint.Root))
 			hasStateInDB := tx.Bucket(stateBucket).Get(checkpoint.Root) != nil
@@ -96,7 +96,7 @@ func (k *Store) SaveFinalizedCheckpoint(ctx context.Context, checkpoint *ethpb.C
 	}
 	return k.db.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket(checkpointBucket)
-		if !featureconfig.Get().DisableNewStateMgmt {
+		if featureconfig.Get().NewStateMgmt {
 			hasStateSummaryInDB := tx.Bucket(stateSummaryBucket).Get(checkpoint.Root) != nil
 			hasStateSummaryInCache := k.stateSummaryCache.Has(bytesutil.ToBytes32(checkpoint.Root))
 			hasStateInDB := tx.Bucket(stateBucket).Get(checkpoint.Root) != nil

beacon-chain/rpc/beacon/assignments.go

@@ -32,7 +32,7 @@ func (bs *Server) ListValidatorAssignments(
 		)
 	}
 
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		return bs.listValidatorAssignmentsUsingOldArchival(ctx, req)
 	}
 

beacon-chain/rpc/beacon/assignments_test.go

@@ -18,11 +18,17 @@ import (
 	dbTest "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/flags"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
+	"github.com/prysmaticlabs/prysm/shared/featureconfig"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: true,
+	}
+	featureconfig.Init(config)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 
@@ -46,6 +52,11 @@ func TestServer_ListAssignments_CannotRequestFutureEpoch(t *testing.T) {
 }
 
 func TestServer_ListAssignments_NoResults(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: true,
+	}
+	featureconfig.Init(config)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 
@@ -94,6 +105,11 @@ func TestServer_ListAssignments_NoResults(t *testing.T) {
 }
 
 func TestServer_ListAssignments_Pagination_InputOutOfRange(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: true,
+	}
+	featureconfig.Init(config)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 
@@ -430,6 +446,11 @@ func TestServer_ListAssignments_FilterPubkeysIndices_NoPagination(t *testing.T)
 }
 
 func TestServer_ListAssignments_CanFilterPubkeysIndices_WithPagination(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: true,
+	}
+	featureconfig.Init(config)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 

beacon-chain/rpc/beacon/attestations.go

@@ -128,7 +128,7 @@ func (bs *Server) ListIndexedAttestations(
 	default:
 		return nil, status.Error(codes.InvalidArgument, "Must specify a filter criteria for fetching attestations")
 	}
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		return nil, status.Error(codes.Internal, "New state management must be turned on to support historic attestation. Please run without --disable-new-state-mgmt flag")
 	}
 

beacon-chain/rpc/beacon/attestations_test.go

@@ -574,17 +574,19 @@ func TestServer_mapAttestationToTargetRoot(t *testing.T) {
 }
 
 func TestServer_ListIndexedAttestations_NewStateManagnmentDisabled(t *testing.T) {
+	config := &featureconfig.Flags{
+		NewStateMgmt: false,
+	}
+	featureconfig.Init(config)
+
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
 	params.OverrideBeaconConfig(params.MainnetConfig())
 	defer params.OverrideBeaconConfig(params.MinimalSpecConfig())
 	ctx := context.Background()
 	numValidators := uint64(128)
 	state, _ := testutil.DeterministicGenesisState(t, numValidators)
-	config := &featureconfig.Flags{
-		DisableNewStateMgmt: true,
-	}
-	featureconfig.Init(config)
+
 	bs := &Server{
 		BeaconDB:           db,
 		GenesisTimeFetcher: &mock.ChainService{State: state},
@@ -1234,9 +1236,9 @@ func TestServer_StreamAttestations_OnSlotTick(t *testing.T) {
 // assertNewStateMgmtIsEnabled asserts that state management feature is enabled.
 func assertNewStateMgmtIsEnabled() *featureconfig.Flags {
 	cfg := featureconfig.Get()
-	if cfg.DisableNewStateMgmt {
+	if cfg.NewStateMgmt {
 		cfgUpd := cfg.Copy()
-		cfgUpd.DisableNewStateMgmt = false
+		cfgUpd.NewStateMgmt = true
 		featureconfig.Init(cfgUpd)
 	}
 	return cfg

beacon-chain/rpc/beacon/committees.go

@@ -46,7 +46,7 @@ func (bs *Server) ListBeaconCommittees(
 	committees := make(map[uint64]*ethpb.BeaconCommittees_CommitteesList)
 	activeIndices := make([]uint64, 0)
 	var err error
-	if featureconfig.Get().DisableNewStateMgmt {
+	if !featureconfig.Get().NewStateMgmt {
 		committees, activeIndices, err = bs.retrieveCommitteesForEpochUsingOldArchival(ctx, requestedEpoch)
 		if err != nil {
 			return nil, status.Errorf(

beacon-chain/rpc/beacon/committees_test.go

@@ -87,9 +87,10 @@ func TestServer_ListBeaconCommittees_CurrentEpoch(t *testing.T) {
 }
 
 func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
-	fc := featureconfig.Get()
-	fc.DisableNewStateMgmt = true
-	featureconfig.Init(fc)
+	config := &featureconfig.Flags{
+		NewStateMgmt: false,
+	}
+	featureconfig.Init(config)
 
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)
@@ -161,9 +162,10 @@ func TestServer_ListBeaconCommittees_PreviousEpoch(t *testing.T) {
 }
 
 func TestServer_ListBeaconCommittees_FromArchive(t *testing.T) {
-	fc := featureconfig.Get()
-	fc.DisableNewStateMgmt = true
-	featureconfig.Init(fc)
+	config := &featureconfig.Flags{
+		NewStateMgmt: false,
+	}
+	featureconfig.Init(config)
 
 	db := dbTest.SetupDB(t)
 	defer dbTest.TeardownDB(t, db)