BCFR-948 make persistence for HeadTracker optional (#14558)

* added configs

* Tests & Docs

.changeset/fifty-pillows-peel.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Added `EVM.HeadTracker.PersistenceEnabled` config option to disable persistence for HeadTracker. #added

ccip/config/evm/Avalanche_ANZ_testnet.toml

@@ -17,3 +17,6 @@ PriceMin = '25 gwei'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 24
+
+[HeadTracker]
+PersistenceEnabled = false

ccip/config/evm/Avalanche_Fuji.toml

@@ -19,3 +19,4 @@ BlockHistorySize = 24
 
 [HeadTracker]
 FinalityTagBypass = false
+PersistenceEnabled = false

ccip/config/evm/Avalanche_Mainnet.toml

@@ -17,3 +17,6 @@ PriceMin = '25 gwei'
 [GasEstimator.BlockHistory]
 # Average block time of 2s
 BlockHistorySize = 24
+
+[HeadTracker]
+PersistenceEnabled = false

ccip/config/evm/BSC_Testnet.toml

@@ -23,6 +23,7 @@ BlockHistorySize = 24
 HistoryDepth = 100
 SamplingInterval = '1s'
 FinalityTagBypass = false
+PersistenceEnabled = false
 
 [OCR]
 DatabaseTimeout = '2s'

ccip/config/evm/Celo_Mainnet.toml

@@ -18,3 +18,4 @@ BlockHistorySize = 12
 
 [HeadTracker]
 HistoryDepth = 50
+PersistenceEnabled = false

ccip/config/evm/Celo_Testnet.toml

@@ -18,3 +18,4 @@ BlockHistorySize = 24
 
 [HeadTracker]
 HistoryDepth = 50
+PersistenceEnabled = false

ccip/config/evm/Simulated.toml

@@ -19,6 +19,7 @@ PriceMax = '100 micro'
 HistoryDepth = 10
 MaxBufferSize = 100
 SamplingInterval = '0s'
+PersistenceEnabled = false
 
 [OCR]
 ContractConfirmations = 1

ccip/config/evm/WeMix_Mainnet.toml

@@ -14,3 +14,6 @@ ContractConfirmations = 1
 [GasEstimator]
 EIP1559DynamicFees = true
 TipCapDefault = '100 gwei'
+
+[HeadTracker]
+PersistenceEnabled = false

ccip/config/evm/WeMix_Testnet.toml

@@ -17,3 +17,4 @@ TipCapDefault = '100 gwei'
 
 [HeadTracker]
 FinalityTagBypass = false
+PersistenceEnabled = false

common/headtracker/types/config.go

@@ -15,4 +15,5 @@ type HeadTrackerConfig interface {
 	SamplingInterval() time.Duration
 	FinalityTagBypass() bool
 	MaxAllowedFinalityDepth() uint32
+	PersistenceEnabled() bool
 }

core/capabilities/ccip/ocrimpls/contract_transmitter_test.go

@@ -24,6 +24,7 @@ import (
 	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
+
 	txmgrcommon "github.com/smartcontractkit/chainlink/v2/common/txmgr"
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
@@ -541,6 +542,10 @@ func (t *TestHeadTrackerConfig) SamplingInterval() time.Duration {
 	return 1 * time.Second
 }
 
+func (t *TestHeadTrackerConfig) PersistenceEnabled() bool {
+	return true
+}
+
 var _ evmconfig.HeadTracker = (*TestHeadTrackerConfig)(nil)
 
 type TestEvmConfig struct {

core/chains/evm/config/chain_scoped_head_tracker.go

@@ -29,3 +29,7 @@ func (h *headTrackerConfig) FinalityTagBypass() bool {
 func (h *headTrackerConfig) MaxAllowedFinalityDepth() uint32 {
 	return *h.c.MaxAllowedFinalityDepth
 }
+
+func (h *headTrackerConfig) PersistenceEnabled() bool {
+	return *h.c.PersistenceEnabled
+}

core/chains/evm/config/config.go

@@ -76,6 +76,7 @@ type HeadTracker interface {
 	SamplingInterval() time.Duration
 	FinalityTagBypass() bool
 	MaxAllowedFinalityDepth() uint32
+	PersistenceEnabled() bool
 }
 
 type BalanceMonitor interface {

core/chains/evm/config/config_test.go

@@ -343,6 +343,7 @@ func TestChainScopedConfig_HeadTracker(t *testing.T) {
 	assert.Equal(t, time.Second, ht.SamplingInterval())
 	assert.Equal(t, true, ht.FinalityTagBypass())
 	assert.Equal(t, uint32(10000), ht.MaxAllowedFinalityDepth())
+	assert.Equal(t, true, ht.PersistenceEnabled())
 }
 
 func TestNodePoolConfig(t *testing.T) {

core/chains/evm/config/toml/config.go

@@ -778,6 +778,7 @@ type HeadTracker struct {
 	SamplingInterval        *commonconfig.Duration
 	MaxAllowedFinalityDepth *uint32
 	FinalityTagBypass       *bool
+	PersistenceEnabled      *bool
 }
 
 func (t *HeadTracker) setFrom(f *HeadTracker) {
@@ -796,6 +797,10 @@ func (t *HeadTracker) setFrom(f *HeadTracker) {
 	if v := f.FinalityTagBypass; v != nil {
 		t.FinalityTagBypass = v
 	}
+	if v := f.PersistenceEnabled; v != nil {
+		t.PersistenceEnabled = v
+	}
+
 }
 
 func (t *HeadTracker) ValidateConfig() (err error) {

core/chains/evm/config/toml/defaults/fallback.toml

@@ -66,6 +66,7 @@ MaxBufferSize = 3
 SamplingInterval = '1s'
 FinalityTagBypass = true
 MaxAllowedFinalityDepth = 10000
+PersistenceEnabled = true
 
 [NodePool]
 PollFailureThreshold = 5

core/chains/evm/headtracker/head_saver_test.go

@@ -42,6 +42,9 @@ func (h *headTrackerConfig) FinalityTagBypass() bool {
 func (h *headTrackerConfig) MaxAllowedFinalityDepth() uint32 {
 	return 10000
 }
+func (h *headTrackerConfig) PersistenceEnabled() bool {
+	return true
+}
 
 type config struct {
 	finalityDepth                     uint32

core/chains/evm/headtracker/head_tracker_test.go

@@ -221,9 +221,9 @@ func TestHeadTracker_Start(t *testing.T) {
 		FinalityTagEnable       *bool
 		MaxAllowedFinalityDepth *uint32
 		FinalityTagBypass       *bool
+		ORM                     headtracker.ORM
 	}
 	newHeadTracker := func(t *testing.T, opts opts) *headTrackerUniverse {
-		db := pgtest.NewSqlxDB(t)
 		config := testutils.NewTestChainScopedConfig(t, func(c *toml.EVMConfig) {
 			if opts.FinalityTagEnable != nil {
 				c.FinalityTagEnabled = opts.FinalityTagEnable
@@ -238,12 +238,15 @@ func TestHeadTracker_Start(t *testing.T) {
 				c.HeadTracker.FinalityTagBypass = opts.FinalityTagBypass
 			}
 		})
-		orm := headtracker.NewORM(*testutils.FixtureChainID, db)
+		if opts.ORM == nil {
+			db := pgtest.NewSqlxDB(t)
+			opts.ORM = headtracker.NewORM(*testutils.FixtureChainID, db)
+		}
 		ethClient := evmtest.NewEthClientMockWithDefaultChain(t)
 		mockEth := &testutils.MockEth{EthClient: ethClient}
 		sub := mockEth.NewSub(t)
 		ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(sub, nil).Maybe()
-		return createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm)
+		return createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), opts.ORM)
 	}
 	t.Run("Starts even if failed to get initialHead", func(t *testing.T) {
 		ht := newHeadTracker(t, opts{})
@@ -274,9 +277,9 @@ func TestHeadTracker_Start(t *testing.T) {
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Error handling initial head")
 	})
-	t.Run("Happy path (finality tag)", func(t *testing.T) {
+	happyPathFT := func(t *testing.T, opts opts) {
 		head := testutils.Head(1000)
-		ht := newHeadTracker(t, opts{FinalityTagEnable: ptr(true), FinalityTagBypass: ptr(false)})
+		ht := newHeadTracker(t, opts)
 		ctx := tests.Context(t)
 		require.NoError(t, ht.orm.IdempotentInsertHead(ctx, testutils.Head(799)))
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
@@ -287,8 +290,12 @@ func TestHeadTracker_Start(t *testing.T) {
 		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("backfill call to finalized failed")).Maybe()
 		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
 		ht.Start(t)
-		tests.AssertLogEventually(t, ht.observer, "Loaded chain from DB")
-	})
+		tests.AssertLogEventually(t, ht.observer, "Received new head")
+		tests.AssertEventually(t, func() bool {
+			latest := ht.headTracker.LatestChain()
+			return latest != nil && latest.Number == head.Number
+		})
+	}
 	happyPathFD := func(t *testing.T, opts opts) {
 		head := testutils.Head(1000)
 		ht := newHeadTracker(t, opts)
@@ -301,28 +308,46 @@ func TestHeadTracker_Start(t *testing.T) {
 		ht.ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(nil, errors.New("backfill call to finalized failed")).Maybe()
 		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
 		ht.Start(t)
-		tests.AssertLogEventually(t, ht.observer, "Loaded chain from DB")
+		tests.AssertLogEventually(t, ht.observer, "Received new head")
+		tests.AssertEventually(t, func() bool {
+			latest := ht.headTracker.LatestChain()
+			return latest != nil && latest.Number == head.Number
+		})
 	}
 	testCases := []struct {
 		Name string
 		Opts opts
+		Run  func(t *testing.T, opts opts)
 	}{
 		{
 			Name: "Happy path (Chain FT is disabled & HeadTracker's FT is disabled)",
 			Opts: opts{FinalityTagEnable: ptr(false), FinalityTagBypass: ptr(true)},
+			Run:  happyPathFD,
 		},
 		{
 			Name: "Happy path (Chain FT is disabled & HeadTracker's FT is enabled, but ignored)",
 			Opts: opts{FinalityTagEnable: ptr(false), FinalityTagBypass: ptr(false)},
+			Run:  happyPathFD,
 		},
 		{
 			Name: "Happy path (Chain FT is enabled & HeadTracker's FT is disabled)",
 			Opts: opts{FinalityTagEnable: ptr(true), FinalityTagBypass: ptr(true)},
+			Run:  happyPathFD,
+		},
+		{
+			Name: "Happy path (Chain FT is enabled)",
+			Opts: opts{FinalityTagEnable: ptr(true), FinalityTagBypass: ptr(false)},
+			Run:  happyPathFT,
 		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.Name, func(t *testing.T) {
-			happyPathFD(t, tc.Opts)
+			tc.Run(t, tc.Opts)
+		})
+		t.Run("Disabled Persistence "+tc.Name, func(t *testing.T) {
+			opts := tc.Opts
+			opts.ORM = headtracker.NewNullORM()
+			tc.Run(t, opts)
 		})
 	}
 }
@@ -769,7 +794,20 @@ func TestHeadTracker_SwitchesToLongestChainWithHeadSamplingDisabled(t *testing.T
 
 func TestHeadTracker_Backfill(t *testing.T) {
 	t.Parallel()
+	t.Run("Enabled Persistence", func(t *testing.T) {
+		testHeadTrackerBackfill(t, func(t *testing.T) headtracker.ORM {
+			db := pgtest.NewSqlxDB(t)
+			return headtracker.NewORM(*testutils.FixtureChainID, db)
+		})
+	})
+	t.Run("Disabled Persistence", func(t *testing.T) {
+		testHeadTrackerBackfill(t, func(t *testing.T) headtracker.ORM {
+			return headtracker.NewNullORM()
+		})
+	})
+}
 
+func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) headtracker.ORM) {
 	// Heads are arranged as follows:
 	// headN indicates an unpersisted ethereum header
 	// hN indicates a persisted head record
@@ -842,14 +880,12 @@ func TestHeadTracker_Backfill(t *testing.T) {
 			}
 		})
 
-		db := pgtest.NewSqlxDB(t)
-		orm := headtracker.NewORM(*testutils.FixtureChainID, db)
-		for i := range opts.Heads {
-			require.NoError(t, orm.IdempotentInsertHead(tests.Context(t), opts.Heads[i]))
-		}
 		ethClient := testutils.NewEthClientMock(t)
 		ethClient.On("ConfiguredChainID", mock.Anything).Return(evmcfg.EVM().ChainID(), nil)
-		ht := createHeadTracker(t, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), orm)
+		ht := createHeadTracker(t, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), newORM(t))
+		for i := range opts.Heads {
+			require.NoError(t, ht.headSaver.Save(tests.Context(t), opts.Heads[i]))
+		}
 		_, err := ht.headSaver.Load(tests.Context(t), 0)
 		require.NoError(t, err)
 		return ht
@@ -925,7 +961,7 @@ func TestHeadTracker_Backfill(t *testing.T) {
 			h = h.Parent.Load()
 		}
 
-		writtenHead, err := htu.orm.HeadByHash(tests.Context(t), head10.Hash)
+		writtenHead := htu.headSaver.Chain(head10.Hash)
 		require.NoError(t, err)
 		assert.Equal(t, int64(10), writtenHead.Number)
 	})

core/chains/evm/headtracker/orm.go

@@ -9,6 +9,7 @@ import (
 	pkgerrors "github.com/pkg/errors"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
+
 	evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 	ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big"
 )
@@ -82,3 +83,29 @@ func (orm *DbORM) HeadByHash(ctx context.Context, hash common.Hash) (head *evmty
 	}
 	return head, err
 }
+
+type nullORM struct{}
+
+func NewNullORM() ORM {
+	return &nullORM{}
+}
+
+func (orm *nullORM) IdempotentInsertHead(ctx context.Context, head *evmtypes.Head) error {
+	return nil
+}
+
+func (orm *nullORM) TrimOldHeads(ctx context.Context, minBlockNumber int64) (err error) {
+	return nil
+}
+
+func (orm *nullORM) LatestHead(ctx context.Context) (head *evmtypes.Head, err error) {
+	return nil, nil
+}
+
+func (orm *nullORM) LatestHeads(ctx context.Context, minBlockNumer int64) (heads []*evmtypes.Head, err error) {
+	return nil, nil
+}
+
+func (orm *nullORM) HeadByHash(ctx context.Context, hash common.Hash) (head *evmtypes.Head, err error) {
+	return nil, nil
+}

core/chains/legacyevm/chain.go

@@ -221,7 +221,12 @@ func newChain(ctx context.Context, cfg *evmconfig.ChainScoped, nodes []*toml.Nod
 	if !opts.AppConfig.EVMRPCEnabled() {
 		headTracker = headtracker.NullTracker
 	} else if opts.GenHeadTracker == nil {
-		orm := headtracker.NewORM(*chainID, opts.DS)
+		var orm headtracker.ORM
+		if cfg.EVM().HeadTracker().PersistenceEnabled() {
+			orm = headtracker.NewORM(*chainID, opts.DS)
+		} else {
+			orm = headtracker.NewNullORM()
+		}
 		headSaver = headtracker.NewHeadSaver(l, orm, cfg.EVM(), cfg.EVM().HeadTracker())
 		headTracker = headtracker.NewHeadTracker(l, client, cfg.EVM(), cfg.EVM().HeadTracker(), headBroadcaster, headSaver, opts.MailMon)
 	} else {

core/config/docs/chains-evm.toml

@@ -347,6 +347,11 @@ FinalityTagBypass = true # Default
 # If actual finality depth exceeds this number, HeadTracker aborts backfill and returns an error.
 # Has no effect if `FinalityTagsEnabled` = false
 MaxAllowedFinalityDepth = 10000 # Default
+# PersistenceEnabled defines whether HeadTracker needs to store heads in the database.
+# Persistence is helpful on chains with large finality depth, where fetching blocks from the latest to the latest finalized takes a lot of time.
+# On chains with fast finality, the persistence layer does not improve the chain's load time and only consumes database resources (mainly IO).
+# NOTE: persistence should not be disabled for products that use LogBroadcaster, as it might lead to missed on-chain events.
+PersistenceEnabled = true # Default
 
 [[EVM.KeySpecific]]
 # Key is the account to apply these settings to

core/services/chainlink/config_test.go

@@ -622,6 +622,7 @@ func TestConfig_Marshal(t *testing.T) {
 					SamplingInterval:        &hour,
 					FinalityTagBypass:       ptr[bool](false),
 					MaxAllowedFinalityDepth: ptr[uint32](1500),
+					PersistenceEnabled:      ptr(false),
 				},
 
 				NodePool: evmcfg.NodePool{
@@ -1101,6 +1102,7 @@ MaxBufferSize = 17
 SamplingInterval = '1h0m0s'
 MaxAllowedFinalityDepth = 1500
 FinalityTagBypass = false
+PersistenceEnabled = false
 
 [[EVM.KeySpecific]]
 Key = '0x2a3e23c6f242F5345320814aC8a1b4E58707D292'

core/services/chainlink/testdata/config-full.toml

@@ -386,6 +386,7 @@ MaxBufferSize = 17
 SamplingInterval = '1h0m0s'
 MaxAllowedFinalityDepth = 1500
 FinalityTagBypass = false
+PersistenceEnabled = false
 
 [[EVM.KeySpecific]]
 Key = '0x2a3e23c6f242F5345320814aC8a1b4E58707D292'