Merge branch 'main' into mock_gen_fix

.github/workflows/ci.yml

@@ -17,7 +17,11 @@ on:
 
 concurrency:
   group: ${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+  cancel-in-progress: ${{ !contains(fromJSON('[
+      "refs/heads/release/2.60",
+      "refs/heads/release/2.61",
+      "refs/heads/main"
+    ]'), github.ref) }}
 
 jobs:
   tests:

.github/workflows/qa-constrained-tip-tracking.yml

@@ -2,7 +2,7 @@ name: QA - Constrained Tip tracking
 
 on:
   schedule:
-    - cron: '0 0 * * 1-6'  # Run every night at 00:00 AM UTC except Sunday
+    - cron: '0 0 * * 0'  # Run on Sunday at 00:00 AM UTC
   workflow_dispatch:     # Run manually
   pull_request:
     branches:

.github/workflows/qa-rpc-integration-tests.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Checkout RPC Tests Repository & Install Requirements
         run: |
           rm -rf ${{ runner.workspace }}/rpc-tests
-          git -c advice.detachedHead=false clone --depth 1 --branch v0.42.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          git -c advice.detachedHead=false clone --depth 1 --branch v1.0.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
           cd ${{ runner.workspace }}/rpc-tests
           pip3 install -r requirements.txt
 
@@ -191,8 +191,6 @@ jobs:
           trace_replayTransaction/test_23.tar,\
           trace_replayTransaction/test_24.json,\
           trace_replayTransaction/test_29.tar,\
-          ots_getTransactionBySenderAndNonce/test_05.json,\
-          ots_getTransactionBySenderAndNonce/test_11.json,\
           ots_searchTransactionsAfter/test_01.json,\
           ots_searchTransactionsAfter/test_03.json,\
           ots_searchTransactionsAfter/test_04.json,\
@@ -267,7 +265,6 @@ jobs:
           ots_getBlockTransactions/test_03.json,\
           ots_getBlockTransactions/test_04.json,\
           ots_getBlockTransactions/test_05.json,\
-          ots_hasCode/test_09.json,\
           ots_searchTransactionsAfter/test_14.json,\
           ots_searchTransactionsBefore/test_13.json,\
           ots_searchTransactionsBefore/test_14.json,\
@@ -280,7 +277,19 @@ jobs:
           trace_call/test_18.json,\
           txpool_content/test_01.json,\
           txpool_status/test_1.json,\
-          web3_clientVersion/test_1.json
+          web3_clientVersion/test_1.json,\
+          eth_estimateGas/test_14.json,\
+          trace_replayBlockTransactions/test_26.tar,\
+          trace_replayBlockTransactions/test_28.tar,\
+          trace_replayBlockTransactions/test_29.tar,\
+          trace_replayBlockTransactions/test_31.tar,\
+          trace_replayBlockTransactions/test_32.tar,\
+          trace_replayBlockTransactions/test_33.tar,\
+          trace_replayBlockTransactions/test_34.tar,\
+          trace_replayBlockTransactions/test_35.tar,\
+          trace_replayTransaction/test_31.json,\
+          trace_replayTransaction/test_32.json,\
+          trace_replayTransaction/test_34.json
           
           # Capture test runner script exit status
           test_exit_status=$?

.github/workflows/qa-rpc-test-bisection-tool.yml

@@ -36,6 +36,13 @@ jobs:
       - name: Create scripts directory
         run: mkdir -p $GITHUB_WORKSPACE/.github/scripts
 
+      - name: Checkout RPC Tests Repository & Install Requirements
+        run: |
+          rm -rf $GITHUB_WORKSPACE/rpc-tests
+          git -c advice.detachedHead=false clone --depth 1 --branch v1.00.0 https://github.com/erigontech/rpc-tests $GITHUB_WORKSPACE/rpc-tests
+          cd $GITHUB_WORKSPACE/rpc-tests
+          pip3 install -r requirements.txt
+
       - name: Create test script
         run: |
           cat << 'EOF' > $GITHUB_WORKSPACE/.github/scripts/test_script.sh
@@ -75,12 +82,6 @@ jobs:
             exit 125  # Skip this commit
           fi
 
-          # Checkout RPC Tests Repository & Install Requirements
-          rm -rf $GITHUB_WORKSPACE/rpc-tests
-          git -c advice.detachedHead=false clone --depth 1 --branch v0.42.0 https://github.com/erigontech/rpc-tests $GITHUB_WORKSPACE/rpc-tests
-          cd $GITHUB_WORKSPACE/rpc-tests
-          pip3 install -r requirements.txt
-
           # Run the specified test
           cd $GITHUB_WORKSPACE/rpc-tests/integration
 

cmd/rpcdaemon/cli/config.go

@@ -73,6 +73,7 @@ import (
 	"github.com/erigontech/erigon/node"
 	"github.com/erigontech/erigon/node/nodecfg"
 	"github.com/erigontech/erigon/polygon/bor"
+	"github.com/erigontech/erigon/polygon/bor/borcfg"
 	"github.com/erigontech/erigon/polygon/bor/valset"
 	"github.com/erigontech/erigon/polygon/bridge"
 	"github.com/erigontech/erigon/polygon/heimdall"
@@ -532,12 +533,12 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger
 				}
 
 				heimdallConfig := heimdall.ReaderConfig{
-					Ctx:                     ctx,
-					CalculateSprintNumberFn: cc.Bor.CalculateSprintNumber,
-					DataDir:                 cfg.DataDir,
-					TempDir:                 cfg.Dirs.Tmp,
-					Logger:                  logger,
-					RoTxLimit:               roTxLimit,
+					Ctx:       ctx,
+					BorConfig: cc.Bor.(*borcfg.BorConfig),
+					DataDir:   cfg.DataDir,
+					TempDir:   cfg.Dirs.Tmp,
+					Logger:    logger,
+					RoTxLimit: roTxLimit,
 				}
 				heimdallReader, err = heimdall.AssembleReader(heimdallConfig)
 				if err != nil {

eth/backend.go

@@ -562,12 +562,12 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 			polygonBridge = bridge.Assemble(bridgeConfig)
 
 			heimdallConfig := heimdall.ServiceConfig{
-				CalculateSprintNumberFn: borConfig.CalculateSprintNumber,
-				HeimdallURL:             config.HeimdallURL,
-				DataDir:                 dirs.DataDir,
-				TempDir:                 tmpdir,
-				Logger:                  logger,
-				RoTxLimit:               roTxLimit,
+				BorConfig:   borConfig,
+				HeimdallURL: config.HeimdallURL,
+				DataDir:     dirs.DataDir,
+				TempDir:     tmpdir,
+				Logger:      logger,
+				RoTxLimit:   roTxLimit,
 			}
 			heimdallService = heimdall.AssembleService(heimdallConfig)
 

eth/stagedsync/exec3.go

@@ -30,10 +30,11 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
-	"github.com/erigontech/erigon/core/rawdb/rawtemporaldb"
 	"github.com/erigontech/mdbx-go/mdbx"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/erigontech/erigon/core/rawdb/rawtemporaldb"
+
 	"github.com/erigontech/erigon-lib/chain"
 	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/cmp"
@@ -912,7 +913,7 @@ Loop:
 					return err
 				}
 				logger.Warn(fmt.Sprintf("[%s] Execution failed", execStage.LogPrefix()), "block", blockNum, "txNum", txTask.TxNum, "hash", header.Hash().String(), "err", err)
-				if cfg.hd != nil && errors.Is(err, consensus.ErrInvalidBlock) {
+				if cfg.hd != nil && cfg.hd.POSSync() && errors.Is(err, consensus.ErrInvalidBlock) {
 					cfg.hd.ReportBadHeaderPoS(header.Hash(), header.ParentHash)
 				}
 				if cfg.badBlockHalt {
@@ -1226,7 +1227,7 @@ func flushAndCheckCommitmentV3(ctx context.Context, header *types.Header, applyT
 	if cfg.badBlockHalt {
 		return false, errors.New("wrong trie root")
 	}
-	if cfg.hd != nil {
+	if cfg.hd != nil && cfg.hd.POSSync() {
 		cfg.hd.ReportBadHeaderPoS(header.Hash(), header.ParentHash)
 	}
 	minBlockNum := e.BlockNumber

eth/stagedsync/stage_execute.go

@@ -63,6 +63,7 @@ const (
 
 type headerDownloader interface {
 	ReportBadHeaderPoS(badHeader, lastValidAncestor common.Hash)
+	POSSync() bool
 }
 
 type ExecuteBlockCfg struct {

eth/stagedsync/stage_polygon_sync.go

@@ -98,8 +98,8 @@ func NewPolygonSyncStageCfg(
 		txActionStream: txActionStream,
 	}
 	borConfig := chainConfig.Bor.(*borcfg.BorConfig)
-	heimdallReader := heimdall.NewReader(borConfig.CalculateSprintNumber, heimdallStore, logger)
-	heimdallService := heimdall.NewService(borConfig.CalculateSprintNumber, heimdallClient, heimdallStore, logger, heimdallReader)
+	heimdallReader := heimdall.NewReader(borConfig, heimdallStore, logger)
+	heimdallService := heimdall.NewService(borConfig, heimdallClient, heimdallStore, logger, heimdallReader)
 	bridgeService := bridge.NewBridge(bridgeStore, logger, borConfig, heimdallClient, nil)
 	p2pService := p2p.NewService(maxPeers, logger, sentry, statusDataProvider.GetStatusData)
 	checkpointVerifier := polygonsync.VerifyCheckpointHeaders

eth/stagedsync/stage_senders.go

@@ -282,7 +282,7 @@ Loop:
 			return minBlockErr
 		}
 		minHeader := rawdb.ReadHeader(tx, minBlockHash, minBlockNum)
-		if cfg.hd != nil && errors.Is(minBlockErr, consensus.ErrInvalidBlock) {
+		if cfg.hd != nil && cfg.hd.POSSync() && errors.Is(minBlockErr, consensus.ErrInvalidBlock) {
 			cfg.hd.ReportBadHeaderPoS(minBlockHash, minHeader.ParentHash)
 		}
 

polygon/bridge/db.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/big"
 	"time"
 
 	libcommon "github.com/erigontech/erigon-lib/common"
@@ -217,12 +218,9 @@ func (s *MdbxStore) PutEventTxnToBlockNum(ctx context.Context, eventTxnToBlockNu
 	}
 	defer tx.Rollback()
 
-	vByte := make([]byte, 8)
-
+	vBigNum := new(big.Int)
 	for k, v := range eventTxnToBlockNum {
-		binary.BigEndian.PutUint64(vByte, v)
-
-		err = tx.Put(kv.BorTxLookup, k.Bytes(), vByte)
+		err = tx.Put(kv.BorTxLookup, k.Bytes(), vBigNum.SetUint64(v).Bytes())
 		if err != nil {
 			return err
 		}
@@ -248,7 +246,7 @@ func (s *MdbxStore) EventTxnToBlockNum(ctx context.Context, borTxHash libcommon.
 		return blockNum, false, nil
 	}
 
-	blockNum = binary.BigEndian.Uint64(v)
+	blockNum = new(big.Int).SetBytes(v).Uint64()
 	return blockNum, true, nil
 }
 
@@ -474,7 +472,7 @@ func Unwind(tx kv.RwTx, blockNum uint64) error {
 	return UnwindEventTxnToBlockNum(tx, blockNum)
 }
 
-// UnwindEventProcessedBlocks deletes data in kv.BorEventProcessedBlocks.
+// UnwindBlockNumToEventID deletes data in kv.BorEventProcessedBlocks.
 // The blockNum parameter is exclusive, i.e. only data in the range (blockNum, last] is deleted.
 func UnwindBlockNumToEventID(tx kv.RwTx, blockNum uint64) error {
 	c, err := tx.RwCursor(kv.BorEventNums)
@@ -532,7 +530,7 @@ func UnwindEventProcessedBlocks(tx kv.RwTx, blockNum uint64) error {
 	return err
 }
 
-// UnwindEventProcessedBlocks deletes data in kv.BorTxLookup.
+// UnwindEventTxnToBlockNum deletes data in kv.BorTxLookup.
 // The blockNum parameter is exclusive, i.e. only data in the range (blockNum, last] is deleted.
 func UnwindEventTxnToBlockNum(tx kv.RwTx, blockNum uint64) error {
 	c, err := tx.RwCursor(kv.BorTxLookup)
@@ -541,11 +539,10 @@ func UnwindEventTxnToBlockNum(tx kv.RwTx, blockNum uint64) error {
 	}
 
 	defer c.Close()
-	blockNumBytes := make([]byte, 8)
-	binary.BigEndian.PutUint64(blockNumBytes, blockNum)
+	blockNumBig := new(big.Int)
 	var k, v []byte
 	for k, v, err = c.Last(); err == nil && k != nil; k, v, err = c.Prev() {
-		if currentBlockNum := binary.BigEndian.Uint64(v); currentBlockNum <= blockNum {
+		if currentBlockNum := blockNumBig.SetBytes(v).Uint64(); currentBlockNum <= blockNum {
 			break
 		}
 

polygon/heimdall/reader.go

@@ -10,6 +10,7 @@ import (
 	"github.com/erigontech/erigon-lib/gointerfaces"
 	remote "github.com/erigontech/erigon-lib/gointerfaces/remoteproto"
 	"github.com/erigontech/erigon-lib/log/v3"
+	"github.com/erigontech/erigon/polygon/bor/borcfg"
 	"github.com/erigontech/erigon/polygon/bor/valset"
 )
 
@@ -20,12 +21,12 @@ type Reader struct {
 }
 
 type ReaderConfig struct {
-	Ctx                     context.Context
-	CalculateSprintNumberFn CalculateSprintNumberFunc
-	DataDir                 string
-	TempDir                 string
-	Logger                  log.Logger
-	RoTxLimit               int64
+	Ctx       context.Context
+	BorConfig *borcfg.BorConfig
+	DataDir   string
+	TempDir   string
+	Logger    log.Logger
+	RoTxLimit int64
 }
 
 // AssembleReader creates and opens the MDBX store. For use cases where the store is only being read from. Must call Close.
@@ -37,14 +38,14 @@ func AssembleReader(config ReaderConfig) (*Reader, error) {
 		return nil, err
 	}
 
-	return NewReader(config.CalculateSprintNumberFn, store, config.Logger), nil
+	return NewReader(config.BorConfig, store, config.Logger), nil
 }
 
-func NewReader(calculateSprintNumber CalculateSprintNumberFunc, store ServiceStore, logger log.Logger) *Reader {
+func NewReader(borConfig *borcfg.BorConfig, store ServiceStore, logger log.Logger) *Reader {
 	return &Reader{
 		logger:                    logger,
 		store:                     store,
-		spanBlockProducersTracker: newSpanBlockProducersTracker(logger, calculateSprintNumber, store.SpanBlockProducerSelections()),
+		spanBlockProducersTracker: newSpanBlockProducersTracker(logger, borConfig, store.SpanBlockProducerSelections()),
 	}
 }
 

polygon/heimdall/service.go

@@ -26,17 +26,18 @@ import (
 
 	libcommon "github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/log/v3"
+	"github.com/erigontech/erigon/polygon/bor/borcfg"
 	"github.com/erigontech/erigon/polygon/bor/valset"
 	"github.com/erigontech/erigon/polygon/polygoncommon"
 )
 
 type ServiceConfig struct {
-	CalculateSprintNumberFn CalculateSprintNumberFunc
-	HeimdallURL             string
-	DataDir                 string
-	TempDir                 string
-	Logger                  log.Logger
-	RoTxLimit               int64
+	BorConfig   *borcfg.BorConfig
+	HeimdallURL string
+	DataDir     string
+	TempDir     string
+	Logger      log.Logger
+	RoTxLimit   int64
 }
 
 type Service interface {
@@ -64,15 +65,15 @@ type service struct {
 func AssembleService(config ServiceConfig) Service {
 	store := NewMdbxServiceStore(config.Logger, config.DataDir, config.TempDir, config.RoTxLimit)
 	client := NewHeimdallClient(config.HeimdallURL, config.Logger)
-	reader := NewReader(config.CalculateSprintNumberFn, store, config.Logger)
-	return NewService(config.CalculateSprintNumberFn, client, store, config.Logger, reader)
+	reader := NewReader(config.BorConfig, store, config.Logger)
+	return NewService(config.BorConfig, client, store, config.Logger, reader)
 }
 
-func NewService(calculateSprintNumberFn CalculateSprintNumberFunc, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) Service {
-	return newService(calculateSprintNumberFn, client, store, logger, reader)
+func NewService(borConfig *borcfg.BorConfig, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) Service {
+	return newService(borConfig, client, store, logger, reader)
 }
 
-func newService(calculateSprintNumberFn CalculateSprintNumberFunc, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) *service {
+func newService(borConfig *borcfg.BorConfig, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) *service {
 	checkpointFetcher := newCheckpointFetcher(client, logger)
 	milestoneFetcher := newMilestoneFetcher(client, logger)
 	spanFetcher := newSpanFetcher(client, logger)
@@ -119,7 +120,7 @@ func newService(calculateSprintNumberFn CalculateSprintNumberFunc, client Heimda
 		checkpointScraper:         checkpointScraper,
 		milestoneScraper:          milestoneScraper,
 		spanScraper:               spanScraper,
-		spanBlockProducersTracker: newSpanBlockProducersTracker(logger, calculateSprintNumberFn, store.SpanBlockProducerSelections()),
+		spanBlockProducersTracker: newSpanBlockProducersTracker(logger, borConfig, store.SpanBlockProducerSelections()),
 	}
 }
 

polygon/heimdall/service_test.go

@@ -103,7 +103,9 @@ func TestServiceWithMainnetData(t *testing.T) {
 			14_000_000,
 			14_250_000,
 			14_300_000,
-			14_323_456, // span 2239 start
+			14_323_456, // span 2239 start (sprint 1 of span 2239)
+			14_323_520, // span 2239 start (sprint 2 of span 2239) - to test recent producers lru caching
+			14_323_584, // span 2239 start (sprint 3 of span 2239) - to test recent producers lru caching
 			14_325_000,
 			14_329_854,
 			14_329_855, // span 2239 end
@@ -161,8 +163,8 @@ func (suite *ServiceTestSuite) SetupSuite() {
 	suite.setupSpans()
 	suite.setupCheckpoints()
 	suite.setupMilestones()
-	reader := NewReader(borConfig.CalculateSprintNumber, store, logger)
-	suite.service = newService(borConfig.CalculateSprintNumber, suite.client, store, logger, reader)
+	reader := NewReader(borConfig, store, logger)
+	suite.service = newService(borConfig, suite.client, store, logger, reader)
 
 	err := suite.service.store.Prepare(suite.ctx)
 	require.NoError(suite.T(), err)