diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index de562c6f50..aaba8d895c 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -32,6 +32,7 @@ import ( "errors" "fmt" "io" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -372,24 +373,65 @@ func (abi *ABI) HasReceive() bool { // revertSelector is a special function selector for revert reason unpacking. var revertSelector = crypto.Keccak256([]byte("Error(string)"))[:4] +// panicSelector is a special function selector for panic reason unpacking. +var panicSelector = crypto.Keccak256([]byte("Panic(uint256)"))[:4] + +// panicReasons map is for readable panic codes +// see this linkage for the deails +// https://docs.soliditylang.org/en/v0.8.21/control-structures.html#panic-via-assert-and-error-via-require +// the reason string list is copied from ether.js +// https://github.com/ethers-io/ethers.js/blob/fa3a883ff7c88611ce766f58bdd4b8ac90814470/src.ts/abi/interface.ts#L207-L218 +var panicReasons = map[uint64]string{ + 0x00: "generic panic", + 0x01: "assert(false)", + 0x11: "arithmetic underflow or overflow", + 0x12: "division or modulo by zero", + 0x21: "enum overflow", + 0x22: "invalid encoded storage byte array accessed", + 0x31: "out-of-bounds array access; popping on an empty array", + 0x32: "out-of-bounds access of an array or bytesN", + 0x41: "out of memory", + 0x51: "uninitialized function", +} + // UnpackRevert resolves the abi-encoded revert reason. According to the solidity // spec https://solidity.readthedocs.io/en/latest/control-structures.html#revert, -// the provided revert reason is abi-encoded as if it were a call to a function -// `Error(string)`. So it's a special tool for it. +// the provided revert reason is abi-encoded as if it were a call to function +// `Error(string)` or `Panic(uint256)`. So it's a special tool for it. func UnpackRevert(data []byte) (string, error) { if len(data) < 4 { return "", errors.New("invalid data for unpacking") } - if !bytes.Equal(data[:4], revertSelector) { + switch { + case bytes.Equal(data[:4], revertSelector): + typ, err := NewType("string", "", nil) + if err != nil { + return "", err + } + unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:]) + if err != nil { + return "", err + } + return unpacked[0].(string), nil + case bytes.Equal(data[:4], panicSelector): + typ, err := NewType("uint256", "", nil) + if err != nil { + return "", err + } + unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:]) + if err != nil { + return "", err + } + pCode := unpacked[0].(*big.Int) + // uint64 safety check for future + // but the code is not bigger than MAX(uint64) now + if pCode.IsUint64() { + if reason, ok := panicReasons[pCode.Uint64()]; ok { + return reason, nil + } + } + return fmt.Sprintf("unknown panic code: %#x", pCode), nil + default: return "", errors.New("invalid data for unpacking") } - typ, err := NewType("string", "", nil) - if err != nil { - return "", err - } - unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:]) - if err != nil { - return "", err - } - return unpacked[0].(string), nil } diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 669fe2e69c..c81a82d1d7 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -1184,6 +1184,8 @@ func TestUnpackRevert(t *testing.T) { {"", "", errors.New("invalid data for unpacking")}, {"08c379a1", "", errors.New("invalid data for unpacking")}, {"08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d72657665727420726561736f6e00000000000000000000000000000000000000", "revert reason", nil}, + {"4e487b710000000000000000000000000000000000000000000000000000000000000000", "generic panic", nil}, + {"4e487b7100000000000000000000000000000000000000000000000000000000000000ff", "unknown panic code: 0xff", nil}, } for index, c := range cases { t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) { diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index d401e086fe..d7c0fa5103 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -235,7 +235,6 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, if err != nil { return nil, err } - return stateDB.GetCode(contract), nil } @@ -248,7 +247,6 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres if err != nil { return nil, err } - return stateDB.GetBalance(contract), nil } @@ -261,7 +259,6 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, if err != nil { return 0, err } - return stateDB.GetNonce(contract), nil } @@ -274,7 +271,6 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres if err != nil { return nil, err } - val := stateDB.GetState(contract, key) return val[:], nil } @@ -739,8 +735,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa if err != nil { return err } - stateDB, _ := b.blockchain.State() - + stateDB, err := b.blockchain.State() + if err != nil { + return err + } b.acceptedBlock = blocks[0] b.acceptedState, _ = state.New(b.acceptedBlock.Root(), stateDB.Database(), nil) return nil @@ -858,11 +856,12 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { blocks, _, _ := core.GenerateChain(b.config, block, dummy.NewFaker(), b.database, 1, 10, func(number int, block *core.BlockGen) { block.OffsetTime(int64(adjustment.Seconds())) }) - stateDB, _ := b.blockchain.State() - + stateDB, err := b.blockchain.State() + if err != nil { + return err + } b.acceptedBlock = blocks[0] b.acceptedState, _ = state.New(b.acceptedBlock.Root(), stateDB.Database(), nil) - return nil } diff --git a/accounts/abi/method.go b/accounts/abi/method.go index df8d2f6df0..f4cf0a55f3 100644 --- a/accounts/abi/method.go +++ b/accounts/abi/method.go @@ -137,11 +137,12 @@ func NewMethod(name string, rawName string, funType FunctionType, mutability str state = state + " " } identity := fmt.Sprintf("function %v", rawName) - if funType == Fallback { + switch funType { + case Fallback: identity = "fallback" - } else if funType == Receive { + case Receive: identity = "receive" - } else if funType == Constructor { + case Constructor: identity = "constructor" } str := fmt.Sprintf("%v(%v) %sreturns(%v)", identity, strings.Join(inputNames, ", "), state, strings.Join(outputNames, ", ")) diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go index ace22edfee..25c43e1995 100644 --- a/accounts/abi/method_test.go +++ b/accounts/abi/method_test.go @@ -94,11 +94,12 @@ func TestMethodString(t *testing.T) { for _, test := range table { var got string - if test.method == "fallback" { + switch test.method { + case "fallback": got = abi.Fallback.String() - } else if test.method == "receive" { + case "receive": got = abi.Receive.String() - } else { + default: got = abi.Methods[test.method].String() } if got != test.expectation { diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 2899e5a5b3..b975cb8802 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -170,13 +170,14 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) // this value will become our slice or our array, depending on the type var refSlice reflect.Value - if t.T == SliceTy { + switch t.T { + case SliceTy: // declare our slice refSlice = reflect.MakeSlice(t.GetType(), size, size) - } else if t.T == ArrayTy { + case ArrayTy: // declare our array refSlice = reflect.New(t.GetType()).Elem() - } else { + default: return nil, errors.New("abi: invalid type in array/slice unpacking stage") } diff --git a/accounts/keystore/watch.go b/accounts/keystore/watch.go index e84e4c8546..93f086a9a5 100644 --- a/accounts/keystore/watch.go +++ b/accounts/keystore/watch.go @@ -30,6 +30,7 @@ package keystore import ( + "os" "time" "github.com/ethereum/go-ethereum/log" @@ -87,7 +88,9 @@ func (w *watcher) loop() { } defer watcher.Close() if err := watcher.Add(w.ac.keydir); err != nil { - logger.Warn("Failed to watch keystore folder", "err", err) + if !os.IsNotExist(err) { + logger.Warn("Failed to watch keystore folder", "err", err) + } return } diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 7b33af0d02..c8adefd055 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -76,25 +76,26 @@ type ommer struct { //go:generate go run github.com/fjl/gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go type stEnv struct { - Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` - Difficulty *big.Int `json:"currentDifficulty"` - Random *big.Int `json:"currentRandom"` - ParentDifficulty *big.Int `json:"parentDifficulty"` - ParentBaseFee *big.Int `json:"parentBaseFee,omitempty"` - ParentGasUsed uint64 `json:"parentGasUsed,omitempty"` - ParentGasLimit uint64 `json:"parentGasLimit,omitempty"` - MinBaseFee *big.Int `json:"minBaseFee,omitempty"` - GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` - Number uint64 `json:"currentNumber" gencodec:"required"` - Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` - ParentTimestamp uint64 `json:"parentTimestamp,omitempty"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *big.Int `json:"currentBaseFee,omitempty"` - ParentUncleHash common.Hash `json:"parentUncleHash"` - ExcessBlobGas *uint64 `json:"excessBlobGas,omitempty"` - ParentExcessBlobGas *uint64 `json:"parentExcessBlobGas,omitempty"` - ParentBlobGasUsed *uint64 `json:"parentBlobGasUsed,omitempty"` + Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` + Difficulty *big.Int `json:"currentDifficulty"` + Random *big.Int `json:"currentRandom"` + ParentDifficulty *big.Int `json:"parentDifficulty"` + ParentBaseFee *big.Int `json:"parentBaseFee,omitempty"` + ParentGasUsed uint64 `json:"parentGasUsed,omitempty"` + ParentGasLimit uint64 `json:"parentGasLimit,omitempty"` + MinBaseFee *big.Int `json:"minBaseFee,omitempty"` + GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` + Number uint64 `json:"currentNumber" gencodec:"required"` + Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp uint64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *big.Int `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` + ExcessBlobGas *uint64 `json:"excessBlobGas,omitempty"` + ParentExcessBlobGas *uint64 `json:"parentExcessBlobGas,omitempty"` + ParentBlobGasUsed *uint64 `json:"parentBlobGasUsed,omitempty"` + ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` } type stEnvMarshaling struct { @@ -191,6 +192,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 { // misc.ApplyDAOHardFork(statedb) // } + if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { + evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + } var blobGasUsed uint64 for i, tx := range txs { if tx.Type() == types.BlobTxType && vmContext.ExcessBlobGas == nil { diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index ed8652d741..739545bcbd 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -16,25 +16,26 @@ var _ = (*stEnvMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { - Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` - Random *math.HexOrDecimal256 `json:"currentRandom"` - ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` - ParentBaseFee *math.HexOrDecimal256 `json:"parentBaseFee,omitempty"` - ParentGasUsed math.HexOrDecimal64 `json:"parentGasUsed,omitempty"` - ParentGasLimit math.HexOrDecimal64 `json:"parentGasLimit,omitempty"` - MinBaseFee *math.HexOrDecimal256 `json:"minBaseFee,omitempty"` - GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - ParentTimestamp math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` - ParentUncleHash common.Hash `json:"parentUncleHash"` - ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas,omitempty"` - ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` - ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` + Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + Random *math.HexOrDecimal256 `json:"currentRandom"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + ParentBaseFee *math.HexOrDecimal256 `json:"parentBaseFee,omitempty"` + ParentGasUsed math.HexOrDecimal64 `json:"parentGasUsed,omitempty"` + ParentGasLimit math.HexOrDecimal64 `json:"parentGasLimit,omitempty"` + MinBaseFee *math.HexOrDecimal256 `json:"minBaseFee,omitempty"` + GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` + ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas,omitempty"` + ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` + ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` + ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) @@ -56,31 +57,33 @@ func (s stEnv) MarshalJSON() ([]byte, error) { enc.ExcessBlobGas = (*math.HexOrDecimal64)(s.ExcessBlobGas) enc.ParentExcessBlobGas = (*math.HexOrDecimal64)(s.ParentExcessBlobGas) enc.ParentBlobGasUsed = (*math.HexOrDecimal64)(s.ParentBlobGasUsed) + enc.ParentBeaconBlockRoot = s.ParentBeaconBlockRoot return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { - Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` - Random *math.HexOrDecimal256 `json:"currentRandom"` - ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` - ParentBaseFee *math.HexOrDecimal256 `json:"parentBaseFee,omitempty"` - ParentGasUsed *math.HexOrDecimal64 `json:"parentGasUsed,omitempty"` - ParentGasLimit *math.HexOrDecimal64 `json:"parentGasLimit,omitempty"` - MinBaseFee *math.HexOrDecimal256 `json:"minBaseFee,omitempty"` - GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - ParentTimestamp *math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` - ParentUncleHash *common.Hash `json:"parentUncleHash"` - ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas,omitempty"` - ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` - ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` + Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + Random *math.HexOrDecimal256 `json:"currentRandom"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + ParentBaseFee *math.HexOrDecimal256 `json:"parentBaseFee,omitempty"` + ParentGasUsed *math.HexOrDecimal64 `json:"parentGasUsed,omitempty"` + ParentGasLimit *math.HexOrDecimal64 `json:"parentGasLimit,omitempty"` + MinBaseFee *math.HexOrDecimal256 `json:"minBaseFee,omitempty"` + GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp *math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash *common.Hash `json:"parentUncleHash"` + ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas,omitempty"` + ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` + ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` + ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -147,5 +150,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.ParentBlobGasUsed != nil { s.ParentBlobGasUsed = (*uint64)(dec.ParentBlobGasUsed) } + if dec.ParentBeaconBlockRoot != nil { + s.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot + } return nil } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 50c40399cc..3f75960bb0 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -202,93 +202,21 @@ func Transition(ctx *cli.Context) error { // Set the chain id chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name)) - var txsWithKeys []*txWithKey - if txStr != stdinSelector { - inFile, err := os.Open(txStr) - if err != nil { - return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err)) - } - defer inFile.Close() - decoder := json.NewDecoder(inFile) - if strings.HasSuffix(txStr, ".rlp") { - var body hexutil.Bytes - if err := decoder.Decode(&body); err != nil { - return err - } - var txs types.Transactions - if err := rlp.DecodeBytes(body, &txs); err != nil { - return err - } - for _, tx := range txs { - txsWithKeys = append(txsWithKeys, &txWithKey{ - key: nil, - tx: tx, - }) - } - } else { - if err := decoder.Decode(&txsWithKeys); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) - } - } - } else { - if len(inputData.TxRlp) > 0 { - // Decode the body of already signed transactions - body := common.FromHex(inputData.TxRlp) - var txs types.Transactions - if err := rlp.DecodeBytes(body, &txs); err != nil { - return err - } - for _, tx := range txs { - txsWithKeys = append(txsWithKeys, &txWithKey{ - key: nil, - tx: tx, - }) - } - } else { - // JSON encoded transactions - txsWithKeys = inputData.Txs - } - } - // We may have to sign the transactions. - signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp) - - if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil { - return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err)) + if txs, err = loadTransactions(txStr, inputData, prestate.Env, chainConfig); err != nil { + return err } - // Sanity check, to not `panic` in state_transition - // NOTE: IsLondon replaced with IsSubnetEVM here - if chainConfig.IsSubnetEVM(prestate.Env.Timestamp) { - if prestate.Env.BaseFee != nil { - // Already set, base fee has precedent over parent base fee. - } else if prestate.Env.ParentBaseFee != nil && prestate.Env.Number != 0 { - parent := &types.Header{ - Number: new(big.Int).SetUint64(prestate.Env.Number - 1), - Time: prestate.Env.ParentTimestamp, - BaseFee: prestate.Env.ParentBaseFee, - GasUsed: prestate.Env.ParentGasUsed, - GasLimit: prestate.Env.ParentGasLimit, - Extra: make([]byte, params.DynamicFeeExtraDataSize), // TODO: consider passing extra through env - } - feeConfig := params.DefaultFeeConfig - if prestate.Env.MinBaseFee != nil { - // Override the default min base fee if it's set in the env - feeConfig.MinBaseFee = prestate.Env.MinBaseFee - } - _, prestate.Env.BaseFee, err = dummy.CalcBaseFee(chainConfig, feeConfig, parent, prestate.Env.Timestamp) - if err != nil { - return NewError(ErrorConfig, fmt.Errorf("failed calculating base fee: %v", err)) - } - } else { - return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) - } + if err := applyLondonChecks(&prestate.Env, chainConfig); err != nil { + return err } // NOTE: Removed isMerged logic here. - // isMerged := chainConfig.TerminalTotalDifficulty != nil && chainConfig.TerminalTotalDifficulty.BitLen() == 0 if prestate.Env.Random != nil { // NOTE: subnet-evm continues to return the difficulty value for the RANDOM opcode, // so for testing if Random is set in the environment, we copy it to difficulty instead. prestate.Env.Difficulty = prestate.Env.Random } + if err := applyCancunChecks(&prestate.Env, chainConfig); err != nil { + return err + } // Run the test and aggregate the result s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { @@ -355,33 +283,118 @@ func (t *txWithKey) UnmarshalJSON(input []byte) error { // and secondly to read them with the standard tx json format func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) { var signedTxs []*types.Transaction - for i, txWithKey := range txs { - tx := txWithKey.tx - key := txWithKey.key - v, r, s := tx.RawSignatureValues() - if key != nil && v.BitLen()+r.BitLen()+s.BitLen() == 0 { - // This transaction needs to be signed - var ( - signed *types.Transaction - err error - ) - if txWithKey.protected { - signed, err = types.SignTx(tx, signer, key) - } else { - signed, err = types.SignTx(tx, types.FrontierSigner{}, key) - } - if err != nil { - return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err)) - } - signedTxs = append(signedTxs, signed) - } else { + for i, tx := range txs { + var ( + v, r, s = tx.tx.RawSignatureValues() + signed *types.Transaction + err error + ) + if tx.key == nil || v.BitLen()+r.BitLen()+s.BitLen() != 0 { // Already signed - signedTxs = append(signedTxs, tx) + signedTxs = append(signedTxs, tx.tx) + continue + } + // This transaction needs to be signed + if tx.protected { + signed, err = types.SignTx(tx.tx, signer, tx.key) + } else { + signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key) + } + if err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err)) } + signedTxs = append(signedTxs, signed) } return signedTxs, nil } +func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *params.ChainConfig) (types.Transactions, error) { + var txsWithKeys []*txWithKey + var signed types.Transactions + if txStr != stdinSelector { + data, err := os.ReadFile(txStr) + if err != nil { + return nil, NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err)) + } + if strings.HasSuffix(txStr, ".rlp") { // A file containing an rlp list + var body hexutil.Bytes + if err := json.Unmarshal(data, &body); err != nil { + return nil, err + } + // Already signed transactions + if err := rlp.DecodeBytes(body, &signed); err != nil { + return nil, err + } + return signed, nil + } + if err := json.Unmarshal(data, &txsWithKeys); err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err)) + } + } else { + if len(inputData.TxRlp) > 0 { + // Decode the body of already signed transactions + body := common.FromHex(inputData.TxRlp) + // Already signed transactions + if err := rlp.DecodeBytes(body, &signed); err != nil { + return nil, err + } + return signed, nil + } + // JSON encoded transactions + txsWithKeys = inputData.Txs + } + // We may have to sign the transactions. + signer := types.MakeSigner(chainConfig, big.NewInt(int64(env.Number)), env.Timestamp) + return signUnsignedTransactions(txsWithKeys, signer) +} + +func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error { + // NOTE: IsLondon replaced with IsSubnetEVM here + if !chainConfig.IsSubnetEVM(env.Timestamp) { + return nil + } + // Sanity check, to not `panic` in state_transition + if env.BaseFee != nil { + // Already set, base fee has precedent over parent base fee. + return nil + } + if env.ParentBaseFee == nil || env.Number == 0 { + return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) + } + parent := &types.Header{ + Number: new(big.Int).SetUint64(env.Number - 1), + Time: env.ParentTimestamp, + BaseFee: env.ParentBaseFee, + GasUsed: env.ParentGasUsed, + GasLimit: env.ParentGasLimit, + Extra: make([]byte, params.DynamicFeeExtraDataSize), // TODO: consider passing extra through env + } + feeConfig := params.DefaultFeeConfig + if env.MinBaseFee != nil { + // Override the default min base fee if it's set in the env + feeConfig.MinBaseFee = env.MinBaseFee + } + var err error + _, env.BaseFee, err = dummy.CalcBaseFee(chainConfig, feeConfig, parent, env.Timestamp) + if err != nil { + return NewError(ErrorConfig, fmt.Errorf("failed calculating base fee: %v", err)) + } + return nil +} + +func applyCancunChecks(env *stEnv, chainConfig *params.ChainConfig) error { + if !chainConfig.IsCancun(big.NewInt(int64(env.Number)), env.Timestamp) { + env.ParentBeaconBlockRoot = nil // un-set it if it has been set too early + return nil + } + // Post-cancun + // We require EIP-4788 beacon root to be set in the env + if env.ParentBeaconBlockRoot == nil { + return NewError(ErrorConfig, errors.New("post-cancun env requires parentBeaconBlockRoot to be set")) + } + return nil +} + type Alloc map[common.Address]core.GenesisAccount func (g Alloc) OnRoot(common.Hash) {} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index bcc5be4b73..f1dc0dfaa8 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -33,107 +33,116 @@ import ( "os" "github.com/ava-labs/subnet-evm/cmd/evm/internal/t8ntool" + "github.com/ava-labs/subnet-evm/internal/debug" "github.com/ava-labs/subnet-evm/internal/flags" "github.com/urfave/cli/v2" ) var ( DebugFlag = &cli.BoolFlag{ - Name: "debug", - Usage: "output full trace logs", - } - MemProfileFlag = &cli.StringFlag{ - Name: "memprofile", - Usage: "creates a memory profile at the given path", - } - CPUProfileFlag = &cli.StringFlag{ - Name: "cpuprofile", - Usage: "creates a CPU profile at the given path", + Name: "debug", + Usage: "output full trace logs", + Category: flags.VMCategory, } StatDumpFlag = &cli.BoolFlag{ - Name: "statdump", - Usage: "displays stack and heap memory information", + Name: "statdump", + Usage: "displays stack and heap memory information", + Category: flags.VMCategory, } CodeFlag = &cli.StringFlag{ - Name: "code", - Usage: "EVM code", + Name: "code", + Usage: "EVM code", + Category: flags.VMCategory, } CodeFileFlag = &cli.StringFlag{ - Name: "codefile", - Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", + Name: "codefile", + Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", + Category: flags.VMCategory, } GasFlag = &cli.Uint64Flag{ - Name: "gas", - Usage: "gas limit for the evm", - Value: 10000000000, + Name: "gas", + Usage: "gas limit for the evm", + Value: 10000000000, + Category: flags.VMCategory, } PriceFlag = &flags.BigFlag{ - Name: "price", - Usage: "price set for the evm", - Value: new(big.Int), + Name: "price", + Usage: "price set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, } ValueFlag = &flags.BigFlag{ - Name: "value", - Usage: "value set for the evm", - Value: new(big.Int), + Name: "value", + Usage: "value set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, } DumpFlag = &cli.BoolFlag{ - Name: "dump", - Usage: "dumps the state after the run", + Name: "dump", + Usage: "dumps the state after the run", + Category: flags.VMCategory, } InputFlag = &cli.StringFlag{ - Name: "input", - Usage: "input for the EVM", + Name: "input", + Usage: "input for the EVM", + Category: flags.VMCategory, } InputFileFlag = &cli.StringFlag{ - Name: "inputfile", - Usage: "file containing input for the EVM", - } - VerbosityFlag = &cli.IntFlag{ - Name: "verbosity", - Usage: "sets the verbosity level", + Name: "inputfile", + Usage: "file containing input for the EVM", + Category: flags.VMCategory, } BenchFlag = &cli.BoolFlag{ - Name: "bench", - Usage: "benchmark the execution", + Name: "bench", + Usage: "benchmark the execution", + Category: flags.VMCategory, } CreateFlag = &cli.BoolFlag{ - Name: "create", - Usage: "indicates the action should be create rather than call", + Name: "create", + Usage: "indicates the action should be create rather than call", + Category: flags.VMCategory, } GenesisFlag = &cli.StringFlag{ - Name: "prestate", - Usage: "JSON file with prestate (genesis) config", + Name: "prestate", + Usage: "JSON file with prestate (genesis) config", + Category: flags.VMCategory, } MachineFlag = &cli.BoolFlag{ - Name: "json", - Usage: "output trace logs in machine readable format (json)", + Name: "json", + Usage: "output trace logs in machine readable format (json)", + Category: flags.VMCategory, } SenderFlag = &cli.StringFlag{ - Name: "sender", - Usage: "The transaction origin", + Name: "sender", + Usage: "The transaction origin", + Category: flags.VMCategory, } ReceiverFlag = &cli.StringFlag{ - Name: "receiver", - Usage: "The transaction receiver (execution context)", + Name: "receiver", + Usage: "The transaction receiver (execution context)", + Category: flags.VMCategory, } DisableMemoryFlag = &cli.BoolFlag{ - Name: "nomemory", - Value: true, - Usage: "disable memory output", + Name: "nomemory", + Value: true, + Usage: "disable memory output", + Category: flags.VMCategory, } DisableStackFlag = &cli.BoolFlag{ - Name: "nostack", - Usage: "disable stack output", + Name: "nostack", + Usage: "disable stack output", + Category: flags.VMCategory, } DisableStorageFlag = &cli.BoolFlag{ - Name: "nostorage", - Usage: "disable storage output", + Name: "nostorage", + Usage: "disable storage output", + Category: flags.VMCategory, } DisableReturnDataFlag = &cli.BoolFlag{ - Name: "noreturndata", - Value: true, - Usage: "enable return data output", + Name: "noreturndata", + Value: true, + Usage: "enable return data output", + Category: flags.VMCategory, } ) @@ -192,34 +201,38 @@ var blockBuilderCommand = &cli.Command{ }, } +// vmFlags contains flags related to running the EVM. +var vmFlags = []cli.Flag{ + CodeFlag, + CodeFileFlag, + CreateFlag, + GasFlag, + PriceFlag, + ValueFlag, + InputFlag, + InputFileFlag, + GenesisFlag, + SenderFlag, + ReceiverFlag, +} + +// traceFlags contains flags that configure tracing output. +var traceFlags = []cli.Flag{ + BenchFlag, + DebugFlag, + DumpFlag, + MachineFlag, + StatDumpFlag, + DisableMemoryFlag, + DisableStackFlag, + DisableStorageFlag, + DisableReturnDataFlag, +} + var app = flags.NewApp("the evm command line interface") func init() { - app.Flags = []cli.Flag{ - BenchFlag, - CreateFlag, - DebugFlag, - VerbosityFlag, - CodeFlag, - CodeFileFlag, - GasFlag, - PriceFlag, - ValueFlag, - DumpFlag, - InputFlag, - InputFileFlag, - MemProfileFlag, - CPUProfileFlag, - StatDumpFlag, - GenesisFlag, - MachineFlag, - SenderFlag, - ReceiverFlag, - DisableMemoryFlag, - DisableStackFlag, - DisableStorageFlag, - DisableReturnDataFlag, - } + app.Flags = flags.Merge(vmFlags, traceFlags, debug.Flags) app.Commands = []*cli.Command{ compileCommand, disasmCommand, @@ -229,6 +242,14 @@ func init() { transactionCommand, blockBuilderCommand, } + app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) + return debug.Setup(ctx) + } + app.After = func(ctx *cli.Context) error { + debug.Exit() + return nil + } } func main() { diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 019e1ab989..3a47966774 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -34,7 +34,6 @@ import ( "math/big" "os" goruntime "runtime" - "runtime/pprof" "testing" "time" @@ -43,15 +42,14 @@ import ( "github.com/ava-labs/subnet-evm/core" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" - "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/core/vm/runtime" "github.com/ava-labs/subnet-evm/eth/tracers/logger" "github.com/ava-labs/subnet-evm/internal/flags" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/urfave/cli/v2" ) @@ -61,6 +59,7 @@ var runCommand = &cli.Command{ Usage: "run arbitrary evm binary", ArgsUsage: "", Description: `The run command runs arbitrary EVM code.`, + Flags: flags.Merge(vmFlags, traceFlags), } // readGenesis will read the given JSON format genesis file and return @@ -118,9 +117,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by } func runCmd(ctx *cli.Context) error { - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) - log.Root().SetHandler(glogger) logconfig := &logger.Config{ EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), DisableStack: ctx.Bool(DisableStackFlag.Name), @@ -130,15 +126,14 @@ func runCmd(ctx *cli.Context) error { } var ( - tracer vm.EVMLogger - debugLogger *logger.StructLogger - statedb *state.StateDB - chainConfig *params.ChainConfig - sender = common.BytesToAddress([]byte("sender")) - receiver = common.BytesToAddress([]byte("receiver")) - genesisConfig *core.Genesis - preimages = ctx.Bool(DumpFlag.Name) - blobHashes []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests + tracer vm.EVMLogger + debugLogger *logger.StructLogger + statedb *state.StateDB + chainConfig *params.ChainConfig + sender = common.BytesToAddress([]byte("sender")) + receiver = common.BytesToAddress([]byte("receiver")) + preimages = ctx.Bool(DumpFlag.Name) + blobHashes []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests ) if ctx.Bool(MachineFlag.Name) { tracer = logger.NewJSONLogger(logconfig, os.Stdout) @@ -148,19 +143,30 @@ func runCmd(ctx *cli.Context) error { } else { debugLogger = logger.NewStructLogger(logconfig) } + + initialGas := ctx.Uint64(GasFlag.Name) + genesisConfig := new(core.Genesis) + genesisConfig.GasLimit = initialGas if ctx.String(GenesisFlag.Name) != "" { - gen := readGenesis(ctx.String(GenesisFlag.Name)) - genesisConfig = gen - db := rawdb.NewMemoryDatabase() - genesis := gen.MustCommit(db) - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages}) - statedb, _ = state.New(genesis.Root(), sdb, nil) - chainConfig = gen.Config + genesisConfig = readGenesis(ctx.String(GenesisFlag.Name)) + if genesisConfig.GasLimit != 0 { + initialGas = genesisConfig.GasLimit + } } else { - sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages}) - statedb, _ = state.New(types.EmptyRootHash, sdb, nil) - genesisConfig = new(core.Genesis) + genesisConfig.Config = params.TestSubnetEVMConfig } + + db := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(db, &trie.Config{ + Preimages: preimages, + HashDB: hashdb.Defaults, + }) + defer triedb.Close() + genesis := genesisConfig.MustCommit(db, triedb) + sdb := state.NewDatabaseWithNodeDB(db, triedb) + statedb, _ = state.New(genesis.Root(), sdb, nil) + chainConfig = genesisConfig.Config + if ctx.String(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.String(SenderFlag.Name)) } @@ -214,10 +220,6 @@ func runCmd(ctx *cli.Context) error { } code = common.Hex2Bytes(bin) } - initialGas := ctx.Uint64(GasFlag.Name) - if genesisConfig.GasLimit != 0 { - initialGas = genesisConfig.GasLimit - } runtimeConfig := runtime.Config{ Origin: sender, State: statedb, @@ -234,19 +236,6 @@ func runCmd(ctx *cli.Context) error { }, } - if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" { - f, err := os.Create(cpuProfilePath) - if err != nil { - fmt.Println("could not create CPU profile: ", err) - os.Exit(1) - } - if err := pprof.StartCPUProfile(f); err != nil { - fmt.Println("could not start CPU profile: ", err) - os.Exit(1) - } - defer pprof.StopCPUProfile() - } - if chainConfig != nil { runtimeConfig.ChainConfig = chainConfig } else { @@ -294,19 +283,6 @@ func runCmd(ctx *cli.Context) error { fmt.Println(string(statedb.Dump(nil))) } - if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" { - f, err := os.Create(memProfilePath) - if err != nil { - fmt.Println("could not create memory profile: ", err) - os.Exit(1) - } - if err := pprof.WriteHeapProfile(f); err != nil { - fmt.Println("could not write memory profile: ", err) - os.Exit(1) - } - f.Close() - } - if ctx.Bool(DebugFlag.Name) { if debugLogger != nil { fmt.Fprintln(os.Stderr, "#### TRACE ####") diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 81b7442394..c3dd468768 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -32,12 +32,13 @@ import ( "fmt" "os" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" + "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/tracers/logger" "github.com/ava-labs/subnet-evm/tests" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" "github.com/urfave/cli/v2" ) @@ -60,11 +61,6 @@ type StatetestResult struct { } func stateTestCmd(ctx *cli.Context) error { - // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) - log.Root().SetHandler(glogger) - // Configure the EVM logger config := &logger.Config{ EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), @@ -114,25 +110,23 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - _, s, err := test.Run(st, cfg, false) - // print state root for evmlab tracing - if s != nil { - root := s.IntermediateRoot(false) - result.Root = &root - if jsonOut { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { + if state != nil { + root := state.IntermediateRoot(false) + result.Root = &root + if jsonOut { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + } } - } - if err != nil { - // Test failed, mark as so and dump any state to aid debugging - result.Pass, result.Error = false, err.Error() - if dump && s != nil { - s, _ = state.New(*result.Root, s.Database(), nil) - dump := s.RawDump(nil) - result.State = &dump + if err != nil { + // Test failed, mark as so and dump any state to aid debugging + result.Pass, result.Error = false, err.Error() + if dump { + dump := state.RawDump(nil) + result.State = &dump + } } - } - + }) results = append(results, *result) } } diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 0cfa0151e6..fcd0a6dd95 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -284,6 +284,14 @@ func TestT8n(t *testing.T) { output: t8nOutput{alloc: true, result: true}, expOut: "exp.json", }, + { // More cancun tests + base: "./testdata/29", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Cancun", "", + }, + output: t8nOutput{alloc: true, result: true}, + expOut: "exp.json", + }, } { args := []string{"t8n"} args = append(args, tc.output.get()...) diff --git a/cmd/evm/testdata/28/env.json b/cmd/evm/testdata/28/env.json index 14a1f3ed1e..4ff2b554eb 100644 --- a/cmd/evm/testdata/28/env.json +++ b/cmd/evm/testdata/28/env.json @@ -1,4 +1,5 @@ { + "minBaseFee" : "0x9", "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", "currentNumber" : "0x01", "currentTimestamp" : "0x079e", @@ -19,5 +20,5 @@ "blockHashes" : { "0" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6" }, - "minBaseFee" : "0x9" + "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00" } \ No newline at end of file diff --git a/cmd/evm/testdata/29/alloc.json b/cmd/evm/testdata/29/alloc.json new file mode 100644 index 0000000000..70d47862a0 --- /dev/null +++ b/cmd/evm/testdata/29/alloc.json @@ -0,0 +1,16 @@ +{ + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : { + "balance" : "0x016345785d8a0000", + "code" : "0x", + "nonce" : "0x00", + "storage" : { + } + }, + "0xbEac00dDB15f3B6d645C48263dC93862413A222D" : { + "balance" : "0x1", + "code" : "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500", + "nonce" : "0x00", + "storage" : { + } + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/29/env.json b/cmd/evm/testdata/29/env.json new file mode 100644 index 0000000000..c0e4192564 --- /dev/null +++ b/cmd/evm/testdata/29/env.json @@ -0,0 +1,21 @@ +{ + "minBaseFee" : "0x9", + "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentNumber" : "0x01", + "currentTimestamp" : "0x079e", + "currentGasLimit" : "0x7fffffffffffffff", + "previousHash" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6", + "currentBlobGasUsed" : "0x00", + "parentTimestamp" : "0x03b6", + "parentDifficulty" : "0x00", + "parentUncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "currentRandom" : "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "withdrawals" : [ + ], + "parentBaseFee" : "0x0a", + "parentGasUsed" : "0x00", + "parentGasLimit" : "0x7fffffffffffffff", + "parentExcessBlobGas" : "0x00", + "parentBlobGasUsed" : "0x00", + "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00" +} \ No newline at end of file diff --git a/cmd/evm/testdata/29/exp.json b/cmd/evm/testdata/29/exp.json new file mode 100644 index 0000000000..9cc41e5ed0 --- /dev/null +++ b/cmd/evm/testdata/29/exp.json @@ -0,0 +1,47 @@ +{ + "alloc": { + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x2e248" + }, + "0xbeac00ddb15f3b6d645c48263dc93862413a222d": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500", + "storage": { + "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e", + "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00" + }, + "balance": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x16345785d871db8", + "nonce": "0x1" + } + }, + "result": { + "stateRoot": "0x79f47a970555f9f8aab12135f754201a46f48affaba011593339523127496c73", + "txRoot": "0x248074fabe112f7d93917f292b64932394f835bb98da91f21501574d58ec92ab", + "receiptsRoot": "0xf78dfb743fbd92ade140711c8bbc542b5e307f0ab7984eff35d751969fe57efa", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "type": "0x2", + "root": "0x", + "status": "0x1", + "cumulativeGasUsed": "0x5208", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x84f70aba406a55628a0620f26d260f90aeb6ccc55fed6ec2ac13dd4f727032ed", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x5208", + "effectiveGasPrice": null, + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + } + ], + "currentDifficulty": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "gasUsed": "0x5208", + "currentBaseFee": "0x9", + "currentExcessBlobGas": "0x0", + "currentBlobGasUsed": "0x0" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/29/readme.md b/cmd/evm/testdata/29/readme.md new file mode 100644 index 0000000000..4383e328ed --- /dev/null +++ b/cmd/evm/testdata/29/readme.md @@ -0,0 +1,29 @@ +## EIP 4788 + +This test contains testcases for EIP-4788. The 4788-contract is +located at address `0xbeac00ddb15f3b6d645c48263dc93862413a222d`, and this test executes a simple transaction. It also +implicitly invokes the system tx, which sets calls the contract and sets the +storage values +``` +$ dir=./testdata/29/ && go run . t8n --state.fork=Cancun --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout +INFO [08-15|20:07:56.335] Trie dumping started root=ecde45..2af8a7 +INFO [08-15|20:07:56.335] Trie dumping complete accounts=2 elapsed="225.848µs" +INFO [08-15|20:07:56.335] Wrote file file=result.json +{ + "alloc": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x16345785d871db8", + "nonce": "0x1" + }, + "0xbeac00541d49391ed88abf392bfc1f4dea8c4143": { + "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500", + "storage": { + "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e", + "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00" + }, + "balance": "0x + } + } +} + +``` diff --git a/cmd/evm/testdata/29/txs.json b/cmd/evm/testdata/29/txs.json new file mode 100644 index 0000000000..d6743cc4d2 --- /dev/null +++ b/cmd/evm/testdata/29/txs.json @@ -0,0 +1,19 @@ +[ + { + "input" : "0x", + "gas" : "0x10000000", + "nonce" : "0x0", + "to" : "0x1111111111111111111111111111111111111111", + "value" : "0x0", + "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8", + "chainId" : "0x1", + "type" : "0x2", + "v": "0x0", + "r": "0x0", + "s": "0x0", + "maxFeePerGas" : "0xfa0", + "maxPriorityFeePerGas" : "0x0", + "accessList" : [ + ] + } +] \ No newline at end of file diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 8be8b32969..9e3444ec38 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -52,8 +52,8 @@ func VerifyEIP4844Header(parent, header *types.Header) error { return errors.New("header is missing blobGasUsed") } // Verify that the blob gas used remains within reasonable limits. - if *header.BlobGasUsed > params.BlobTxMaxBlobGasPerBlock { - return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.BlobTxMaxBlobGasPerBlock) + if *header.BlobGasUsed > params.MaxBlobGasPerBlock { + return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.MaxBlobGasPerBlock) } if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 { return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) diff --git a/consensus/misc/eip4844/eip4844_test.go b/consensus/misc/eip4844/eip4844_test.go index 3780acc15a..04c735f068 100644 --- a/consensus/misc/eip4844/eip4844_test.go +++ b/consensus/misc/eip4844/eip4844_test.go @@ -45,14 +45,14 @@ func TestCalcExcessBlobGas(t *testing.T) { // The excess blob gas should decrease by however much the target was // under-shot, capped at zero. {params.BlobTxTargetBlobGasPerBlock, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, params.BlobTxTargetBlobGasPerBlock}, - {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxBlobGasPerBlob}, - {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, 0}, + {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxTargetBlobGasPerBlock - params.BlobTxBlobGasPerBlob}, + {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, params.BlobTxTargetBlobGasPerBlock - (2 * params.BlobTxBlobGasPerBlob)}, {params.BlobTxBlobGasPerBlob - 1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, 0}, } - for _, tt := range tests { + for i, tt := range tests { result := CalcExcessBlobGas(tt.excess, tt.blobs*params.BlobTxBlobGasPerBlob) if result != tt.want { - t.Errorf("excess blob gas mismatch: have %v, want %v", result, tt.want) + t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want) } } } @@ -63,9 +63,9 @@ func TestCalcBlobFee(t *testing.T) { blobfee int64 }{ {0, 1}, - {1542706, 1}, - {1542707, 2}, - {10 * 1024 * 1024, 111}, + {2314057, 1}, + {2314058, 2}, + {10 * 1024 * 1024, 23}, } for i, tt := range tests { have := CalcBlobFee(tt.excessBlobGas) diff --git a/core/block_validator.go b/core/block_validator.go index 86160826c4..1d97574810 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -78,14 +78,23 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch (header value %x, calculated %x)", header.TxHash, hash) } + // Blob transactions may be present after the Cancun fork. var blobs int - for _, tx := range block.Transactions() { + for i, tx := range block.Transactions() { // Count the number of blobs to validate against the header's blobGasUsed blobs += len(tx.BlobHashes()) + + // If the tx is a blob tx, it must NOT have a sidecar attached to be valid in a block. + if tx.BlobTxSidecar() != nil { + return fmt.Errorf("unexpected blob sidecar in transaction at index %d", i) + } + // The individual checks for blob validity (version-check + not empty) - // happens in the state_transition check. + // happens in StateTransition. } + + // Check blob gas usage. if header.BlobGasUsed != nil { if want := *header.BlobGasUsed / params.BlobTxBlobGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated return fmt.Errorf("blob gas used mismatch (header %v, calculated %v)", *header.BlobGasUsed, blobs*params.BlobTxBlobGasPerBlob) @@ -95,6 +104,8 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { return errors.New("data blobs present in block body") } } + + // Ancestor block must be known. if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { return consensus.ErrUnknownAncestor diff --git a/core/blockchain.go b/core/blockchain.go index f13b478a94..e4643e8e5a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -51,6 +51,8 @@ import ( "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/ethdb" @@ -164,7 +166,7 @@ type cacheableCoinbaseConfig struct { } // CacheConfig contains the configuration values for the trie database -// that's resident in a blockchain. +// and state snapshot these are resident in a blockchain. type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory TrieDirtyLimit int // Memory limit (MB) at which to block on insert and force a flush of dirty trie nodes to disk @@ -183,11 +185,34 @@ type CacheConfig struct { AcceptedCacheSize int // Depth of accepted headers cache and accepted logs cache at the accepted tip TxLookupLimit uint64 // Number of recent blocks for which to maintain transaction lookup indices SkipTxIndexing bool // Whether to skip transaction indexing + StateHistory uint64 // Number of blocks from head whose state histories are reserved. + StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top SnapshotNoBuild bool // Whether the background generation is allowed SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } +// triedbConfig derives the configures for trie database. +func (c *CacheConfig) triedbConfig() *trie.Config { + config := &trie.Config{Preimages: c.Preimages} + if c.StateScheme == rawdb.HashScheme { + config.HashDB = &hashdb.Config{ + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + StatsPrefix: trieCleanCacheStatsNamespace, + } + } + if c.StateScheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{ + StateHistory: c.StateHistory, + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024, + } + } + return config +} + +// DefaultCacheConfig are the default caching values if none are specified by the +// user (also used during testing). var DefaultCacheConfig = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, @@ -198,6 +223,15 @@ var DefaultCacheConfig = &CacheConfig{ AcceptorQueueLimit: 64, // Provides 2 minutes of buffer (2s block target) for a commit delay SnapshotLimit: 256, AcceptedCacheSize: 32, + StateScheme: rawdb.HashScheme, +} + +// DefaultCacheConfigWithScheme returns a deep copied default cache config with +// a provided trie node scheme. +func DefaultCacheConfigWithScheme(scheme string) *CacheConfig { + config := *DefaultCacheConfig + config.StateScheme = scheme + return &config } // BlockChain represents the canonical chain given a database with a genesis @@ -314,11 +348,8 @@ func NewBlockChain( return nil, errCacheConfigNotSpecified } // Open trie database with provided config - triedb := trie.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Preimages: cacheConfig.Preimages, - StatsPrefix: trieCleanCacheStatsNamespace, - }) + triedb := trie.NewDatabase(db, cacheConfig.triedbConfig()) + // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the // stored one from database. @@ -433,6 +464,12 @@ func (bc *BlockChain) unindexBlocks(tail uint64, head uint64, done chan struct{} close(done) }() + // If head is 0, it means the chain is just initialized and no blocks are inserted, + // so don't need to indexing anything. + if head == 0 { + return + } + if head-txLookupLimit+1 >= tail { // Unindex a part of stale indices and forward index tail to HEAD-limit rawdb.UnindexTransactions(bc.db, tail, head-txLookupLimit+1, bc.quit) @@ -464,6 +501,17 @@ func (bc *BlockChain) dispatchTxUnindexer() { return } defer sub.Unsubscribe() + log.Info("Initialized transaction unindexer", "limit", txLookupLimit) + + // TODO: Uncomment this code when the tx-unindexer fix is ready. + // Launch the initial processing if chain is not empty. This step is + // useful in these scenarios that chain has no progress and indexer + // is never triggered. + // if head := bc.lastAccepted; head != nil && head.NumberU64() > txLookupLimit { + // done = make(chan struct{}) + // tail := rawdb.ReadTxIndexTail(bc.db) + // go bc.unindexBlocks(*tail, head.NumberU64(), done) + // } for { select { @@ -968,15 +1016,21 @@ func (bc *BlockChain) stopWithoutSaving() { func (bc *BlockChain) Stop() { bc.stopWithoutSaving() + if bc.triedb.Scheme() == rawdb.PathScheme { + // Ensure that the in-memory trie nodes are journaled to disk properly. + if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil { + log.Info("Failed to journal in-memory trie nodes", "err", err) + } + } log.Info("Shutting down state manager") start := time.Now() if err := bc.stateManager.Shutdown(); err != nil { log.Error("Failed to Shutdown state manager", "err", err) } log.Info("State manager shut down", "t", time.Since(start)) - // Flush the collected preimages to disk - if err := bc.stateCache.TrieDB().Close(); err != nil { - log.Error("Failed to close trie db", "err", err) + // Close the trie database, release all the held resources as the last step. + if err := bc.triedb.Close(); err != nil { + log.Error("Failed to close trie database", "err", err) } log.Info("Blockchain stopped") } @@ -1224,6 +1278,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. if err != nil { return err } + // If node is running in path mode, skip explicit gc operation + // which is unnecessary in this mode. + if bc.triedb.Scheme() == rawdb.PathScheme { + return nil + } // Note: if InsertTrie must be the last step in verification that can return an error. // This allows [stateManager] to assume that if it inserts a trie without returning an @@ -1927,7 +1986,7 @@ func (bc *BlockChain) reprocessState(current *types.Block, reexec uint64) error } } - nodes, imgs := triedb.Size() + _, nodes, imgs := triedb.Size() log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) if previousRoot != (common.Hash{}) { return triedb.Commit(previousRoot, true) @@ -2028,7 +2087,7 @@ func (bc *BlockChain) populateMissingTries() error { return fmt.Errorf("failed to write offline pruning success marker: %w", err) } - nodes, imgs := triedb.Size() + _, nodes, imgs := triedb.Size() log.Info("All missing tries populated", "startHeight", startHeight, "lastAcceptedHeight", lastAccepted, "missing", missing, "elapsed", time.Since(startTime), "nodes", nodes, "preimages", imgs) return nil } diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 0dc3d20f44..554c20e730 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -236,12 +236,6 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { return bc.HasState(block.Root()) } -// TrieNode retrieves a blob of data associated with a trie node -// either from ephemeral in-memory cache, or from persistent storage. -func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { - return bc.stateCache.TrieDB().Node(hash) -} - // State returns a new mutable state based on the current HEAD block. func (bc *BlockChain) State() (*state.StateDB, error) { return bc.StateAt(bc.CurrentBlock().Root) diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index e33dcda9ec..f124b0efaa 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -39,7 +39,10 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/params" + "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/stretchr/testify/require" ) // rewindTest is a test case for chain rollback upon user request. @@ -501,6 +504,12 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) { } func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + testRepairWithScheme(t, tt, snapshots, scheme) + } +} + +func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) { // It's hard to follow the test case, visualize the input //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // fmt.Println(tt.dump(true)) @@ -510,6 +519,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -517,22 +527,31 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { defer db.Close() // Might double close, should be fine // Initialize a fresh chain + chainConfig := *params.TestChainConfig + chainConfig.FeeConfig.MinBaseFee = big.NewInt(1) var ( - gspec = &Genesis{ - BaseFee: big.NewInt(params.TestInitialBaseFee), - Config: params.TestChainConfig, + require = require.New(t) + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + gspec = &Genesis{ + BaseFee: chainConfig.FeeConfig.MinBaseFee, + Config: &chainConfig, + Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(params.Ether)}}, } + signer = types.LatestSigner(gspec.Config) engine = dummy.NewFullFaker() config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TriePrefetcherParallelism: 4, SnapshotLimit: 0, // Disable snapshot by default + StateScheme: scheme, } ) defer engine.Close() if snapshots { config.SnapshotLimit = 256 + config.SnapshotWait = true } chain, err := NewBlockChain(db, config, gspec, engine, vm.Config{}, common.Hash{}, false) if err != nil { @@ -544,17 +563,29 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // If sidechain blocks are needed, make a light chain and import it var sideblocks types.Blocks if tt.sidechainBlocks > 0 { - sideblocks, _, _ = GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, 10, func(i int, b *BlockGen) { + genDb := rawdb.NewMemoryDatabase() + gspec.MustCommit(genDb, trie.NewDatabase(genDb, nil)) + sideblocks, _, err = GenerateChain(gspec.Config, gspec.ToBlock(), engine, genDb, tt.sidechainBlocks, 10, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x01}) + tx, err := types.SignTx(types.NewTransaction(b.TxNonce(addr1), common.Address{0x01}, big.NewInt(10000), params.TxGas, common.Big1, nil), signer, key1) + require.NoError(err) + b.AddTx(tx) }) + require.NoError(err) if _, err := chain.InsertChain(sideblocks); err != nil { t.Fatalf("Failed to import side chain: %v", err) } } - canonblocks, _, _ := GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, 10, func(i int, b *BlockGen) { + genDb := rawdb.NewMemoryDatabase() + gspec.MustCommit(genDb, trie.NewDatabase(genDb, nil)) + canonblocks, _, err := GenerateChain(gspec.Config, gspec.ToBlock(), engine, genDb, tt.canonicalBlocks, 10, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x02}) b.SetDifficulty(big.NewInt(1000000)) + tx, err := types.SignTx(types.NewTransaction(b.TxNonce(addr1), common.Address{0x02}, big.NewInt(10000), params.TxGas, common.Big1, nil), signer, key1) + require.NoError(err) + b.AddTx(tx) }) + require.NoError(err) if _, err := chain.InsertChain(canonblocks[:tt.commitBlock]); err != nil { t.Fatalf("Failed to import canonical chain start: %v", err) } @@ -574,12 +605,14 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } // Pull the plug on the database, simulating a hard crash + chain.triedb.Close() db.Close() chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us db, err = rawdb.Open(rawdb.OpenOptions{ Directory: datadir, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 14083555f8..9b5cea4aa1 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -34,6 +34,7 @@ import ( "fmt" "math/big" "os" + "path" "strings" "testing" @@ -49,6 +50,7 @@ import ( // snapshotTestBasic wraps the common testing fields in the snapshot tests. type snapshotTestBasic struct { + scheme string // Disk scheme used for storing trie nodes chainBlocks int // Number of blocks to generate for the canonical chain snapshotBlock uint64 // Block number of the relevant snapshot disk layer @@ -58,6 +60,7 @@ type snapshotTestBasic struct { // share fields, set in runtime datadir string + ancient string db ethdb.Database genDb ethdb.Database engine consensus.Engine @@ -69,9 +72,11 @@ type snapshotTestBasic struct { func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { // Create a temporary persistent database datadir := t.TempDir() + ancient := path.Join(datadir, "ancient") db, err := rawdb.Open(rawdb.OpenOptions{ Directory: datadir, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to create persistent database: %v", err) @@ -83,13 +88,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo Config: params.TestChainConfig, } engine = dummy.NewFullFaker() - - // Snapshot is enabled, the first snapshot is created from the Genesis. - // The snapshot memory allowance is 256MB, it means no snapshot flush - // will happen during the block insertion. - cacheConfig = DefaultCacheConfig ) - chain, err := NewBlockChain(db, cacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false) + chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, engine, vm.Config{}, common.Hash{}, false) if err != nil { t.Fatalf("Failed to create chain: %v", err) } @@ -130,6 +130,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // Set runtime fields basic.datadir = datadir + basic.ancient = ancient basic.db = db basic.genDb = genDb basic.engine = engine @@ -211,6 +212,7 @@ func (basic *snapshotTestBasic) teardown() { basic.db.Close() basic.genDb.Close() os.RemoveAll(basic.datadir) + os.RemoveAll(basic.ancient) } // snapshotTest is a test case type for normal snapshot recovery. @@ -227,7 +229,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) + newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -236,7 +238,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { snaptest.verify(t, newchain, blocks) } -// crashSnapshotTest is a test case type for innormal snapshot recovery. +// crashSnapshotTest is a test case type for irregular snapshot recovery. // It can be used for testing that restart Geth after the crash. type crashSnapshotTest struct { snapshotTestBasic @@ -252,10 +254,12 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { db := chain.db db.Close() chain.stopWithoutSaving() + chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us newdb, err := rawdb.Open(rawdb.OpenOptions{ Directory: snaptest.datadir, + Ephemeral: true, }) if err != nil { t.Fatalf("Failed to reopen persistent database: %v", err) @@ -266,13 +270,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - newchain, err := NewBlockChain(newdb, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) + newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) + newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -299,7 +303,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { // Insert blocks without enabling snapshot if gapping is required. chain.Stop() - gappedBlocks, _, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, 10, func(i int, b *BlockGen) {}) + gappedBlocks, _, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, 10, func(i int, b *BlockGen) {}) // Insert a few more blocks without enabling snapshot var cacheConfig = &CacheConfig{ @@ -308,6 +312,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { SnapshotLimit: 0, Pruning: true, CommitInterval: 4096, + StateScheme: snaptest.scheme, } newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { @@ -317,7 +322,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -352,22 +357,25 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 0, Pruning: true, CommitInterval: 4096, + StateScheme: snaptest.scheme, } newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - newBlocks, _, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, 10, func(i int, b *BlockGen) {}) + newBlocks, _, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, 10, func(i int, b *BlockGen) {}) newchain.InsertChain(newBlocks) newchain.Stop() - // Restart the chain, the wiper should starts working + // Restart the chain, the wiper should start working config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, SnapshotLimit: 256, Pruning: true, CommitInterval: 4096, + SnapshotWait: false, // Don't wait rebuild + StateScheme: snaptest.scheme, } tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { @@ -375,14 +383,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { } // Simulate the blockchain crash. + tmp.triedb.Close() tmp.stopWithoutSaving() - newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) + newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - defer newchain.Stop() snaptest.verify(t, newchain, blocks) + newchain.Stop() } // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot @@ -402,17 +411,20 @@ func TestRestartWithNewSnapshot(t *testing.T) { // Expected head header : C8 // Expected head block : C4 // Expected snapshot disk : C4 - test := &snapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - expCanonicalBlocks: 8, - expHeadBlock: 4, - expSnapshotBottom: 4, // Initial disk layer built from genesis - }, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &snapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + expCanonicalBlocks: 8, + expHeadBlock: 4, + expSnapshotBottom: 4, // Initial disk layer built from genesis + }, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -435,17 +447,20 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) { // // Expected head block : C4 // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - expCanonicalBlocks: 8, - expHeadBlock: 4, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + expCanonicalBlocks: 8, + expHeadBlock: 4, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case the @@ -468,17 +483,20 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) { // // Expected head block : C4 // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - expCanonicalBlocks: 8, - expHeadBlock: 4, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + expCanonicalBlocks: 8, + expHeadBlock: 4, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } // Tests a Geth was crashed and restarts with a broken snapshot. In this case @@ -501,17 +519,20 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) { // // Expected head block : C4 // Expected snapshot disk : C4 - test := &crashSnapshotTest{ - snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - expCanonicalBlocks: 8, - expHeadBlock: 4, - expSnapshotBottom: 4, // Last committed disk layer, wait recovery - }, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &crashSnapshotTest{ + snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + expCanonicalBlocks: 8, + expHeadBlock: 4, + expSnapshotBottom: 4, // Last committed disk layer, wait recovery + }, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } // Tests a Geth was running with snapshot enabled. Then restarts without @@ -530,18 +551,21 @@ func TestGappedNewSnapshot(t *testing.T) { // // Expected head block : G // Expected snapshot disk : G - test := &gappedSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 0, - expCanonicalBlocks: 10, - expHeadBlock: 0, - expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD - }, - gapped: 2, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &gappedSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 0, + expCanonicalBlocks: 10, + expHeadBlock: 0, + expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD + }, + gapped: 2, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } // Tests the Geth was running with a complete snapshot and then imports a few @@ -560,16 +584,19 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) { // // Expected head block : C4 // Expected snapshot disk : C4 - test := &wipeCrashSnapshotTest{ - snapshotTestBasic: snapshotTestBasic{ - chainBlocks: 8, - snapshotBlock: 4, - expCanonicalBlocks: 10, - expHeadBlock: 4, - expSnapshotBottom: 4, - }, - newBlocks: 2, + for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} { + test := &wipeCrashSnapshotTest{ + snapshotTestBasic: snapshotTestBasic{ + scheme: scheme, + chainBlocks: 8, + snapshotBlock: 4, + expCanonicalBlocks: 10, + expHeadBlock: 4, + expSnapshotBottom: 4, + }, + newBlocks: 2, + } + test.test(t) + test.teardown() } - test.test(t) - test.teardown() } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index bf099e3cdc..c7ceb960ac 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -541,7 +541,6 @@ func TestUngracefulAsyncShutdown(t *testing.T) { // TODO: simplify the unindexer logic and this test. func TestTransactionIndices(t *testing.T) { // Configure and generate a sample block chain - require := require.New(t) var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") @@ -556,19 +555,20 @@ func TestTransactionIndices(t *testing.T) { ) genDb, blocks, _, err := GenerateChainWithGenesis(gspec, dummy.NewFaker(), 128, 10, func(i int, block *BlockGen) { tx, err := types.SignTx(types.NewTransaction(block.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil), signer, key1) - require.NoError(err) + require.NoError(t, err) block.AddTx(tx) }) - require.NoError(err) + require.NoError(t, err) blocks2, _, err := GenerateChain(gspec.Config, blocks[len(blocks)-1], dummy.NewFaker(), genDb, 10, 10, func(i int, block *BlockGen) { tx, err := types.SignTx(types.NewTransaction(block.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil), signer, key1) - require.NoError(err) + require.NoError(t, err) block.AddTx(tx) }) - require.NoError(err) + require.NoError(t, err) - check := func(tail *uint64, chain *BlockChain) { + check := func(t *testing.T, tail *uint64, chain *BlockChain) { + require := require.New(t) stored := rawdb.ReadTxIndexTail(chain.db) var tailValue uint64 if tail == nil { @@ -617,19 +617,19 @@ func TestTransactionIndices(t *testing.T) { // Init block chain and check all needed indices has been indexed. chainDB := rawdb.NewMemoryDatabase() chain, err := createBlockChain(chainDB, conf, gspec, common.Hash{}) - require.NoError(err) + require.NoError(t, err) _, err = chain.InsertChain(blocks) - require.NoError(err) + require.NoError(t, err) for _, block := range blocks { err := chain.Accept(block) - require.NoError(err) + require.NoError(t, err) } chain.DrainAcceptorQueue() chain.Stop() - check(nil, chain) // check all indices has been indexed + check(t, nil, chain) // check all indices has been indexed lastAcceptedHash := chain.CurrentHeader().Hash() @@ -647,14 +647,14 @@ func TestTransactionIndices(t *testing.T) { conf.TxLookupLimit = l chain, err := createBlockChain(chainDB, conf, gspec, lastAcceptedHash) - require.NoError(err) + require.NoError(t, err) newBlks := blocks2[i : i+1] _, err = chain.InsertChain(newBlks) // Feed chain a higher block to trigger indices updater. - require.NoError(err) + require.NoError(t, err) err = chain.Accept(newBlks[0]) // Accept the block to trigger indices updater. - require.NoError(err) + require.NoError(t, err) chain.DrainAcceptorQueue() time.Sleep(50 * time.Millisecond) // Wait for indices initialisation @@ -673,7 +673,7 @@ func TestTransactionIndices(t *testing.T) { tail = &tl } - check(tail, chain) + check(t, tail, chain) lastAcceptedHash = chain.CurrentHeader().Hash() }) @@ -801,6 +801,11 @@ func TestTransactionSkipIndexing(t *testing.T) { // TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted // correctly in case reorg is called. func TestCanonicalHashMarker(t *testing.T) { + testCanonicalHashMarker(t, rawdb.HashScheme) + testCanonicalHashMarker(t, rawdb.PathScheme) +} + +func testCanonicalHashMarker(t *testing.T, scheme string) { var cases = []struct { forkA int forkB int @@ -854,8 +859,7 @@ func TestCanonicalHashMarker(t *testing.T) { } // Initialize test chain - diskdb := rawdb.NewMemoryDatabase() - chain, err := NewBlockChain(diskdb, DefaultCacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, engine, vm.Config{}, common.Hash{}, false) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1049,6 +1053,120 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { } } +func TestDeleteThenCreate(t *testing.T) { + var ( + engine = dummy.NewFaker() + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + factoryAddr = crypto.CreateAddress(address, 0) + funds = big.NewInt(params.Ether) // Note: additional funds are provided here compared to go-ethereum so test completes. + ) + /* + contract Factory { + function deploy(bytes memory code) public { + address addr; + assembly { + addr := create2(0, add(code, 0x20), mload(code), 0) + if iszero(extcodesize(addr)) { + revert(0, 0) + } + } + } + } + */ + factoryBIN := common.Hex2Bytes("608060405234801561001057600080fd5b50610241806100206000396000f3fe608060405234801561001057600080fd5b506004361061002a5760003560e01c80627743601461002f575b600080fd5b610049600480360381019061004491906100d8565b61004b565b005b6000808251602084016000f59050803b61006457600080fd5b5050565b600061007b61007684610146565b610121565b905082815260208101848484011115610097576100966101eb565b5b6100a2848285610177565b509392505050565b600082601f8301126100bf576100be6101e6565b5b81356100cf848260208601610068565b91505092915050565b6000602082840312156100ee576100ed6101f5565b5b600082013567ffffffffffffffff81111561010c5761010b6101f0565b5b610118848285016100aa565b91505092915050565b600061012b61013c565b90506101378282610186565b919050565b6000604051905090565b600067ffffffffffffffff821115610161576101606101b7565b5b61016a826101fa565b9050602081019050919050565b82818337600083830152505050565b61018f826101fa565b810181811067ffffffffffffffff821117156101ae576101ad6101b7565b5b80604052505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f830116905091905056fea2646970667358221220ea8b35ed310d03b6b3deef166941140b4d9e90ea2c92f6b41eb441daf49a59c364736f6c63430008070033") + + /* + contract C { + uint256 value; + constructor() { + value = 100; + } + function destruct() public payable { + selfdestruct(payable(msg.sender)); + } + receive() payable external {} + } + */ + contractABI := common.Hex2Bytes("6080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c63430008070033") + contractAddr := crypto.CreateAddress2(factoryAddr, [32]byte{}, crypto.Keccak256(contractABI)) + + gspec := &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{ + address: {Balance: funds}, + }, + } + nonce := uint64(0) + signer := types.HomesteadSigner{} + _, blocks, _, err := GenerateChainWithGenesis(gspec, engine, 2, 10, func(i int, b *BlockGen) { + fee := big.NewInt(1) + if b.header.BaseFee != nil { + fee = b.header.BaseFee + } + b.SetCoinbase(common.Address{1}) + + // Block 1 + if i == 0 { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + Data: factoryBIN, + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, + Data: data, + }) + b.AddTx(tx) + nonce++ + } else { + // Block 2 + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &contractAddr, + Data: common.Hex2Bytes("2b68b9c6"), // destruct + }) + nonce++ + b.AddTx(tx) + + data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000") + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: nonce, + GasPrice: new(big.Int).Set(fee), + Gas: 500000, + To: &factoryAddr, // re-creation + Data: data, + }) + b.AddTx(tx) + nonce++ + } + }) + if err != nil { + t.Fatal(err) + } + // Import the canonical chain + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + defer chain.Stop() + for _, block := range blocks { + if _, err := chain.InsertChain([]*types.Block{block}); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err) + } + } +} + // TestTransientStorageReset ensures the transient storage is wiped correctly // between transactions. func TestTransientStorageReset(t *testing.T) { diff --git a/core/chain_makers.go b/core/chain_makers.go index 4e9f794fc8..5387cefefb 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -33,6 +33,7 @@ import ( "github.com/ava-labs/subnet-evm/commontype" "github.com/ava-labs/subnet-evm/consensus" "github.com/ava-labs/subnet-evm/consensus/dummy" + "github.com/ava-labs/subnet-evm/consensus/misc/eip4844" "github.com/ava-labs/subnet-evm/constants" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" @@ -98,6 +99,11 @@ func (b *BlockGen) SetDifficulty(diff *big.Int) { b.header.Difficulty = diff } +// SetBlobGas sets the data gas used by the blob in the generated block. +func (b *BlockGen) SetBlobGas(blobGasUsed uint64) { + b.header.BlobGasUsed = &blobGasUsed +} + // addTx adds a transaction to the generated block. If no coinbase has // been set, the block's coinbase is set to the zero address. // @@ -250,7 +256,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) chainreader := &fakeChainReader{config: config} - genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts, error) { + genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts, error) { b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} b.header = makeHeader(chainreader, config, parent, gap, statedb, b.engine) @@ -275,7 +281,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } - if err := statedb.Database().TrieDB().Commit(root, false); err != nil { + if err = triedb.Commit(root, false); err != nil { panic(fmt.Sprintf("trie write error: %v", err)) } if b.onBlockGenerated != nil { @@ -285,12 +291,16 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } return nil, nil, nil } + // Forcibly use hash-based state scheme for retaining all nodes in disk. + triedb := trie.NewDatabase(db, trie.HashDefaults) + defer triedb.Close() + for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil) + statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { return nil, nil, err } - block, receipt, err := genblock(i, parent, statedb) + block, receipt, err := genblock(i, parent, triedb, statedb) if err != nil { return nil, nil, err } @@ -306,7 +316,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse // then generate chain on top. func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gap uint64, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts, error) { db := rawdb.NewMemoryDatabase() - _, err := genesis.Commit(db, trie.NewDatabase(db)) + triedb := trie.NewDatabase(db, trie.HashDefaults) + defer triedb.Close() + _, err := genesis.Commit(db, triedb) if err != nil { return nil, nil, nil, err } @@ -349,6 +361,20 @@ func makeHeader(chain consensus.ChainReader, config *params.ChainConfig, parent } else { header.GasLimit = CalcGasLimit(parent.GasUsed(), parent.GasLimit(), parent.GasLimit(), parent.GasLimit()) } + if chain.Config().IsCancun(header.Number, header.Time) { + var ( + parentExcessBlobGas uint64 + parentBlobGasUsed uint64 + ) + if parent.ExcessBlobGas() != nil { + parentExcessBlobGas = *parent.ExcessBlobGas() + parentBlobGasUsed = *parent.BlobGasUsed() + } + excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed) + header.ExcessBlobGas = &excessBlobGas + header.BlobGasUsed = new(uint64) + header.ParentBeaconRoot = new(common.Hash) + } return header } diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index d322550790..c3f6f576e4 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -35,6 +35,7 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/params" + "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -48,6 +49,7 @@ func ExampleGenerateChain() { addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) db = rawdb.NewMemoryDatabase() + genDb = rawdb.NewMemoryDatabase() ) // Ensure that key1 has some funds in the genesis block. @@ -55,12 +57,13 @@ func ExampleGenerateChain() { Config: ¶ms.ChainConfig{HomesteadBlock: new(big.Int)}, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, } + genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults)) // This call generates a chain of 3 blocks. The function runs for // each block and adds different features to gen based on the // block index. signer := types.HomesteadSigner{} - _, chain, _, err := GenerateChainWithGenesis(gspec, dummy.NewCoinbaseFaker(), 3, 10, func(i int, gen *BlockGen) { + chain, _, err := GenerateChain(gspec.Config, genesis, dummy.NewCoinbaseFaker(), genDb, 3, 10, func(i int, gen *BlockGen) { switch i { case 0: // In block 1, addr1 sends addr2 some ether. @@ -81,7 +84,7 @@ func ExampleGenerateChain() { } // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, DefaultCacheConfig, gspec, dummy.NewCoinbaseFaker(), vm.Config{}, common.Hash{}, false) + blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, dummy.NewCoinbaseFaker(), vm.Config{}, common.Hash{}, false) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/genesis.go b/core/genesis.go index fc39b6f965..76120aad8d 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -197,9 +197,12 @@ func SetupGenesisBlock( } return genesis.Config, block.Hash(), nil } - // We have the genesis block in database but the corresponding state is missing. + // The genesis block is present(perhaps in ancient database) while the + // state database is not initialized yet. It can happen that the node + // is initialized with an external ancient store. Commit genesis state + // in this case. header := rawdb.ReadHeader(db, stored, 0) - if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) { + if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) { // Ensure the stored genesis matches with the given one. hash := genesis.ToBlock().Hash() if hash != stored { @@ -261,7 +264,7 @@ func SetupGenesisBlock( // to the given database (or discards it if nil). func (g *Genesis) ToBlock() *types.Block { db := rawdb.NewMemoryDatabase() - return g.toBlock(db, trie.NewDatabase(db)) + return g.toBlock(db, trie.NewDatabase(db, nil)) } // TODO: migrate this function to "flush" for more similarity with upstream. @@ -339,6 +342,11 @@ func (g *Genesis) toBlock(db ethdb.Database, triedb *trie.Database) *types.Block } } if conf.IsCancun(num, g.Timestamp) { + // EIP-4788: The parentBeaconBlockRoot of the genesis block is always + // the zero hash. This is because the genesis block does not have a parent + // by definition. + head.ParentBeaconRoot = new(common.Hash) + // EIP-4844 fields head.ExcessBlobGas = g.ExcessBlobGas head.BlobGasUsed = g.BlobGasUsed if head.ExcessBlobGas == nil { @@ -389,10 +397,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block // MustCommit writes the genesis block and state to db, panicking on error. // The block is committed as the canonical head block. -// Note the state changes will be committed in hash-based scheme, use Commit -// if path-scheme is preferred. -func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { - block, err := g.Commit(db, trie.NewDatabase(db)) +func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { + block, err := g.Commit(db, triedb) if err != nil { panic(err) } @@ -423,7 +429,7 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big Alloc: GenesisAlloc{addr: {Balance: balance}}, BaseFee: big.NewInt(params.TestMaxBaseFee), } - return g.MustCommit(db) + return g.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) } // ReadBlockByHash reads the block with the given hash from the database. diff --git a/core/genesis_test.go b/core/genesis_test.go index a59beda33f..4667810925 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -41,6 +41,7 @@ import ( "github.com/ava-labs/subnet-evm/precompile/allowlist" "github.com/ava-labs/subnet-evm/precompile/contracts/deployerallowlist" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ava-labs/subnet-evm/utils" "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" @@ -62,6 +63,11 @@ func TestGenesisBlockForTesting(t *testing.T) { } func TestSetupGenesis(t *testing.T) { + testSetupGenesis(t, rawdb.HashScheme) + testSetupGenesis(t, rawdb.PathScheme) +} + +func testSetupGenesis(t *testing.T, scheme string) { preSubnetConfig := *params.TestPreSubnetEVMConfig preSubnetConfig.SubnetEVMTimestamp = utils.NewUint64(100) var ( @@ -79,6 +85,7 @@ func TestSetupGenesis(t *testing.T) { rollbackpreSubnetConfig := preSubnetConfig rollbackpreSubnetConfig.SubnetEVMTimestamp = utils.NewUint64(90) oldcustomg.Config = &rollbackpreSubnetConfig + tests := []struct { name string fn func(ethdb.Database) (*params.ChainConfig, common.Hash, error) @@ -89,7 +96,7 @@ func TestSetupGenesis(t *testing.T) { { name: "genesis without ChainConfig", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return setupGenesisBlock(db, trie.NewDatabase(db), new(Genesis), common.Hash{}) + return setupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis), common.Hash{}) }, wantErr: errGenesisNoConfig, wantConfig: nil, @@ -97,7 +104,7 @@ func TestSetupGenesis(t *testing.T) { { name: "no block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - return setupGenesisBlock(db, trie.NewDatabase(db), nil, common.Hash{}) + return setupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil, common.Hash{}) }, wantErr: ErrNoGenesis, wantConfig: nil, @@ -105,8 +112,9 @@ func TestSetupGenesis(t *testing.T) { { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - customg.MustCommit(db) - return setupGenesisBlock(db, trie.NewDatabase(db), nil, common.Hash{}) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + customg.Commit(db, tdb) + return setupGenesisBlock(db, tdb, nil, common.Hash{}) }, wantErr: ErrNoGenesis, wantConfig: nil, @@ -114,8 +122,9 @@ func TestSetupGenesis(t *testing.T) { { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { - oldcustomg.MustCommit(db) - return setupGenesisBlock(db, trie.NewDatabase(db), &customg, customghash) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + oldcustomg.Commit(db, tdb) + return setupGenesisBlock(db, tdb, &customg, customghash) }, wantHash: customghash, wantConfig: customg.Config, @@ -125,12 +134,19 @@ func TestSetupGenesis(t *testing.T) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { // Commit the 'old' genesis block with SubnetEVM transition at 90. // Advance to block #4, past the SubnetEVM transition block of customg. - genesis := oldcustomg.MustCommit(db) + tdb := trie.NewDatabase(db, newDbConfig(scheme)) + genesis, err := oldcustomg.Commit(db, tdb) + if err != nil { + t.Fatal(err) + } - bc, _ := NewBlockChain(db, DefaultCacheConfig, &oldcustomg, dummy.NewFullFaker(), vm.Config{}, genesis.Hash(), false) + bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, dummy.NewFullFaker(), vm.Config{}, genesis.Hash(), false) defer bc.Stop() - blocks, _, _ := GenerateChain(oldcustomg.Config, genesis, dummy.NewFullFaker(), db, 4, 25, nil) + _, blocks, _, err := GenerateChainWithGenesis(&oldcustomg, dummy.NewFullFaker(), 4, 25, nil) + if err != nil { + t.Fatal(err) + } bc.InsertChain(blocks) for _, block := range blocks { @@ -140,7 +156,7 @@ func TestSetupGenesis(t *testing.T) { } // This should return a compatibility error. - return setupGenesisBlock(db, trie.NewDatabase(db), &customg, bc.lastAccepted.Hash()) + return setupGenesisBlock(db, tdb, &customg, bc.lastAccepted.Hash()) }, wantHash: customghash, wantConfig: customg.Config, @@ -218,7 +234,7 @@ func TestStatefulPrecompilesConfigure(t *testing.T) { genesisBlock := genesis.ToBlock() genesisRoot := genesisBlock.Root() - _, _, err := setupGenesisBlock(db, trie.NewDatabase(db), genesis, genesisBlock.Hash()) + _, _, err := setupGenesisBlock(db, trie.NewDatabase(db, trie.HashDefaults), genesis, genesisBlock.Hash()) if err != nil { t.Fatal(err) } @@ -280,7 +296,7 @@ func TestPrecompileActivationAfterHeaderBlock(t *testing.T) { require.Less(bc.lastAccepted.Time(), *contractDeployerConfig.Timestamp()) // This should not return any error since the last accepted block is before the activation block. - config, _, err := setupGenesisBlock(db, trie.NewDatabase(db), &customg, bc.lastAccepted.Hash()) + config, _, err := setupGenesisBlock(db, trie.NewDatabase(db, nil), &customg, bc.lastAccepted.Hash()) require.NoError(err) if !reflect.DeepEqual(config, customg.Config) { t.Errorf("returned %v\nwant %v", config, customg.Config) @@ -299,8 +315,8 @@ func TestGenesisWriteUpgradesRegression(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock := genesis.ToBlock() - trieDB := trie.NewDatabase(db) + trieDB := trie.NewDatabase(db, trie.HashDefaults) + genesisBlock := genesis.MustCommit(db, trieDB) _, _, err := SetupGenesisBlock(db, trieDB, genesis, genesisBlock.Hash(), false) require.NoError(err) @@ -328,3 +344,10 @@ func TestGenesisWriteUpgradesRegression(t *testing.T) { _, _, err = SetupGenesisBlock(db, trieDB, genesis, lastAcceptedBlock.Hash(), false) require.NoError(err) } + +func newDbConfig(scheme string) *trie.Config { + if scheme == rawdb.HashScheme { + return trie.HashDefaults + } + return &trie.Config{PathDB: pathdb.Defaults} +} diff --git a/core/mkalloc.go b/core/mkalloc.go index 4ab78b31b7..bfb68eed28 100644 --- a/core/mkalloc.go +++ b/core/mkalloc.go @@ -43,23 +43,50 @@ import ( "strconv" "github.com/ava-labs/subnet-evm/core" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/exp/slices" ) -type allocItem struct{ Addr, Balance *big.Int } +type allocItem struct { + Addr *big.Int + Balance *big.Int + Misc *allocItemMisc `rlp:"optional"` +} + +type allocItemMisc struct { + Nonce uint64 + Code []byte + Slots []allocItemStorageItem +} + +type allocItemStorageItem struct { + Key common.Hash + Val common.Hash +} func makelist(g *core.Genesis) []allocItem { items := make([]allocItem, 0, len(g.Alloc)) for addr, account := range g.Alloc { + var misc *allocItemMisc if len(account.Storage) > 0 || len(account.Code) > 0 || account.Nonce != 0 { - panic(fmt.Sprintf("can't encode account %x", addr)) + misc = &allocItemMisc{ + Nonce: account.Nonce, + Code: account.Code, + Slots: make([]allocItemStorageItem, 0, len(account.Storage)), + } + for key, val := range account.Storage { + misc.Slots = append(misc.Slots, allocItemStorageItem{key, val}) + } + slices.SortFunc(misc.Slots, func(a, b allocItemStorageItem) int { + return a.Key.Cmp(b.Key) + }) } bigAddr := new(big.Int).SetBytes(addr.Bytes()) - items = append(items, allocItem{bigAddr, account.Balance}) + items = append(items, allocItem{bigAddr, account.Balance, misc}) } - slices.SortFunc(items, func(a, b allocItem) bool { - return a.Addr.Cmp(b.Addr) < 0 + slices.SortFunc(items, func(a, b allocItem) int { + return a.Addr.Cmp(b.Addr) }) return items } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 47dd95abf9..eca8abfbc7 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -227,7 +227,7 @@ func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header return nil } header := new(types.Header) - if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + if err := rlp.DecodeBytes(data, header); err != nil { log.Error("Invalid block header RLP", "hash", hash, "err", err) return nil } @@ -313,7 +313,7 @@ func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body { return nil } body := new(types.Body) - if err := rlp.Decode(bytes.NewReader(data), body); err != nil { + if err := rlp.DecodeBytes(data, body); err != nil { log.Error("Invalid block body RLP", "hash", hash, "err", err) return nil } diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 8267fea4ec..eafc43d83e 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -127,10 +127,10 @@ const crashesToKeep = 10 func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) { var uncleanShutdowns crashList // Read old data - if data, err := db.Get(uncleanShutdownKey); err != nil { - log.Warn("Error reading unclean shutdown markers", "error", err) - } else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { - return nil, 0, err + if data, err := db.Get(uncleanShutdownKey); err == nil { + if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil { + return nil, 0, err + } } var discarded = uncleanShutdowns.Discarded var previous = make([]uint64, len(uncleanShutdowns.Recent)) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 4e020e11ca..7b85c528ee 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -46,7 +46,7 @@ import ( // // Now this scheme is still kept for backward compatibility, and it will be used // for archive node and some other tries(e.g. light trie). -const HashScheme = "hashScheme" +const HashScheme = "hash" // PathScheme is the new path-based state scheme with which trie nodes are stored // in the disk with node path as the database key. This scheme will only store one @@ -54,7 +54,7 @@ const HashScheme = "hashScheme" // is native. At the same time, this scheme will put adjacent trie nodes in the same // area of the disk with good data locality property. But this scheme needs to rely // on extra state diffs to survive deep reorg. -const PathScheme = "pathScheme" +const PathScheme = "path" // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } @@ -99,6 +99,16 @@ func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) return h.hash(data) == hash } +// ExistsAccountTrieNode checks the presence of the account trie node with the +// specified node path, regardless of the node hash. +func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool { + has, err := db.Has(accountTrieNodeKey(path)) + if err != nil { + return false + } + return has +} + // WriteAccountTrieNode writes the provided account trie node into database. func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) { if err := db.Put(accountTrieNodeKey(path), node); err != nil { @@ -137,6 +147,16 @@ func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path [ return h.hash(data) == hash } +// ExistsStorageTrieNode checks the presence of the storage trie node with the +// specified account hash and node path, regardless of the node hash. +func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool { + has, err := db.Has(storageTrieNodeKey(accountHash, path)) + if err != nil { + return false + } + return has +} + // WriteStorageTrieNode writes the provided storage trie node into database. func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) { if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil { @@ -273,3 +293,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has panic(fmt.Sprintf("Unknown scheme %v", scheme)) } } + +// ReadStateScheme reads the state scheme of persistent state, or none +// if the state is not present in database. +func ReadStateScheme(db ethdb.Reader) string { + // Check if state in path-based scheme is present + blob, _ := ReadAccountTrieNode(db, nil) + if len(blob) != 0 { + return PathScheme + } + // In a hash-based scheme, the genesis state is consistently stored + // on the disk. To assess the scheme of the persistent state, it + // suffices to inspect the scheme of the genesis state. + header := ReadHeader(db, ReadCanonicalHash(db, 0), 0) + if header == nil { + return "" // empty datadir + } + blob = ReadLegacyTrieNode(db, header.Root) + if len(blob) == 0 { + return "" // no state in disk + } + return HashScheme +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index e0e85fb3b7..c16887fa6f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -159,10 +159,10 @@ const ( dbLeveldb = "leveldb" ) -// hasPreexistingDb checks the given data directory whether a database is already +// PreexistingDatabase checks the given data directory whether a database is already // instantiated at that location, and if so, returns the type of database (or the // empty string). -func hasPreexistingDb(path string) string { +func PreexistingDatabase(path string) string { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return "" // No pre-existing db } @@ -184,6 +184,9 @@ type OpenOptions struct { Cache int // the capacity(in megabytes) of the data caching Handles int // number of files to be open simultaneously ReadOnly bool + // Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of + // a crash is not important. This option should typically be used in tests. + Ephemeral bool } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -199,14 +202,14 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { } // Retrieve any pre-existing database's type and use that or the requested one // as long as there's no conflict between the two types - existingDb := hasPreexistingDb(o.Directory) + existingDb := PreexistingDatabase(o.Directory) if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb { return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) } if o.Type == dbPebble || existingDb == dbPebble { if PebbleEnabled { log.Info("Using pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } else { return nil, errors.New("db.engine 'pebble' not supported on this platform") } @@ -219,7 +222,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { // on supported platforms and LevelDB on anything else. if PebbleEnabled { log.Info("Defaulting to pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } else { log.Info("Defaulting to leveldb as the backing database") return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) @@ -286,7 +289,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { receipts stat numHashPairings stat hashNumPairings stat - tries stat + legacyTries stat + stateLookups stat + accountTries stat + storageTries stat codes stat txLookups stat accountSnaps stat @@ -330,8 +336,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { numHashPairings.Add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.Add(size) - case len(key) == common.HashLength: - tries.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) + case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: + stateLookups.Add(size) + case IsAccountTrieNode(key): + accountTries.Add(size) + case IsStorageTrieNode(key): + storageTries.Add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: codes.Add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): @@ -364,6 +376,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { databaseVersionKey, headHeaderKey, headBlockKey, snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey, uncleanShutdownKey, syncRootKey, txIndexTailKey, + persistentStateIDKey, trieJournalKey, } { if bytes.Equal(key, meta) { metadata.Add(size) @@ -391,7 +404,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Trie nodes", tries.Size(), tries.Count()}, + {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, + {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, + {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, + {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, diff --git a/core/rawdb/databases_64bit.go b/core/rawdb/databases_64bit.go index 73bfeb2083..1593e89bfe 100644 --- a/core/rawdb/databases_64bit.go +++ b/core/rawdb/databases_64bit.go @@ -28,8 +28,8 @@ const PebbleEnabled = true // NewPebbleDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly) +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) if err != nil { return nil, err } diff --git a/core/rawdb/databases_non64bit.go b/core/rawdb/databases_non64bit.go index 65ad802a6f..aff38c7867 100644 --- a/core/rawdb/databases_non64bit.go +++ b/core/rawdb/databases_non64bit.go @@ -39,6 +39,6 @@ const PebbleEnabled = false // NewPebbleDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { return nil, errors.New("pebble is not supported on this platform") } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 261169ba13..78d1cb4a72 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -184,7 +184,11 @@ func accountSnapshotKey(hash common.Hash) []byte { // storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash func storageSnapshotKey(accountHash, storageHash common.Hash) []byte { - return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...) + buf := make([]byte, len(SnapshotStoragePrefix)+common.HashLength+common.HashLength) + n := copy(buf, SnapshotStoragePrefix) + n += copy(buf[n:], accountHash.Bytes()) + copy(buf[n:], storageHash.Bytes()) + return buf } // storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash @@ -243,7 +247,11 @@ func accountTrieNodeKey(path []byte) []byte { // storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...) + buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path)) + n := copy(buf, trieNodeStoragePrefix) + n += copy(buf[n:], accountHash.Bytes()) + copy(buf[n:], path) + return buf } // IsLegacyTrieNode reports whether a provided database entry is a legacy trie @@ -257,9 +265,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { return bytes.Equal(key, crypto.Keccak256(val)) } -// IsAccountTrieNode reports whether a provided database entry is an account -// trie node in path-based state scheme. -func IsAccountTrieNode(key []byte) (bool, []byte) { +// ResolveAccountTrieNodeKey reports whether a provided database entry is an +// account trie node in path-based state scheme, and returns the resolved +// node path if so. +func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { if !bytes.HasPrefix(key, trieNodeAccountPrefix) { return false, nil } @@ -272,9 +281,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) { return true, key[len(trieNodeAccountPrefix):] } -// IsStorageTrieNode reports whether a provided database entry is a storage +// IsAccountTrieNode reports whether a provided database entry is an account // trie node in path-based state scheme. -func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { +func IsAccountTrieNode(key []byte) bool { + ok, _ := ResolveAccountTrieNodeKey(key) + return ok +} + +// ResolveStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme, and returns the resolved account hash +// and node path if so. +func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { if !bytes.HasPrefix(key, trieNodeStoragePrefix) { return false, common.Hash{}, nil } @@ -290,3 +307,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) { accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] } + +// IsStorageTrieNode reports whether a provided database entry is a storage +// trie node in path-based state scheme. +func IsStorageTrieNode(key []byte) bool { + ok, _, _ := ResolveStorageTrieNode(key) + return ok +} diff --git a/core/state/database.go b/core/state/database.go index 15d1367b56..e9f5cb4cb1 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -68,7 +68,7 @@ type Database interface { // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore - // TrieDB retrieves the low level trie database used for data storage. + // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *trie.Database } @@ -157,7 +157,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { disk: db, codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: trie.NewDatabaseWithConfig(db, config), + triedb: trie.NewDatabase(db, config), } } diff --git a/core/state/iterator.go b/core/state/iterator.go index 9129ce16ca..c41e71b46a 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -122,7 +122,7 @@ func (it *nodeIterator) step() error { } // Otherwise we've reached an account node, initiate data iteration var account types.StateAccount - if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { + if err := rlp.DecodeBytes(it.stateIt.LeafBlob(), &account); err != nil { return err } // Lookup the preimage of account hash diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index c4a9f0c74a..6eae770a24 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -36,9 +36,14 @@ import ( // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { + testNodeIteratorCoverage(t, rawdb.HashScheme) + testNodeIteratorCoverage(t, rawdb.PathScheme) +} + +func testNodeIteratorCoverage(t *testing.T, scheme string) { // Create some arbitrary test state to iterate - db, sdb, root, _ := makeTestState() - sdb.TrieDB().Commit(root, false) + db, sdb, ndb, root, _ := makeTestState(scheme) + ndb.Commit(root, false) state, err := New(root, sdb, nil) if err != nil { @@ -58,7 +63,7 @@ func TestNodeIteratorCoverage(t *testing.T) { ) it := db.NewIterator(nil, nil) for it.Next() { - ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value()) + ok, hash := isTrieNode(scheme, it.Key(), it.Value()) if !ok { continue } @@ -100,11 +105,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) { return true, common.BytesToHash(key) } } else { - ok, _ := rawdb.IsAccountTrieNode(key) + ok := rawdb.IsAccountTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } - ok, _, _ = rawdb.IsStorageTrieNode(key) + ok = rawdb.IsStorageTrieNode(key) if ok { return true, crypto.Keccak256Hash(val) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 1a5a342384..6f24b27279 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -95,16 +95,20 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { if headBlock == nil { return nil, errors.New("failed to load head block") } + // Offline pruning is only supported in legacy hash based scheme. + triedb := trie.NewDatabase(db, trie.HashDefaults) + // Note: we refuse to start a pruning session unless the snapshot disk layer exists, which should prevent // us from ever needing to enter RecoverPruning in an invalid pruning session (a session where we do not have // the protected trie in the triedb and in the snapshot disk layer). + snapconfig := snapshot.Config{ CacheSize: 256, AsyncBuild: false, NoBuild: true, SkipVerify: true, } - snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Hash(), headBlock.Root()) + snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Hash(), headBlock.Root()) if err != nil { return nil, fmt.Errorf("failed to create snapshot for pruning, must restart without offline pruning disabled to recover: %w", err) // The relevant snapshot(s) might not exist } @@ -342,7 +346,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults)) if err != nil { return err } @@ -366,7 +370,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { } if acc.Root != types.EmptyRootHash { id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) - storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults)) if err != nil { return err } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 9eb19764dc..84f0bdbf6f 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -36,6 +36,8 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" @@ -57,9 +59,15 @@ func hashData(input []byte) common.Hash { // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { + testGeneration(t, rawdb.HashScheme) + testGeneration(t, rawdb.PathScheme) +} + +func testGeneration(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, - var helper = newHelper() + // two of which also has the same 3-slot storage trie attached. + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -91,10 +99,15 @@ func TestGeneration(t *testing.T) { // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { + testGenerateExistentState(t, rawdb.HashScheme) + testGenerateExistentState(t, rawdb.PathScheme) +} + +func testGenerateExistentState(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var helper = newHelper() + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -158,9 +171,15 @@ type testHelper struct { nodes *trienode.MergedNodeSet } -func newHelper() *testHelper { +func newHelper(scheme string) *testHelper { diskdb := rawdb.NewMemoryDatabase() - triedb := trie.NewDatabase(diskdb) + config := &trie.Config{} + if scheme == rawdb.PathScheme { + config.PathDB = &pathdb.Config{} // disable caching + } else { + config.HashDB = &hashdb.Config{} // disable caching + } + triedb := trie.NewDatabase(diskdb, config) accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb) return &testHelper{ diskdb: diskdb, @@ -243,7 +262,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { // - extra slots in the middle // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { - helper := newHelper() + testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) { + helper := newHelper(scheme) // Account one, empty root but non-empty database helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -335,7 +359,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - wrong accounts // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { - helper := newHelper() + testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme) + testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme) +} + +func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -390,10 +419,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in the account trie. func TestGenerateCorruptAccountTrie(t *testing.T) { + testGenerateCorruptAccountTrie(t, rawdb.HashScheme) + testGenerateCorruptAccountTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptAccountTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - helper := newHelper() + helper := newHelper(scheme) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 @@ -401,9 +435,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 - // Delete an account trie leaf and ensure the generator chokes - helper.triedb.Commit(root, false) - helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + // Delete an account trie node and ensure the generator chokes + targetPath := []byte{0xc} + targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7") + + rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) select { @@ -424,11 +460,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // trie node for a storage trie. It's similar to internal corruption but it is // handled differently inside the generator. func TestGenerateMissingStorageTrie(t *testing.T) { + testGenerateMissingStorageTrie(t, rawdb.HashScheme) + testGenerateMissingStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateMissingStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() - + var ( + acc1 = hashData([]byte("acc-1")) + acc3 = hashData([]byte("acc-3")) + helper = newHelper(scheme) + ) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 @@ -437,8 +481,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) { root := helper.Commit() - // Delete a storage trie root and ensure the generator chokes - helper.diskdb.Delete(stRoot.Bytes()) + // Delete storage trie root of account one and three. + rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme) + rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) select { @@ -458,10 +503,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // Tests that snapshot generation errors out correctly in case of a missing trie // node in a storage trie. func TestGenerateCorruptStorageTrie(t *testing.T) { + testGenerateCorruptStorageTrie(t, rawdb.HashScheme) + testGenerateCorruptStorageTrie(t, rawdb.PathScheme) +} + +func testGenerateCorruptStorageTrie(t *testing.T, scheme string) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - helper := newHelper() + helper := newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e @@ -471,8 +521,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { root := helper.Commit() - // Delete a storage trie leaf and ensure the generator chokes - helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + // Delete a node in the storage trie. + targetPath := []byte{0x4} + targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371") + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme) + rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme) snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) select { @@ -491,7 +544,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - helper := newHelper() + testGenerateWithExtraAccounts(t, rawdb.HashScheme) + testGenerateWithExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithExtraAccounts(t *testing.T, scheme string) { + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), @@ -559,10 +617,15 @@ func enableLogging() { // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithManyExtraAccounts(t *testing.T) { + testGenerateWithManyExtraAccounts(t, rawdb.HashScheme) + testGenerateWithManyExtraAccounts(t, rawdb.PathScheme) +} + +func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { // Account one in the trie stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), @@ -615,10 +678,15 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { + testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme) + testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme) +} + +func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) { if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -651,10 +719,15 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { // TestGenerateWithMalformedSnapdata tests what happes if we have some junk // in the snapshot database, which cannot be parsed back to an account func TestGenerateWithMalformedSnapdata(t *testing.T) { + testGenerateWithMalformedSnapdata(t, rawdb.HashScheme) + testGenerateWithMalformedSnapdata(t, rawdb.PathScheme) +} + +func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) { if false { enableLogging() } - helper := newHelper() + helper := newHelper(scheme) { acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) @@ -687,8 +760,13 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { } func TestGenerateFromEmptySnap(t *testing.T) { + testGenerateFromEmptySnap(t, rawdb.HashScheme) + testGenerateFromEmptySnap(t, rawdb.PathScheme) +} + +func testGenerateFromEmptySnap(t *testing.T, scheme string) { //enableLogging() - helper := newHelper() + helper := newHelper(scheme) // Add 1K accounts to the trie for i := 0; i < 400; i++ { stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) @@ -720,7 +798,12 @@ func TestGenerateFromEmptySnap(t *testing.T) { // This hits a case where the snap verification passes, but there are more elements in the trie // which we must also add. func TestGenerateWithIncompleteStorage(t *testing.T) { - helper := newHelper() + testGenerateWithIncompleteStorage(t, rawdb.HashScheme) + testGenerateWithIncompleteStorage(t, rawdb.PathScheme) +} + +func testGenerateWithIncompleteStorage(t *testing.T, scheme string) { + helper := newHelper(scheme) stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} // We add 8 accounts, each one is missing exactly one of the storage slots. This means @@ -818,7 +901,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { - var helper = newHelper() + testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme) + testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme) +} + +func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) { + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) @@ -853,7 +941,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { - var helper = newHelper() + testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme) + testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme) +} + +func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) { + var helper = newHelper(scheme) stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()}) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 2b29225d89..f176de7b54 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -38,7 +38,6 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/trie" - "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -921,51 +920,20 @@ func (t *Tree) DiskRoot() common.Hash { return t.diskRoot() } -func (t *Tree) DiskAccountIterator(seek common.Hash) AccountIterator { - t.lock.Lock() - defer t.lock.Unlock() - - return t.disklayer().AccountIterator(seek) -} - -func (t *Tree) DiskStorageIterator(account common.Hash, seek common.Hash) StorageIterator { - t.lock.Lock() - defer t.lock.Unlock() - - it, _ := t.disklayer().StorageIterator(account, seek) - return it -} - -// NewDiskLayer creates a diskLayer for direct access to the contents of the on-disk -// snapshot. Does not perform any validation. -func NewDiskLayer(diskdb ethdb.KeyValueStore) Snapshot { - return &diskLayer{ - diskdb: diskdb, - created: time.Now(), - - // state sync uses iterators to access data, so this cache is not used. - // initializing it out of caution. - cache: utils.NewMeteredCache(32*1024, "", 0), - } -} +// Size returns the memory usage of the diff layers above the disk layer and the +// dirty nodes buffered in the disk layer. Currently, the implementation uses a +// special diff layer (the first) as an aggregator simulating a dirty buffer, so +// the second return will always be 0. However, this will be made consistent with +// the pathdb, which will require a second return. +func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) { + t.lock.RLock() + defer t.lock.RUnlock() -// NewTestTree creates a *Tree with a pre-populated diskLayer -func NewTestTree(diskdb ethdb.KeyValueStore, blockHash, root common.Hash) *Tree { - base := &diskLayer{ - diskdb: diskdb, - root: root, - blockHash: blockHash, - cache: utils.NewMeteredCache(128*256, "", 0), - created: time.Now(), - } - return &Tree{ - blockLayers: map[common.Hash]snapshot{ - blockHash: base, - }, - stateLayers: map[common.Hash]map[common.Hash]snapshot{ - root: { - blockHash: base, - }, - }, + var size common.StorageSize + for _, layer := range t.blockLayers { + if layer, ok := layer.(*diffLayer); ok { + size += common.StorageSize(layer.memory) + } } + return size, 0 } diff --git a/core/state/snapshot/snapshot_ext.go b/core/state/snapshot/snapshot_ext.go new file mode 100644 index 0000000000..ee06542c19 --- /dev/null +++ b/core/state/snapshot/snapshot_ext.go @@ -0,0 +1,58 @@ +package snapshot + +import ( + "time" + + "github.com/ava-labs/subnet-evm/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" +) + +func (t *Tree) DiskAccountIterator(seek common.Hash) AccountIterator { + t.lock.Lock() + defer t.lock.Unlock() + + return t.disklayer().AccountIterator(seek) +} + +func (t *Tree) DiskStorageIterator(account common.Hash, seek common.Hash) StorageIterator { + t.lock.Lock() + defer t.lock.Unlock() + + it, _ := t.disklayer().StorageIterator(account, seek) + return it +} + +// NewDiskLayer creates a diskLayer for direct access to the contents of the on-disk +// snapshot. Does not perform any validation. +func NewDiskLayer(diskdb ethdb.KeyValueStore) Snapshot { + return &diskLayer{ + diskdb: diskdb, + created: time.Now(), + + // state sync uses iterators to access data, so this cache is not used. + // initializing it out of caution. + cache: utils.NewMeteredCache(32*1024, "", 0), + } +} + +// NewTestTree creates a *Tree with a pre-populated diskLayer +func NewTestTree(diskdb ethdb.KeyValueStore, blockHash, root common.Hash) *Tree { + base := &diskLayer{ + diskdb: diskdb, + root: root, + blockHash: blockHash, + cache: utils.NewMeteredCache(128*256, "", 0), + created: time.Now(), + } + return &Tree{ + blockLayers: map[common.Hash]snapshot{ + blockHash: base, + }, + stateLayers: map[common.Hash]map[common.Hash]snapshot{ + root: { + blockHash: base, + }, + }, + } +} diff --git a/core/state/state_object.go b/core/state/state_object.go index 8846b81e49..69e55a6c49 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -274,12 +274,17 @@ func (s *stateObject) finalise(prefetch bool) { } } -// updateTrie writes cached storage modifications into the object's storage trie. -// It will return nil if the trie has not been loaded and no changes have been -// made. An error will be returned if the trie can't be loaded/updated correctly. +// updateTrie is responsible for persisting cached storage changes into the +// object's storage trie. In case the storage trie is not yet loaded, this +// function will load the trie automatically. If any issues arise during the +// loading or updating of the trie, an error will be returned. Furthermore, +// this function will return the mutated storage trie, or nil if there is no +// storage change at all. func (s *stateObject) updateTrie() (Trie, error) { // Make sure all dirty slots are finalized into the pending storage area - s.finalise(false) // Don't prefetch anymore, pull directly if need be + s.finalise(false) + + // Short circuit if nothing changed, don't bother with hashing anything if len(s.pendingStorage) == 0 { return s.trie, nil } @@ -291,14 +296,13 @@ func (s *stateObject) updateTrie() (Trie, error) { var ( storage map[common.Hash][]byte origin map[common.Hash][]byte - hasher = s.db.hasher ) tr, err := s.getTrie() if err != nil { s.db.setError(err) return nil, err } - // Insert all the pending updates into the trie + // Insert all the pending storage updates into the trie usedStorage := make([][]byte, 0, len(s.pendingStorage)) for key, value := range s.pendingStorage { // Skip noop changes, persist actual changes @@ -308,8 +312,7 @@ func (s *stateObject) updateTrie() (Trie, error) { prev := s.originStorage[key] s.originStorage[key] = value - // rlp-encoded value to be used by the snapshot - var snapshotVal []byte + var encoded []byte // rlp-encoded value to be used by the snapshot if (value == common.Hash{}) { if err := tr.DeleteStorage(s.address, key[:]); err != nil { s.db.setError(err) @@ -317,10 +320,10 @@ func (s *stateObject) updateTrie() (Trie, error) { } s.db.StorageDeleted += 1 } else { - trimmedVal := common.TrimLeftZeroes(value[:]) // Encoding []byte cannot fail, ok to ignore the error. - snapshotVal, _ = rlp.EncodeToBytes(trimmedVal) - if err := tr.UpdateStorage(s.address, key[:], trimmedVal); err != nil { + trimmed := common.TrimLeftZeroes(value[:]) + encoded, _ = rlp.EncodeToBytes(trimmed) + if err := tr.UpdateStorage(s.address, key[:], trimmed); err != nil { s.db.setError(err) return nil, err } @@ -333,8 +336,8 @@ func (s *stateObject) updateTrie() (Trie, error) { s.db.storages[s.addrHash] = storage } } - khash := crypto.HashData(hasher, key[:]) - storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted + khash := crypto.HashData(s.db.hasher, key[:]) + storage[khash] = encoded // encoded will be nil if it's deleted // Cache the original value of mutated storage slots if origin == nil { @@ -359,21 +362,17 @@ func (s *stateObject) updateTrie() (Trie, error) { if s.db.prefetcher != nil { s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) } - if len(s.pendingStorage) > 0 { - s.pendingStorage = make(Storage) - } + s.pendingStorage = make(Storage) // reset pending map return tr, nil } -// UpdateRoot sets the trie root to the current root hash of. An error -// will be returned if trie root hash is not computed correctly. +// updateRoot flushes all cached storage mutations to trie, recalculating the +// new storage trie root. func (s *stateObject) updateRoot() { + // Flush cached storage mutations into trie, short circuit if any error + // is occurred or there is not change in the trie. tr, err := s.updateTrie() - if err != nil { - return - } - // If nothing changed, don't bother with hashing anything - if tr == nil { + if err != nil || tr == nil { return } // Track the amount of time wasted on hashing the storage trie @@ -383,14 +382,12 @@ func (s *stateObject) updateRoot() { s.data.Root = tr.Hash() } -// commit returns the changes made in storage trie and updates the account data. +// commit obtains a set of dirty storage trie nodes and updates the account data. +// The returned set can be nil if nothing to commit. This function assumes all +// storage mutations have already been flushed into trie by updateRoot. func (s *stateObject) commit() (*trienode.NodeSet, error) { - tr, err := s.updateTrie() - if err != nil { - return nil, err - } - // If nothing changed, don't bother with committing anything - if tr == nil { + // Short circuit if trie is not even loaded, don't bother with committing anything + if s.trie == nil { s.origin = s.data.Copy() return nil, nil } @@ -398,7 +395,10 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) { if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, nodes, err := tr.Commit(false) + // The trie is currently in an open state and could potentially contain + // cached mutations. Call commit to acquire a set of nodes that have been + // modified, the set can be nil if nothing to commit. + root, nodes, err := s.trie.Commit(false) if err != nil { return nil, err } @@ -546,3 +546,7 @@ func (s *stateObject) Balance() *big.Int { func (s *stateObject) Nonce() uint64 { return s.data.Nonce } + +func (s *stateObject) Root() common.Hash { + return s.data.Root +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 6fc489c228..adb20f5d0d 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ package state import ( - "errors" "fmt" "math/big" "sort" @@ -46,7 +45,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" +) + +const ( + // storageDeleteLimit denotes the highest permissible memory allocation + // employed for contract storage deletion. + storageDeleteLimit = 512 * 1024 * 1024 ) type revision struct { @@ -54,17 +58,6 @@ type revision struct { journalIndex int } -type proofList [][]byte - -func (n *proofList) Put(key []byte, value []byte) error { - *n = append(*n, value) - return nil -} - -func (n *proofList) Delete(key []byte) error { - panic("not supported") -} - // StateDB structs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: @@ -155,6 +148,9 @@ type StateDB struct { StorageUpdated int AccountDeleted int StorageDeleted int + + // Testing hooks + onCommit func(states *triestate.Set) // Hook invoked when commit is performed } // New creates a new state from a given trie. @@ -345,6 +341,7 @@ func (s *StateDB) GetBalance(addr common.Address) *big.Int { return new(big.Int).Set(common.Big0) } +// GetNonce retrieves the nonce from the given address or 0 if object not found func (s *StateDB) GetNonce(addr common.Address) uint64 { stateObject := s.getStateObject(addr) if stateObject != nil { @@ -354,6 +351,16 @@ func (s *StateDB) GetNonce(addr common.Address) uint64 { return 0 } +// GetStorageRoot retrieves the storage root from the given address or empty +// if object not found. +func (s *StateDB) GetStorageRoot(addr common.Address) common.Hash { + stateObject := s.getStateObject(addr) + if stateObject != nil { + return stateObject.Root() + } + return common.Hash{} +} + // TxIndex returns the current transaction index set by Prepare. func (s *StateDB) TxIndex() int { return s.txIndex @@ -392,35 +399,6 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { return common.Hash{} } -// GetProof returns the Merkle proof for a given account. -func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) { - return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes())) -} - -// GetProofByHash returns the Merkle proof for a given account. -func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) { - var proof proofList - err := s.trie.Prove(addrHash[:], &proof) - return proof, err -} - -// GetStorageProof returns the Merkle proof for given storage slot. -func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) { - trie, err := s.StorageTrie(a) - if err != nil { - return nil, err - } - if trie == nil { - return nil, errors.New("storage trie for requested address does not exist") - } - var proof proofList - err = trie.Prove(crypto.Keccak256(key.Bytes()), &proof) - if err != nil { - return nil, err - } - return proof, nil -} - // GetCommittedState retrieves a value from the given account's committed storage trie. func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) @@ -435,21 +413,6 @@ func (s *StateDB) Database() Database { return s.db } -// StorageTrie returns the storage trie of an account. The return value is a copy -// and is nil for non-existent accounts. An error will be returned if storage trie -// is existent but can't be loaded correctly. -func (s *StateDB) StorageTrie(addr common.Address) (Trie, error) { - stateObject := s.getStateObject(addr) - if stateObject == nil { - return nil, nil - } - cpy := stateObject.deepCopy(s) - if _, err := cpy.updateTrie(); err != nil { - return nil, err - } - return cpy.getTrie() -} - func (s *StateDB) HasSelfDestructed(addr common.Address) bool { stateObject := s.getStateObject(addr) if stateObject != nil { @@ -775,43 +738,6 @@ func (s *StateDB) CreateAccount(addr common.Address) { } } -func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { - so := s.getStateObject(addr) - if so == nil { - return nil - } - tr, err := so.getTrie() - if err != nil { - return err - } - trieIt, err := tr.NodeIterator(nil) - if err != nil { - return err - } - it := trie.NewIterator(trieIt) - - for it.Next() { - key := common.BytesToHash(s.trie.GetKey(it.Key)) - if value, dirty := so.dirtyStorage[key]; dirty { - if !cb(key, value) { - return nil - } - continue - } - - if len(it.Value) > 0 { - _, content, _, err := rlp.Split(it.Value) - if err != nil { - return err - } - if !cb(key, common.BytesToHash(content)) { - return nil - } - } - } - return nil -} - // copyPredicateStorageSlots creates a deep copy of the provided predicateStorageSlots map. func copyPredicateStorageSlots(predicateStorageSlots map[common.Address][][]byte) map[common.Address][][]byte { res := make(map[common.Address][][]byte, len(predicateStorageSlots)) @@ -1088,59 +1014,125 @@ func (s *StateDB) clearJournalAndRefund() { s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } -// deleteStorage iterates the storage trie belongs to the account and mark all -// slots inside as deleted. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { - start := time.Now() +// fastDeleteStorage is the function that efficiently deletes the storage trie +// of a specific account. It leverages the associated state snapshot for fast +// storage iteration and constructs trie node deletion markers by creating +// stack trie with iterated slots. +func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { + iter, _ := s.snap.StorageIterator(addrHash, common.Hash{}) + defer iter.Release() + + var ( + size common.StorageSize + nodes = trienode.NewNodeSet(addrHash) + slots = make(map[common.Hash][]byte) + ) + stack := trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + nodes.AddNode(path, trienode.NewDeleted()) + size += common.StorageSize(len(path)) + }) + for iter.Next() { + if size > storageDeleteLimit { + return true, size, nil, nil, nil + } + slot := common.CopyBytes(iter.Slot()) + if err := iter.Error(); err != nil { // error might occur after Slot function + return false, 0, nil, nil, err + } + size += common.StorageSize(common.HashLength + len(slot)) + slots[iter.Hash()] = slot + + if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { + return false, 0, nil, nil, err + } + } + if err := iter.Error(); err != nil { // error might occur during iteration + return false, 0, nil, nil, err + } + if stack.Hash() != root { + return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + } + return false, size, slots, nodes, nil +} + +// slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," +// employed when the associated state snapshot is not available. It iterates the +// storage slots along with all internal trie nodes via trie directly. +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root) if err != nil { - return false, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return false, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( - set = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) - stateSize common.StorageSize - nodeSize common.StorageSize + size common.StorageSize + nodes = trienode.NewNodeSet(addrHash) + slots = make(map[common.Hash][]byte) ) for it.Next(true) { - // arbitrary stateSize limit, make it configurable - if stateSize+nodeSize > 512*1024*1024 { - log.Info("Skip large storage deletion", "address", addr.Hex(), "states", stateSize, "nodes", nodeSize) - if metrics.EnabledExpensive { - slotDeletionSkip.Inc(1) - } - return true, nil, nil, nil + if size > storageDeleteLimit { + return true, size, nil, nil, nil } if it.Leaf() { slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) - stateSize += common.StorageSize(common.HashLength + len(it.LeafBlob())) + size += common.StorageSize(common.HashLength + len(it.LeafBlob())) continue } if it.Hash() == (common.Hash{}) { continue } - nodeSize += common.StorageSize(len(it.Path())) - set.AddNode(it.Path(), trienode.NewDeleted()) + size += common.StorageSize(len(it.Path())) + nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { + return false, 0, nil, nil, err + } + return false, size, slots, nodes, nil +} + +// deleteStorage is designed to delete the storage trie of a designated account. +// It could potentially be terminated if the storage size is excessively large, +// potentially leading to an out-of-memory panic. The function will make an attempt +// to utilize an efficient strategy if the associated state snapshot is reachable; +// otherwise, it will resort to a less-efficient approach. +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { + var ( + start = time.Now() + err error + aborted bool + size common.StorageSize + slots map[common.Hash][]byte + nodes *trienode.NodeSet + ) + // The fast approach can be failed if the snapshot is not fully + // generated, or it's internally corrupted. Fallback to the slow + // one just in case. + if s.snap != nil { + aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root) + } + if s.snap == nil || err != nil { + aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + } + if err != nil { return false, nil, nil, err } if metrics.EnabledExpensive { - if int64(len(slots)) > slotDeletionMaxCount.Value() { - slotDeletionMaxCount.Update(int64(len(slots))) - } - if int64(stateSize+nodeSize) > slotDeletionMaxSize.Value() { - slotDeletionMaxSize.Update(int64(stateSize + nodeSize)) + if aborted { + slotDeletionSkip.Inc(1) } + n := int64(len(slots)) + + slotDeletionMaxCount.UpdateIfGt(int64(len(slots))) + slotDeletionMaxSize.UpdateIfGt(int64(size)) + slotDeletionTimer.UpdateSince(start) - slotDeletionCount.Mark(int64(len(slots))) - slotDeletionSize.Mark(int64(stateSize + nodeSize)) + slotDeletionCount.Mark(n) + slotDeletionSize.Mark(int64(size)) } - return false, slots, set, nil + return aborted, slots, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1168,7 +1160,13 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root // In case (d), **original** account along with its storages should be deleted, // with their values be tracked as original value. func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) { + // Short circuit if geth is running with hash mode. This procedure can consume + // considerable time and storage deletion isn't supported in hash mode, thus + // preemptively avoiding unnecessary expenses. incomplete := make(map[common.Address]struct{}) + if s.db.TrieDB().Scheme() == rawdb.HashScheme { + return incomplete, nil + } for addr, prev := range s.stateObjectsDestruct { // The original account was non-existing, and it's marked as destructed // in the scope of block. It can be case (a) or (b). @@ -1345,12 +1343,13 @@ func (s *StateDB) commit(block uint64, deleteEmptyObjects bool, snaps *snapshot. } if root != origin { start := time.Now() + set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete) if referenceRoot { - if err := s.db.TrieDB().UpdateAndReferenceRoot(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil { + if err := s.db.TrieDB().UpdateAndReferenceRoot(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } } else { - if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil { + if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } } @@ -1358,6 +1357,9 @@ func (s *StateDB) commit(block uint64, deleteEmptyObjects bool, snaps *snapshot. if metrics.EnabledExpensive { s.TrieDBCommits += time.Since(start) } + if s.onCommit != nil { + s.onCommit(set) + } } // Clear all internal flags at the end of commit operation. s.accounts = make(map[common.Hash][]byte) diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 60c2d1df82..c149d073ba 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -40,8 +40,10 @@ import ( "testing/quick" "github.com/ava-labs/subnet-evm/core/rawdb" + "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ava-labs/subnet-evm/trie/triestate" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -189,19 +191,32 @@ func (test *stateTest) run() bool { storageList = append(storageList, copy2DSet(states.Storages)) } disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) + tdb = trie.NewDatabase(disk, &trie.Config{PathDB: pathdb.Defaults}) sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) + defer disk.Close() + defer tdb.Close() + + var snaps *snapshot.Tree + if rand.Intn(3) == 0 { + snaps, _ = snapshot.New(snapshot.Config{ + CacheSize: 1, + NoBuild: false, + AsyncBuild: false, + }, disk, tdb, common.Hash{}, types.EmptyRootHash) + } for i, actions := range test.actions { root := types.EmptyRootHash if i != 0 { root = roots[len(roots)-1] } - state, err := New(root, sdb, nil) + state, err := New(root, sdb, snaps) if err != nil { panic(err) } + state.onCommit = onCommit + for i, action := range actions { if i%test.chunk == 0 && i != 0 { if byzantium { diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index b999a6274a..9bf3139e87 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -44,16 +44,24 @@ import ( "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" + "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" ) // Tests that updating a state trie does not leak any database writes prior to // actually committing the state. func TestUpdateLeaks(t *testing.T) { // Create an empty state database - db := rawdb.NewMemoryDatabase() - state, _ := New(types.EmptyRootHash, NewDatabase(db), nil) + var ( + db = rawdb.NewMemoryDatabase() + tdb = trie.NewDatabase(db, nil) + ) + state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil) // Update it with some accounts for i := byte(0); i < 255; i++ { @@ -69,7 +77,7 @@ func TestUpdateLeaks(t *testing.T) { } root := state.IntermediateRoot(false) - if err := state.Database().TrieDB().Commit(root, false); err != nil { + if err := tdb.Commit(root, false); err != nil { t.Errorf("can not commit trie %v to persistent database", root.Hex()) } @@ -87,8 +95,10 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb := rawdb.NewMemoryDatabase() finalDb := rawdb.NewMemoryDatabase() - transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil) - finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil) + transNdb := trie.NewDatabase(transDb, nil) + finalNdb := trie.NewDatabase(finalDb, nil) + transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil) + finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) modify := func(state *StateDB, addr common.Address, i, tweak byte) { state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) @@ -120,7 +130,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit transition state: %v", err) } - if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil { + if err = transNdb.Commit(transRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) } @@ -128,7 +138,7 @@ func TestIntermediateLeaks(t *testing.T) { if err != nil { t.Fatalf("failed to commit final state: %v", err) } - if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil { + if err = finalNdb.Commit(finalRoot, false); err != nil { t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex()) } @@ -450,6 +460,43 @@ func (test *snapshotTest) run() bool { return true } +func forEachStorage(s *StateDB, addr common.Address, cb func(key, value common.Hash) bool) error { + so := s.getStateObject(addr) + if so == nil { + return nil + } + tr, err := so.getTrie() + if err != nil { + return err + } + trieIt, err := tr.NodeIterator(nil) + if err != nil { + return err + } + it := trie.NewIterator(trieIt) + + for it.Next() { + key := common.BytesToHash(s.trie.GetKey(it.Key)) + if value, dirty := so.dirtyStorage[key]; dirty { + if !cb(key, value) { + return nil + } + continue + } + + if len(it.Value) > 0 { + _, content, _, err := rlp.Split(it.Value) + if err != nil { + return err + } + if !cb(key, common.BytesToHash(content)) { + return nil + } + } + } + return nil +} + // checkEqual checks that methods of state and checkstate return the same values. func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { for _, addr := range test.addrs { @@ -471,10 +518,10 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { checkeq("GetCodeSize", state.GetCodeSize(addr), checkstate.GetCodeSize(addr)) // Check storage. if obj := state.getStateObject(addr); obj != nil { - state.ForEachStorage(addr, func(key, value common.Hash) bool { + forEachStorage(state, addr, func(key, value common.Hash) bool { return checkeq("GetState("+key.Hex()+")", checkstate.GetState(addr, key), value) }) - checkstate.ForEachStorage(addr, func(key, value common.Hash) bool { + forEachStorage(checkstate, addr, func(key, value common.Hash) bool { return checkeq("GetState("+key.Hex()+")", checkstate.GetState(addr, key), value) }) } @@ -757,9 +804,28 @@ func TestDeleteCreateRevert(t *testing.T) { // the Commit operation fails with an error // If we are missing trie nodes, we should not continue writing to the trie func TestMissingTrieNodes(t *testing.T) { + testMissingTrieNodes(t, rawdb.HashScheme) + testMissingTrieNodes(t, rawdb.PathScheme) +} + +func testMissingTrieNodes(t *testing.T, scheme string) { // Create an initial state with a few accounts - memDb := rawdb.NewMemoryDatabase() - db := NewDatabase(memDb) + var ( + triedb *trie.Database + memDb = rawdb.NewMemoryDatabase() + ) + if scheme == rawdb.PathScheme { + triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{ + CleanCacheSize: 0, + DirtyCacheSize: 0, + }}) // disable caching + } else { + triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{ + CleanCacheSize: 0, + }}) // disable caching + } + db := NewDatabaseWithNodeDB(memDb, triedb) + var root common.Hash state, _ := New(types.EmptyRootHash, db, nil) addr := common.BytesToAddress([]byte("so")) @@ -772,7 +838,7 @@ func TestMissingTrieNodes(t *testing.T) { root, _ = state.Commit(0, false, false) t.Logf("root: %x", root) // force-flush - state.Database().TrieDB().Cap(0) + triedb.Commit(root, false) } // Create a new state on the old root state, _ = New(root, db, nil) @@ -979,7 +1045,8 @@ func TestFlushOrderDataLoss(t *testing.T) { // Create a state trie with many accounts and slots var ( memdb = rawdb.NewMemoryDatabase() - statedb = NewDatabase(memdb) + triedb = trie.NewDatabase(memdb, nil) + statedb = NewDatabaseWithNodeDB(memdb, triedb) state, _ = New(types.EmptyRootHash, statedb, nil) ) for a := byte(0); a < 10; a++ { @@ -992,11 +1059,11 @@ func TestFlushOrderDataLoss(t *testing.T) { if err != nil { t.Fatalf("failed to commit state trie: %v", err) } - statedb.TrieDB().Reference(root, common.Hash{}) - if err := statedb.TrieDB().Cap(1024); err != nil { + triedb.Reference(root, common.Hash{}) + if err := triedb.Cap(1024); err != nil { t.Fatalf("failed to cap trie dirty cache: %v", err) } - if err := statedb.TrieDB().Commit(root, false); err != nil { + if err := triedb.Commit(root, false); err != nil { t.Fatalf("failed to commit state trie: %v", err) } // Reopen the state trie from flushed disk and verify it @@ -1050,7 +1117,7 @@ func TestStateDBTransientStorage(t *testing.T) { func TestResetObject(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() - tdb = trie.NewDatabase(disk) + tdb = trie.NewDatabase(disk, nil) db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, common.Hash{}, types.EmptyRootHash) state, _ = New(types.EmptyRootHash, db, snaps) @@ -1080,3 +1147,57 @@ func TestResetObject(t *testing.T) { t.Fatalf("Unexpected storage slot value %v", slot) } } + +func TestDeleteStorage(t *testing.T) { + var ( + disk = rawdb.NewMemoryDatabase() + tdb = trie.NewDatabase(disk, nil) + db = NewDatabaseWithNodeDB(disk, tdb) + snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, common.Hash{}, types.EmptyRootHash) + state, _ = New(types.EmptyRootHash, db, snaps) + addr = common.HexToAddress("0x1") + ) + // Initialize account and populate storage + state.SetBalance(addr, big.NewInt(1)) + state.CreateAccount(addr) + for i := 0; i < 1000; i++ { + slot := common.Hash(uint256.NewInt(uint64(i)).Bytes32()) + value := common.Hash(uint256.NewInt(uint64(10 * i)).Bytes32()) + state.SetState(addr, slot, value) + } + root, _ := state.CommitWithSnap(0, true, snaps, common.Hash{}, common.Hash{}, false) + // Init phase done, create two states, one with snap and one without + fastState, _ := New(root, db, snaps) + slowState, _ := New(root, db, nil) + + obj := fastState.GetOrNewStateObject(addr) + storageRoot := obj.data.Root + + _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + if err != nil { + t.Fatal(err) + } + + _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + if err != nil { + t.Fatal(err) + } + check := func(set *trienode.NodeSet) string { + var a []string + set.ForEachWithOrder(func(path string, n *trienode.Node) { + if n.Hash != (common.Hash{}) { + t.Fatal("delete should have empty hashes") + } + if len(n.Blob) != 0 { + t.Fatal("delete should have have empty blobs") + } + a = append(a, fmt.Sprintf("%x", path)) + }) + return strings.Join(a, ",") + } + slowRes := check(slowNodes) + fastRes := check(fastNodes) + if slowRes != fastRes { + t.Fatalf("difference found:\nfast: %v\nslow: %v\n", fastRes, slowRes) + } +} diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 96912962fc..a468024e9a 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -22,6 +22,8 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -36,10 +38,17 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { +func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) { // Create an empty state + config := &trie.Config{Preimages: true} + if scheme == rawdb.PathScheme { + config.PathDB = pathdb.Defaults + } else { + config.HashDB = hashdb.Defaults + } db := rawdb.NewMemoryDatabase() - sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + nodeDb := trie.NewDatabase(db, config) + sdb := NewDatabaseWithNodeDB(db, nodeDb) state, _ := New(types.EmptyRootHash, sdb, nil) // Fill it with some arbitrary data @@ -64,11 +73,10 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { obj.SetState(hash, hash) } } - state.updateStateObject(obj) accounts = append(accounts, acc) } root, _ := state.Commit(0, false, false) // Return the generated state - return db, sdb, root, accounts + return db, sdb, nodeDb, root, accounts } diff --git a/core/state_manager.go b/core/state_manager.go index 77b5c5e739..a22f51041c 100644 --- a/core/state_manager.go +++ b/core/state_manager.go @@ -67,7 +67,7 @@ type TrieWriter interface { type TrieDB interface { Dereference(root common.Hash) error Commit(root common.Hash, report bool) error - Size() (common.StorageSize, common.StorageSize) + Size() (common.StorageSize, common.StorageSize, common.StorageSize) Cap(limit common.StorageSize) error } @@ -126,7 +126,7 @@ type cappedMemoryTrieWriter struct { func (cm *cappedMemoryTrieWriter) InsertTrie(block *types.Block) error { // The use of [Cap] in [InsertTrie] prevents exceeding the configured memory // limit (and OOM) in case there is a large backlog of processing (unaccepted) blocks. - nodes, imgs := cm.TrieDB.Size() + _, nodes, imgs := cm.TrieDB.Size() // all memory is contained within the nodes return for hashdb if nodes <= cm.memoryCap && imgs <= cm.imageCap { return nil } @@ -174,7 +174,7 @@ func (cm *cappedMemoryTrieWriter) AcceptTrie(block *types.Block) error { return nil } targetMemory := cm.targetCommitSize + cm.flushStepSize*common.StorageSize(distanceFromCommit) - nodes, _ := cm.TrieDB.Size() + _, nodes, _ := cm.TrieDB.Size() if nodes <= targetMemory { return nil } diff --git a/core/state_manager_test.go b/core/state_manager_test.go index c47057f8ca..65c56dcf7e 100644 --- a/core/state_manager_test.go +++ b/core/state_manager_test.go @@ -26,8 +26,8 @@ func (t *MockTrieDB) Commit(root common.Hash, report bool) error { t.LastCommit = root return nil } -func (t *MockTrieDB) Size() (common.StorageSize, common.StorageSize) { - return 0, 0 +func (t *MockTrieDB) Size() (common.StorageSize, common.StorageSize, common.StorageSize) { + return 0, 0, 0 } func (t *MockTrieDB) Cap(limit common.StorageSize) error { return nil diff --git a/core/state_processor.go b/core/state_processor.go index 34fe972923..ccf4ad78e9 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -32,6 +32,7 @@ import ( "math/big" "github.com/ava-labs/subnet-evm/consensus" + "github.com/ava-labs/subnet-evm/consensus/misc/eip4844" "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" @@ -93,6 +94,9 @@ func (p *StateProcessor) Process(block *types.Block, parent *types.Header, state vmenv = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) signer = types.MakeSigner(p.config, header.Number, header.Time) ) + if beaconRoot := block.BeaconRoot(); beaconRoot != nil { + ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + } // Iterate over and process the individual transactions for i, tx := range block.Transactions() { msg, err := TransactionToMessage(tx, signer, header.BaseFee) @@ -146,6 +150,11 @@ func applyTransaction(msg *Message, config *params.ChainConfig, gp *GasPool, sta receipt.TxHash = tx.Hash() receipt.GasUsed = result.UsedGas + if tx.Type() == types.BlobTxType { + receipt.BlobGasUsed = uint64(len(tx.BlobHashes()) * params.BlobTxBlobGasPerBlob) + receipt.BlobGasPrice = eip4844.CalcBlobFee(*evm.Context.ExcessBlobGas) + } + // If the transaction created a contract, store the creation address in the receipt. if msg.To == nil { receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) @@ -174,6 +183,26 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, blockContext return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } +// ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root +// contract. This method is exported to be used in tests. +func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *state.StateDB) { + // If EIP-4788 is enabled, we need to invoke the beaconroot storage contract with + // the new root + msg := &Message{ + From: params.SystemAddress, + GasLimit: 30_000_000, + GasPrice: common.Big0, + GasFeeCap: common.Big0, + GasTipCap: common.Big0, + To: ¶ms.BeaconRootsStorageAddress, + Data: beaconRoot[:], + } + vmenv.Reset(NewEVMTxContext(msg), statedb) + statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress) + _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.Big0) + statedb.Finalise(true) +} + // ApplyPrecompileActivations checks if any of the precompiles specified by the chain config are enabled or disabled by the block // transition from [parentTimestamp] to the timestamp set in [blockContext]. If this is the case, it calls [Configure] // to apply the necessary state transitions for the upgrade. diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 4368a51d18..049f5250a0 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -471,6 +471,9 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr used := uint64(nBlobs * params.BlobTxBlobGasPerBlob) header.ExcessBlobGas = &excess header.BlobGasUsed = &used + + beaconRoot := common.HexToHash("0xbeac00") + header.ParentBeaconRoot = &beaconRoot } // Assemble and return the final block for sealing return types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil)) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index e4ab5dbfd7..94263d3b11 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -29,6 +29,7 @@ package blobpool import ( "container/heap" + "errors" "fmt" "math" "math/big" @@ -47,7 +48,6 @@ import ( "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -63,7 +63,7 @@ const ( // maxBlobsPerTransaction is the maximum number of blobs a single transaction // is allowed to contain. Whilst the spec states it's unlimited, the block // data slots are protocol bound, which implicitly also limit this. - maxBlobsPerTransaction = params.BlobTxMaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob + maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob // txAvgSize is an approximate byte size of a transaction metadata to avoid // tiny overflows causing all txs to move a shelf higher, wasting disk space. @@ -93,16 +93,6 @@ const ( limboedTransactionStore = "limbo" ) -// blobTx is a wrapper around types.BlobTx which also contains the literal blob -// data along with all the transaction metadata. -type blobTx struct { - Tx *types.Transaction - - Blobs []kzg4844.Blob - Commits []kzg4844.Commitment - Proofs []kzg4844.Proof -} - // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and // schedule the blob transactions into the following blocks. Only ever add the // bare minimum needed fields to keep the size down (and thus number of entries @@ -375,7 +365,13 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr return err } } + // Initialize the state with head block, or fallback to empty one in + // case the head state is not available(might occur when node is not + // fully synced). state, err := p.chain.StateAt(head.Root) + if err != nil { + state, err = p.chain.StateAt(types.EmptyRootHash) + } if err != nil { return err } @@ -483,22 +479,27 @@ func (p *BlobPool) Close() error { // parseTransaction is a callback method on pool creation that gets called for // each transaction on disk to create the in-memory metadata index. func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { - item := new(blobTx) - if err := rlp.DecodeBytes(blob, item); err != nil { + tx := new(types.Transaction) + if err := rlp.DecodeBytes(blob, tx); err != nil { // This path is impossible unless the disk data representation changes // across restarts. For that ever unprobable case, recover gracefully // by ignoring this data entry. log.Error("Failed to decode blob pool entry", "id", id, "err", err) return err } - meta := newBlobTxMeta(id, size, item.Tx) + if tx.BlobTxSidecar() == nil { + log.Error("Missing sidecar in blob pool entry", "id", id, "hash", tx.Hash()) + return errors.New("missing blob sidecar") + } - sender, err := p.signer.Sender(item.Tx) + meta := newBlobTxMeta(id, size, tx) + + sender, err := p.signer.Sender(tx) if err != nil { // This path is impossible unless the signature validity changes across // restarts. For that ever unprobable case, recover gracefully by ignoring // this data entry. - log.Error("Failed to recover blob tx sender", "id", id, "hash", item.Tx.Hash(), "err", err) + log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err) return err } if _, ok := p.index[sender]; !ok { @@ -746,17 +747,17 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err) return } - item := new(blobTx) - if err = rlp.DecodeBytes(data, item); err != nil { + var tx types.Transaction + if err = rlp.DecodeBytes(data, &tx); err != nil { log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err) return } - block, ok := inclusions[item.Tx.Hash()] + block, ok := inclusions[tx.Hash()] if !ok { log.Warn("Blob transaction swapped out by signer", "from", addr, "nonce", nonce, "id", id) return } - if err := p.limbo.push(item.Tx.Hash(), block, item.Blobs, item.Commits, item.Proofs); err != nil { + if err := p.limbo.push(&tx, block); err != nil { log.Warn("Failed to offload blob tx into limbo", "err", err) return } @@ -788,7 +789,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { for addr, txs := range reinject { // Blindly push all the lost transactions back into the pool for _, tx := range txs { - p.reinject(addr, tx) + p.reinject(addr, tx.Hash()) } // Recheck the account's pooled transactions to drop included and // invalidated one @@ -964,16 +965,19 @@ func (p *BlobPool) reorg(oldHead, newHead *types.Header) (map[common.Address][]* // Note, the method will not initialize the eviction cache values as those will // be done once for all transactions belonging to an account after all individual // transactions are injected back into the pool. -func (p *BlobPool) reinject(addr common.Address, tx *types.Transaction) { +func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) { // Retrieve the associated blob from the limbo. Without the blobs, we cannot // add the transaction back into the pool as it is not mineable. - blobs, commits, proofs, err := p.limbo.pull(tx.Hash()) + tx, err := p.limbo.pull(txhash) if err != nil { log.Error("Blobs unavailable, dropping reorged tx", "err", err) return } - // Serialize the transaction back into the primary datastore - blob, err := rlp.EncodeToBytes(&blobTx{Tx: tx, Blobs: blobs, Commits: commits, Proofs: proofs}) + // TODO: seems like an easy optimization here would be getting the serialized tx + // from limbo instead of re-serializing it here. + + // Serialize the transaction back into the primary datastore. + blob, err := rlp.EncodeToBytes(tx) if err != nil { log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err) return @@ -983,9 +987,9 @@ func (p *BlobPool) reinject(addr common.Address, tx *types.Transaction) { log.Error("Failed to write transaction into storage", "hash", tx.Hash(), "err", err) return } + // Update the indixes and metrics meta := newBlobTxMeta(id, p.store.Size(id), tx) - if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err) @@ -1067,7 +1071,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { // validateTx checks whether a transaction is valid according to the consensus // rules and adheres to some heuristic limits of the local node (price and size). -func (p *BlobPool) validateTx(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error { +func (p *BlobPool) validateTx(tx *types.Transaction) error { // Ensure the transaction adheres to basic pool filters (type, size, tip) and // consensus rules baseOpts := &txpool.ValidationOptions{ @@ -1076,7 +1080,7 @@ func (p *BlobPool) validateTx(tx *types.Transaction, blobs []kzg4844.Blob, commi MaxSize: txMaxSize, MinTip: p.gasTip.ToBig(), } - if err := txpool.ValidateTransaction(tx, blobs, commits, proofs, p.head, p.signer, baseOpts); err != nil { + if err := txpool.ValidateTransaction(tx, p.head, p.signer, baseOpts); err != nil { return err } // Ensure the transaction adheres to the stateful pool filters (nonce, balance) @@ -1166,7 +1170,7 @@ func (p *BlobPool) HasLocal(hash common.Hash) bool { } // Get returns a transaction if it is contained in the pool, or nil otherwise. -func (p *BlobPool) Get(hash common.Hash) *txpool.Transaction { +func (p *BlobPool) Get(hash common.Hash) *types.Transaction { // Track the amount of time waiting to retrieve a fully resolved blob tx from // the pool and the amount of time actually spent on pulling the data from disk. getStart := time.Now() @@ -1188,32 +1192,27 @@ func (p *BlobPool) Get(hash common.Hash) *txpool.Transaction { log.Error("Tracked blob transaction missing from store", "hash", hash, "id", id, "err", err) return nil } - item := new(blobTx) + item := new(types.Transaction) if err = rlp.DecodeBytes(data, item); err != nil { log.Error("Blobs corrupted for traced transaction", "hash", hash, "id", id, "err", err) return nil } - return &txpool.Transaction{ - Tx: item.Tx, - BlobTxBlobs: item.Blobs, - BlobTxCommits: item.Commits, - BlobTxProofs: item.Proofs, - } + return item } // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restictions). -func (p *BlobPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error { +func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error { errs := make([]error, len(txs)) for i, tx := range txs { - errs[i] = p.add(tx.Tx, tx.BlobTxBlobs, tx.BlobTxCommits, tx.BlobTxProofs) + errs[i] = p.add(tx) } return errs } // Add inserts a new blob transaction into the pool if it passes validation (both // consensus validity and pool restictions). -func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) (err error) { +func (p *BlobPool) add(tx *types.Transaction) (err error) { // The blob pool blocks on adding a transaction. This is because blob txs are // only even pulled form the network, so this method will act as the overload // protection for fetches. @@ -1227,7 +1226,7 @@ func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kz }(time.Now()) // Ensure the transaction is valid from all perspectives - if err := p.validateTx(tx, blobs, commits, proofs); err != nil { + if err := p.validateTx(tx); err != nil { log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) return err } @@ -1252,7 +1251,7 @@ func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kz } // Transaction permitted into the pool from a nonce and cost perspective, // insert it into the database and update the indices - blob, err := rlp.EncodeToBytes(&blobTx{Tx: tx, Blobs: blobs, Commits: commits, Proofs: proofs}) + blob, err := rlp.EncodeToBytes(tx) if err != nil { log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err) return err @@ -1488,7 +1487,7 @@ func (p *BlobPool) PendingFrom(addrs []common.Address, enforceTips bool) map[com // IteratePending iterates over [pool.pending] until [f] returns false. // The caller must not modify [tx]. Returns false if iteration was interrupted. -func (pool *BlobPool) IteratePending(f func(tx *txpool.Transaction) bool) bool { +func (pool *BlobPool) IteratePending(f func(tx *types.Transaction) bool) bool { pool.lock.RLock() defer pool.lock.RUnlock() diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 07422590c8..ba6e5eac11 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -218,8 +218,8 @@ func makeAddressReserver() txpool.AddressReserver { // with a valid key, only setting the interesting fields from the perspective of // the blob pool. func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, key *ecdsa.PrivateKey) *types.Transaction { - tx, _ := types.SignNewTx(key, types.LatestSigner(testChainConfig), makeUnsignedTx(nonce, gasTipCap, gasFeeCap, blobFeeCap)) - return tx + blobtx := makeUnsignedTx(nonce, gasTipCap, gasFeeCap, blobFeeCap) + return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx) } // makeUnsignedTx is a utility method to construct a random blob tranasaction @@ -234,6 +234,11 @@ func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap BlobFeeCap: uint256.NewInt(blobFeeCap), BlobHashes: []common.Hash{emptyBlobVHash}, Value: uint256.NewInt(100), + Sidecar: &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + }, } } @@ -366,7 +371,7 @@ func TestOpenDrops(t *testing.T) { R: new(uint256.Int), S: new(uint256.Int), }) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) badsig, _ := store.Put(blob) // Insert a sequence of transactions with a nonce gap in between to verify @@ -379,7 +384,7 @@ func TestOpenDrops(t *testing.T) { ) for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5 tx := makeTx(nonce, 1, 1, 1, gapper) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) if nonce < 2 { @@ -396,7 +401,7 @@ func TestOpenDrops(t *testing.T) { ) for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling tx := makeTx(nonce, 1, 1, 1, dangler) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) dangling[id] = struct{}{} @@ -409,7 +414,7 @@ func TestOpenDrops(t *testing.T) { ) for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled tx := makeTx(nonce, 1, 1, 1, filler) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) filled[id] = struct{}{} @@ -422,7 +427,7 @@ func TestOpenDrops(t *testing.T) { ) for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled tx := makeTx(nonce, 1, 1, 1, overlapper) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) if nonce >= 2 { @@ -444,7 +449,7 @@ func TestOpenDrops(t *testing.T) { } else { tx = makeTx(uint64(i), 1, 1, 1, underpayer) } - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) underpaid[id] = struct{}{} @@ -463,7 +468,7 @@ func TestOpenDrops(t *testing.T) { } else { tx = makeTx(uint64(i), 1, 1, 1, outpricer) } - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) if i < 2 { @@ -485,7 +490,7 @@ func TestOpenDrops(t *testing.T) { } else { tx = makeTx(nonce, 1, 1, 1, exceeder) } - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) exceeded[id] = struct{}{} @@ -503,7 +508,7 @@ func TestOpenDrops(t *testing.T) { } else { tx = makeTx(nonce, 1, 1, 1, overdrafter) } - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) id, _ := store.Put(blob) if nonce < 1 { @@ -519,7 +524,7 @@ func TestOpenDrops(t *testing.T) { overcapped = make(map[uint64]struct{}) ) for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ { - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: makeTx(nonce, 1, 1, 1, overcapper)}) + blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, overcapper)) id, _ := store.Put(blob) if nonce < maxTxsPerAccount { @@ -650,7 +655,7 @@ func TestOpenIndex(t *testing.T) { ) for _, i := range []int{5, 3, 4, 2, 0, 1} { // Randomize the tx insertion order to force sorting on load tx := makeTx(uint64(i), txExecTipCaps[i], txExecFeeCaps[i], txBlobFeeCaps[i], key) - blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx}) + blob, _ := rlp.EncodeToBytes(tx) store.Put(blob) } store.Close() @@ -743,9 +748,9 @@ func TestOpenHeap(t *testing.T) { tx2 = makeTx(0, 1, 800, 70, key2) tx3 = makeTx(0, 1, 1500, 110, key3) - blob1, _ = rlp.EncodeToBytes(&blobTx{Tx: tx1}) - blob2, _ = rlp.EncodeToBytes(&blobTx{Tx: tx2}) - blob3, _ = rlp.EncodeToBytes(&blobTx{Tx: tx3}) + blob1, _ = rlp.EncodeToBytes(tx1) + blob2, _ = rlp.EncodeToBytes(tx2) + blob3, _ = rlp.EncodeToBytes(tx3) heapOrder = []common.Address{addr2, addr1, addr3} heapIndex = map[common.Address]int{addr2: 0, addr1: 1, addr3: 2} @@ -819,9 +824,9 @@ func TestOpenCap(t *testing.T) { tx2 = makeTx(0, 1, 800, 70, key2) tx3 = makeTx(0, 1, 1500, 110, key3) - blob1, _ = rlp.EncodeToBytes(&blobTx{Tx: tx1, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}}) - blob2, _ = rlp.EncodeToBytes(&blobTx{Tx: tx2, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}}) - blob3, _ = rlp.EncodeToBytes(&blobTx{Tx: tx3, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}}) + blob1, _ = rlp.EncodeToBytes(tx1) + blob2, _ = rlp.EncodeToBytes(tx2) + blob3, _ = rlp.EncodeToBytes(tx3) keep = []common.Address{addr1, addr3} drop = []common.Address{addr2} @@ -1235,10 +1240,8 @@ func TestAdd(t *testing.T) { // Sign the seed transactions and store them in the data store for _, tx := range seed.txs { - var ( - signed, _ = types.SignNewTx(keys[acc], types.LatestSigner(testChainConfig), tx) - blob, _ = rlp.EncodeToBytes(&blobTx{Tx: signed, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}}) - ) + signed := types.MustSignNewTx(keys[acc], types.LatestSigner(testChainConfig), tx) + blob, _ := rlp.EncodeToBytes(signed) store.Put(blob) } } @@ -1261,7 +1264,7 @@ func TestAdd(t *testing.T) { // Add each transaction one by one, verifying the pool internals in between for j, add := range tt.adds { signed, _ := types.SignNewTx(keys[add.from], types.LatestSigner(testChainConfig), add.tx) - if err := pool.add(signed, []kzg4844.Blob{emptyBlob}, []kzg4844.Commitment{emptyBlobCommit}, []kzg4844.Proof{emptyBlobProof}); !errors.Is(err, add.err) { + if err := pool.add(signed); !errors.Is(err, add.err) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err) } verifyPoolInternals(t, pool) diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index 13e1f1f6ef..fcdded9abd 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -54,7 +54,7 @@ type evictHeap struct { index map[common.Address]int // Indices into the heap for replacements } -// newPriceHeap creates a new heap of cheapets accounts in the blob pool to evict +// newPriceHeap creates a new heap of cheapest accounts in the blob pool to evict // from in case of over saturation. func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index *map[common.Address][]*blobTxMeta) *evictHeap { heap := &evictHeap{ diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index c8e7eed476..9d97562f51 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -31,7 +31,6 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/holiman/billy" @@ -41,12 +40,9 @@ import ( // to which it belongs as well as the block number in which it was included for // finality eviction. type limboBlob struct { - Owner common.Hash // Owner transaction's hash to support resurrecting reorged txs - Block uint64 // Block in which the blob transaction was included - - Blobs []kzg4844.Blob // The opaque blobs originally part of the transaction - Commits []kzg4844.Commitment // The commitments for the original blobs - Proofs []kzg4844.Proof // The proofs verifying the commitments + TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs + Block uint64 // Block in which the blob transaction was included + Tx *types.Transaction } // limbo is a light, indexed database to temporarily store recently included @@ -108,19 +104,19 @@ func (l *limbo) parseBlob(id uint64, data []byte) error { log.Error("Failed to decode blob limbo entry", "id", id, "err", err) return err } - if _, ok := l.index[item.Owner]; ok { + if _, ok := l.index[item.TxHash]; ok { // This path is impossible, unless due to a programming error a blob gets // inserted into the limbo which was already part of if. Recover gracefully // by ignoring this data entry. - log.Error("Dropping duplicate blob limbo entry", "owner", item.Owner, "id", id) + log.Error("Dropping duplicate blob limbo entry", "owner", item.TxHash, "id", id) return errors.New("duplicate blob") } - l.index[item.Owner] = id + l.index[item.TxHash] = id if _, ok := l.groups[item.Block]; !ok { l.groups[item.Block] = make(map[uint64]common.Hash) } - l.groups[item.Block][id] = item.Owner + l.groups[item.Block][id] = item.TxHash return nil } @@ -149,15 +145,15 @@ func (l *limbo) finalize(final *types.Header) { // push stores a new blob transaction into the limbo, waiting until finality for // it to be automatically evicted. -func (l *limbo) push(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error { +func (l *limbo) push(tx *types.Transaction, block uint64) error { // If the blobs are already tracked by the limbo, consider it a programming // error. There's not much to do against it, but be loud. - if _, ok := l.index[tx]; ok { + if _, ok := l.index[tx.Hash()]; ok { log.Error("Limbo cannot push already tracked blobs", "tx", tx) return errors.New("already tracked blob transaction") } - if err := l.setAndIndex(tx, block, blobs, commits, proofs); err != nil { - log.Error("Failed to set and index liboed blobs", "tx", tx, "err", err) + if err := l.setAndIndex(tx, block); err != nil { + log.Error("Failed to set and index limboed blobs", "tx", tx, "err", err) return err } return nil @@ -166,21 +162,21 @@ func (l *limbo) push(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits // pull retrieves a previously pushed set of blobs back from the limbo, removing // it at the same time. This method should be used when a previously included blob // transaction gets reorged out. -func (l *limbo) pull(tx common.Hash) ([]kzg4844.Blob, []kzg4844.Commitment, []kzg4844.Proof, error) { +func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) { // If the blobs are not tracked by the limbo, there's not much to do. This // can happen for example if a blob transaction is mined without pushing it // into the network first. id, ok := l.index[tx] if !ok { log.Trace("Limbo cannot pull non-tracked blobs", "tx", tx) - return nil, nil, nil, errors.New("unseen blob transaction") + return nil, errors.New("unseen blob transaction") } item, err := l.getAndDrop(id) if err != nil { log.Error("Failed to get and drop limboed blobs", "tx", tx, "id", id, "err", err) - return nil, nil, nil, err + return nil, err } - return item.Blobs, item.Commits, item.Proofs, nil + return item.Tx, nil } // update changes the block number under which a blob transaction is tracked. This @@ -190,33 +186,33 @@ func (l *limbo) pull(tx common.Hash) ([]kzg4844.Blob, []kzg4844.Commitment, []kz // any of it since there's no clear error case. Some errors may be due to coding // issues, others caused by signers mining MEV stuff or swapping transactions. In // all cases, the pool needs to continue operating. -func (l *limbo) update(tx common.Hash, block uint64) { +func (l *limbo) update(txhash common.Hash, block uint64) { // If the blobs are not tracked by the limbo, there's not much to do. This // can happen for example if a blob transaction is mined without pushing it // into the network first. - id, ok := l.index[tx] + id, ok := l.index[txhash] if !ok { - log.Trace("Limbo cannot update non-tracked blobs", "tx", tx) + log.Trace("Limbo cannot update non-tracked blobs", "tx", txhash) return } // If there was no change in the blob's inclusion block, don't mess around // with heavy database operations. if _, ok := l.groups[block][id]; ok { - log.Trace("Blob transaction unchanged in limbo", "tx", tx, "block", block) + log.Trace("Blob transaction unchanged in limbo", "tx", txhash, "block", block) return } - // Retrieve the old blobs from the data store and write tehm back with a new + // Retrieve the old blobs from the data store and write them back with a new // block number. IF anything fails, there's not much to do, go on. item, err := l.getAndDrop(id) if err != nil { - log.Error("Failed to get and drop limboed blobs", "tx", tx, "id", id, "err", err) + log.Error("Failed to get and drop limboed blobs", "tx", txhash, "id", id, "err", err) return } - if err := l.setAndIndex(tx, block, item.Blobs, item.Commits, item.Proofs); err != nil { - log.Error("Failed to set and index limboed blobs", "tx", tx, "err", err) + if err := l.setAndIndex(item.Tx, block); err != nil { + log.Error("Failed to set and index limboed blobs", "tx", txhash, "err", err) return } - log.Trace("Blob transaction updated in limbo", "tx", tx, "old-block", item.Block, "new-block", block) + log.Trace("Blob transaction updated in limbo", "tx", txhash, "old-block", item.Block, "new-block", block) } // getAndDrop retrieves a blob item from the limbo store and deletes it both from @@ -230,7 +226,7 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) { if err = rlp.DecodeBytes(data, item); err != nil { return nil, err } - delete(l.index, item.Owner) + delete(l.index, item.TxHash) delete(l.groups[item.Block], id) if len(l.groups[item.Block]) == 0 { delete(l.groups, item.Block) @@ -243,13 +239,12 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) { // setAndIndex assembles a limbo blob database entry and stores it, also updating // the in-memory indices. -func (l *limbo) setAndIndex(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error { +func (l *limbo) setAndIndex(tx *types.Transaction, block uint64) error { + txhash := tx.Hash() item := &limboBlob{ - Owner: tx, - Block: block, - Blobs: blobs, - Commits: commits, - Proofs: proofs, + TxHash: txhash, + Block: block, + Tx: tx, } data, err := rlp.EncodeToBytes(item) if err != nil { @@ -259,10 +254,10 @@ func (l *limbo) setAndIndex(tx common.Hash, block uint64, blobs []kzg4844.Blob, if err != nil { return err } - l.index[tx] = id + l.index[txhash] = id if _, ok := l.groups[block]; !ok { l.groups[block] = make(map[uint64]common.Hash) } - l.groups[block][id] = tx + l.groups[block][id] = txhash return nil } diff --git a/core/txpool/blobpool/priority.go b/core/txpool/blobpool/priority.go index dd39927361..13c8c82262 100644 --- a/core/txpool/blobpool/priority.go +++ b/core/txpool/blobpool/priority.go @@ -37,7 +37,7 @@ import ( var log2_1_125 = math.Log2(1.125) // evictionPriority calculates the eviction priority based on the algorithm -// described in the BlobPool docs for a both fee components. +// described in the BlobPool docs for both fee components. // // This method takes about 8ns on a very recent laptop CPU, recalculating about // 125 million transaction priority values per second. diff --git a/core/txpool/blobpool/slotter_test.go b/core/txpool/blobpool/slotter_test.go index 9a89f42d7d..e68525a008 100644 --- a/core/txpool/blobpool/slotter_test.go +++ b/core/txpool/blobpool/slotter_test.go @@ -48,14 +48,16 @@ func TestNewSlotter(t *testing.T) { 2*blobSize + txAvgSize, // 2 blob + some expected tx infos (could be fewer blobs and more tx data) 3*blobSize + txAvgSize, // 3 blob + some expected tx infos (could be fewer blobs and more tx data) 4*blobSize + txAvgSize, // 4 blob + some expected tx infos (could be fewer blobs and more tx data) - 5*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 6*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 7*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 8*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 9*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 10*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 11*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size - 12*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size + 5*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 6*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 7*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 8*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 9*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 10*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 11*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 12*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 13*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size + 14*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size } if len(shelves) != len(want) { t.Errorf("shelves count mismatch: have %d, want %d", len(shelves), len(want)) diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 7ecbfef35f..d9aa1527a3 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -62,6 +62,6 @@ var ( ErrOversizedData = errors.New("oversized data") // ErrFutureReplacePending is returned if a future transaction replaces a pending - // transaction. Future transactions should only be able to replace other future transactions. + // one. Future transactions should only be able to replace other future transactions. ErrFutureReplacePending = errors.New("future transaction tries to replace pending") ) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 8c74f1efbb..9e86790c50 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -328,7 +328,20 @@ func (pool *LegacyPool) Init(gasTip *big.Int, head *types.Header, reserve txpool // Set the basic pool parameters pool.gasTip.Store(gasTip) - pool.reset(nil, head) + + // Initialize the state with head block, or fallback to empty one in + // case the head state is not available(might occur when node is not + // fully synced). + statedb, err := pool.chain.StateAt(head.Root) + if err != nil { + statedb, err = pool.chain.StateAt(types.EmptyRootHash) + } + if err != nil { + return err + } + pool.currentHead.Store(head) + pool.currentState = statedb + pool.pendingNonces = newNoncer(statedb) // Start the reorg loop early, so it can handle requests generated during // journal loading. @@ -441,7 +454,7 @@ func (pool *LegacyPool) Close() error { } // Reset implements txpool.SubPool, allowing the legacy pool's internal state to be -// kept in sync with the main transacion pool's internal state. +// kept in sync with the main transaction pool's internal state. func (pool *LegacyPool) Reset(oldHead, newHead *types.Header) { wait := pool.requestReset(oldHead, newHead) <-wait @@ -586,7 +599,7 @@ func (pool *LegacyPool) PendingWithBaseFee(enforceTips bool, baseFee *big.Int) m lazies[i] = &txpool.LazyTransaction{ Pool: pool, Hash: txs[i].Hash(), - Tx: &txpool.Transaction{Tx: txs[i]}, + Tx: txs[i].WithoutBlobTxSidecar(), Time: txs[i].Time(), GasFeeCap: txs[i].GasFeeCap(), GasTipCap: txs[i].GasTipCap(), @@ -627,7 +640,7 @@ func (pool *LegacyPool) PendingFrom(addrs []common.Address, enforceTips bool) ma lazies[i] = &txpool.LazyTransaction{ Pool: pool, Hash: txs[i].Hash(), - Tx: &txpool.Transaction{Tx: txs[i]}, + Tx: txs[i], Time: txs[i].Time(), GasFeeCap: txs[i].GasFeeCap(), GasTipCap: txs[i].GasTipCap(), @@ -641,13 +654,13 @@ func (pool *LegacyPool) PendingFrom(addrs []common.Address, enforceTips bool) ma // IteratePending iterates over [pool.pending] until [f] returns false. // The caller must not modify [tx]. Returns false if iteration was interrupted. -func (pool *LegacyPool) IteratePending(f func(tx *txpool.Transaction) bool) bool { +func (pool *LegacyPool) IteratePending(f func(tx *types.Transaction) bool) bool { pool.mu.RLock() defer pool.mu.RUnlock() for _, list := range pool.pending { for _, tx := range list.txs.items { - if !f(&txpool.Transaction{Tx: tx}) { + if !f(tx) { return false } } @@ -696,7 +709,7 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro if local { opts.MinTip = new(big.Int) } - if err := txpool.ValidateTransaction(tx, nil, nil, nil, pool.currentHead.Load(), pool.signer, opts); err != nil { + if err := txpool.ValidateTransaction(tx, pool.currentHead.Load(), pool.signer, opts); err != nil { return err } return nil @@ -753,7 +766,7 @@ func (pool *LegacyPool) validateTx(tx *types.Transaction, local bool) error { // pending or queued one, it overwrites the previous transaction if its price is higher. // // If a newly added transaction is marked as local, its sending account will be -// be added to the allowlist, preventing any associated transaction from being dropped +// added to the allowlist, preventing any associated transaction from being dropped // out of the pool due to pricing constraints. func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, err error) { // If the transaction is already known, discard it @@ -1016,26 +1029,13 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ return true } -// Add enqueues a batch of transactions into the pool if they are valid. Depending -// on the local flag, full pricing contraints will or will not be applied. -// -// If sync is set, the method will block until all internal maintenance related -// to the add is finished. Only use this during tests for determinism! -func (pool *LegacyPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error { - unwrapped := make([]*types.Transaction, len(txs)) - for i, tx := range txs { - unwrapped[i] = tx.Tx - } - return pool.addTxs(unwrapped, local, sync) -} - // addLocals enqueues a batch of transactions into the pool if they are valid, marking the -// senders as a local ones, ensuring they go around the local pricing constraints. +// senders as local ones, ensuring they go around the local pricing constraints. // // This method is used to add transactions from the RPC API and performs synchronous pool // reorganization and event propagation. func (pool *LegacyPool) addLocals(txs []*types.Transaction) []error { - return pool.addTxs(txs, !pool.config.NoLocals, true) + return pool.Add(txs, !pool.config.NoLocals, true) } // addLocal enqueues a single local transaction into the pool if it is valid. This is @@ -1051,7 +1051,7 @@ func (pool *LegacyPool) addLocal(tx *types.Transaction) error { // This method is used to add transactions from the p2p network and does not wait for pool // reorganization and internal event propagation. func (pool *LegacyPool) addRemotes(txs []*types.Transaction) []error { - return pool.addTxs(txs, false, false) + return pool.Add(txs, false, false) } // addRemote enqueues a single transaction into the pool if it is valid. This is a convenience @@ -1063,16 +1063,20 @@ func (pool *LegacyPool) addRemote(tx *types.Transaction) error { // addRemotesSync is like addRemotes, but waits for pool reorganization. Tests use this method. func (pool *LegacyPool) addRemotesSync(txs []*types.Transaction) []error { - return pool.addTxs(txs, false, true) + return pool.Add(txs, false, true) } // This is like addRemotes with a single transaction, but waits for pool reorganization. Tests use this method. func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { - return pool.addTxs([]*types.Transaction{tx}, false, true)[0] + return pool.Add([]*types.Transaction{tx}, false, true)[0] } -// addTxs attempts to queue a batch of transactions if they are valid. -func (pool *LegacyPool) addTxs(txs []*types.Transaction, local, sync bool) []error { +// Add enqueues a batch of transactions into the pool if they are valid. Depending +// on the local flag, full pricing constraints will or will not be applied. +// +// If sync is set, the method will block until all internal maintenance related +// to the add is finished. Only use this during tests for determinism! +func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( errs = make([]error, len(txs)) @@ -1158,12 +1162,12 @@ func (pool *LegacyPool) Status(hash common.Hash) txpool.TxStatus { } // Get returns a transaction if it is contained in the pool and nil otherwise. -func (pool *LegacyPool) Get(hash common.Hash) *txpool.Transaction { +func (pool *LegacyPool) Get(hash common.Hash) *types.Transaction { tx := pool.get(hash) if tx == nil { return nil } - return &txpool.Transaction{Tx: tx} + return tx } // get returns a transaction if it is contained in the pool and nil otherwise. diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index 92b4e673eb..190629d7df 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -63,9 +63,10 @@ func (h *nonceHeap) Pop() interface{} { // sortedMap is a nonce->transaction hash map with a heap based index to allow // iterating over the contents in a nonce-incrementing way. type sortedMap struct { - items map[uint64]*types.Transaction // Hash map storing the transaction data - index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode) - cache types.Transactions // Cache of the transactions already sorted + items map[uint64]*types.Transaction // Hash map storing the transaction data + index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode) + cache types.Transactions // Cache of the transactions already sorted + cacheMu sync.Mutex // Mutex covering the cache } // newSortedMap creates a new nonce-sorted transaction map. @@ -88,7 +89,9 @@ func (m *sortedMap) Put(tx *types.Transaction) { if m.items[nonce] == nil { heap.Push(m.index, nonce) } + m.cacheMu.Lock() m.items[nonce], m.cache = tx, nil + m.cacheMu.Unlock() } // Forward removes all transactions from the map with a nonce lower than the @@ -104,9 +107,11 @@ func (m *sortedMap) Forward(threshold uint64) types.Transactions { delete(m.items, nonce) } // If we had a cached order, shift the front + m.cacheMu.Lock() if m.cache != nil { m.cache = m.cache[len(removed):] } + m.cacheMu.Unlock() return removed } @@ -130,7 +135,9 @@ func (m *sortedMap) reheap() { *m.index = append(*m.index, nonce) } heap.Init(m.index) + m.cacheMu.Lock() m.cache = nil + m.cacheMu.Unlock() } // filter is identical to Filter, but **does not** regenerate the heap. This method @@ -146,7 +153,9 @@ func (m *sortedMap) filter(filter func(*types.Transaction) bool) types.Transacti } } if len(removed) > 0 { + m.cacheMu.Lock() m.cache = nil + m.cacheMu.Unlock() } return removed } @@ -170,9 +179,11 @@ func (m *sortedMap) Cap(threshold int) types.Transactions { heap.Init(m.index) // If we had a cache, shift the back + m.cacheMu.Lock() if m.cache != nil { m.cache = m.cache[:len(m.cache)-len(drops)] } + m.cacheMu.Unlock() return drops } @@ -192,7 +203,9 @@ func (m *sortedMap) Remove(nonce uint64) bool { } } delete(m.items, nonce) + m.cacheMu.Lock() m.cache = nil + m.cacheMu.Unlock() return true } @@ -216,7 +229,9 @@ func (m *sortedMap) Ready(start uint64) types.Transactions { delete(m.items, next) heap.Pop(m.index) } + m.cacheMu.Lock() m.cache = nil + m.cacheMu.Unlock() return ready } @@ -227,6 +242,8 @@ func (m *sortedMap) Len() int { } func (m *sortedMap) flatten() types.Transactions { + m.cacheMu.Lock() + defer m.cacheMu.Unlock() // If the sorting was not cached yet, create and cache it if m.cache == nil { m.cache = make(types.Transactions, 0, len(m.items)) @@ -242,8 +259,8 @@ func (m *sortedMap) flatten() types.Transactions { // sorted internal representation. The result of the sorting is cached in case // it's requested again before any modifications are made to the contents. func (m *sortedMap) Flatten() types.Transactions { - // Copy the cache to prevent accidental modifications cache := m.flatten() + // Copy the cache to prevent accidental modification txs := make(types.Transactions, len(cache)) copy(txs, cache) return txs diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index fb00fb1abc..94a20a022c 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -33,27 +33,16 @@ import ( "github.com/ava-labs/subnet-evm/core" "github.com/ava-labs/subnet-evm/core/types" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" ) -// Transaction is a helper struct to group together a canonical transaction with -// satellite data items that are needed by the pool but are not part of the chain. -type Transaction struct { - Tx *types.Transaction // Canonical transaction - - BlobTxBlobs []kzg4844.Blob // Blobs needed by the blob pool - BlobTxCommits []kzg4844.Commitment // Commitments needed by the blob pool - BlobTxProofs []kzg4844.Proof // Proofs needed by the blob pool -} - // LazyTransaction contains a small subset of the transaction properties that is // enough for the miner and other APIs to handle large batches of transactions; // and supports pulling up the entire transaction when really needed. type LazyTransaction struct { - Pool SubPool // Transaction subpool to pull the real transaction up - Hash common.Hash // Transaction hash to pull up if needed - Tx *Transaction // Transaction if already resolved + Pool SubPool // Transaction subpool to pull the real transaction up + Hash common.Hash // Transaction hash to pull up if needed + Tx *types.Transaction // Transaction if already resolved Time time.Time // Time when the transaction was first seen GasFeeCap *big.Int // Maximum fee per gas the transaction may consume @@ -62,7 +51,7 @@ type LazyTransaction struct { // Resolve retrieves the full transaction belonging to a lazy handle if it is still // maintained by the transaction pool. -func (ltx *LazyTransaction) Resolve() *Transaction { +func (ltx *LazyTransaction) Resolve() *types.Transaction { if ltx.Tx == nil { ltx.Tx = ltx.Pool.Get(ltx.Hash) } @@ -111,19 +100,19 @@ type SubPool interface { HasLocal(hash common.Hash) bool // Get returns a transaction if it is contained in the pool, or nil otherwise. - Get(hash common.Hash) *Transaction + Get(hash common.Hash) *types.Transaction // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. - Add(txs []*Transaction, local bool, sync bool) []error + Add(txs []*types.Transaction, local bool, sync bool) []error // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. Pending(enforceTips bool) map[common.Address][]*LazyTransaction PendingWithBaseFee(enforceTips bool, baseFee *big.Int) map[common.Address][]*LazyTransaction PendingFrom(addrs []common.Address, enforceTips bool) map[common.Address][]*LazyTransaction - IteratePending(f func(tx *Transaction) bool) bool // Returns false if iteration was interrupted. + IteratePending(f func(tx *types.Transaction) bool) bool // Returns false if iteration was interrupted. // SubscribeTransactions subscribes to new transaction events. SubscribeTransactions(ch chan<- core.NewTxsEvent) event.Subscription diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index ce3f5f7e73..ecdbe02853 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -86,7 +86,7 @@ type TxPool struct { reservations map[common.Address]SubPool // Map with the account to pool reservations reserveLock sync.Mutex // Lock protecting the account reservations - subs event.SubscriptionScope // Subscription scope to unscubscribe all on shutdown + subs event.SubscriptionScope // Subscription scope to unsubscribe all on shutdown quit chan chan error // Quit channel to tear down the head updater gasTip atomic.Pointer[big.Int] // Remember last value set so it can be retrieved @@ -299,7 +299,7 @@ func (p *TxPool) HasLocal(hash common.Hash) bool { } // Get returns a transaction if it is contained in the pool, or nil otherwise. -func (p *TxPool) Get(hash common.Hash) *Transaction { +func (p *TxPool) Get(hash common.Hash) *types.Transaction { for _, subpool := range p.subpools { if tx := subpool.Get(hash); tx != nil { return tx @@ -311,14 +311,14 @@ func (p *TxPool) Get(hash common.Hash) *Transaction { // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. -func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error { +func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { // Split the input transactions between the subpools. It shouldn't really // happen that we receive merged batches, but better graceful than strange // errors. // // We also need to track how the transactions were split across the subpools, // so we can piece back the returned errors into the original order. - txsets := make([][]*Transaction, len(p.subpools)) + txsets := make([][]*types.Transaction, len(p.subpools)) splits := make([]int, len(txs)) for i, tx := range txs { @@ -327,7 +327,7 @@ func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error { // Try to find a subpool that accepts the transaction for j, subpool := range p.subpools { - if subpool.Filter(tx.Tx) { + if subpool.Filter(tx) { txsets[j] = append(txsets[j], tx) splits[i] = j break @@ -355,11 +355,7 @@ func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error { } func (p *TxPool) AddRemotesSync(txs []*types.Transaction) []error { - wrapped := make([]*Transaction, len(txs)) - for i, tx := range txs { - wrapped[i] = &Transaction{Tx: tx} - } - return p.Add(wrapped, false, true) + return p.Add(txs, false, true) } // Pending retrieves all currently processable transactions, grouped by origin @@ -414,7 +410,7 @@ func (p *TxPool) PendingFrom(addrs []common.Address, enforceTips bool) map[commo // IteratePending iterates over [pool.pending] until [f] returns false. // The caller must not modify [tx]. -func (p *TxPool) IteratePending(f func(tx *Transaction) bool) { +func (p *TxPool) IteratePending(f func(tx *types.Transaction) bool) { for _, subpool := range p.subpools { if !subpool.IteratePending(f) { return @@ -520,7 +516,7 @@ func (p *TxPool) Locals() []common.Address { } // Status returns the known status (unknown/pending/queued) of a transaction -// identified by their hashes. +// identified by its hash. func (p *TxPool) Status(hash common.Hash) TxStatus { for _, subpool := range p.subpools { if status := subpool.Status(hash); status != TxStatusUnknown { diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 7b355bcd7c..6e80ac9a91 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -58,7 +58,7 @@ type ValidationOptions struct { // // This check is public to allow different transaction pools to check the basic // rules without duplicating code and running the risk of missed updates. -func ValidateTransaction(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof, head *types.Header, signer types.Signer, opts *ValidationOptions) error { +func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types.Signer, opts *ValidationOptions) error { // Ensure transactions not implemented by the calling pool are rejected if opts.Accept&(1< params.BlobTxMaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { - return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - } - if len(blobs) != len(hashes) { - return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(blobs), len(hashes)) - } - if len(commits) != len(hashes) { - return fmt.Errorf("invalid number of %d blob commitments compared to %d blob hashes", len(commits), len(hashes)) + if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { + return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) } - if len(proofs) != len(hashes) { - return fmt.Errorf("invalid number of %d blob proofs compared to %d blob hashes", len(proofs), len(hashes)) + if err := validateBlobSidecar(hashes, sidecar); err != nil { + return err } - // Blob quantities match up, validate that the provers match with the - // transaction hash before getting to the cryptography - hasher := sha256.New() - for i, want := range hashes { - hasher.Write(commits[i][:]) - hash := hasher.Sum(nil) - hasher.Reset() + } + return nil +} - var vhash common.Hash - vhash[0] = params.BlobTxHashVersion - copy(vhash[1:], hash[1:]) +func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) error { + if len(sidecar.Blobs) != len(hashes) { + return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes)) + } + if len(sidecar.Commitments) != len(hashes) { + return fmt.Errorf("invalid number of %d blob commitments compared to %d blob hashes", len(sidecar.Commitments), len(hashes)) + } + if len(sidecar.Proofs) != len(hashes) { + return fmt.Errorf("invalid number of %d blob proofs compared to %d blob hashes", len(sidecar.Proofs), len(hashes)) + } + // Blob quantities match up, validate that the provers match with the + // transaction hash before getting to the cryptography + hasher := sha256.New() + for i, want := range hashes { + hasher.Write(sidecar.Commitments[i][:]) + hash := hasher.Sum(nil) + hasher.Reset() - if vhash != want { - return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, vhash, want) - } + var vhash common.Hash + vhash[0] = params.BlobTxHashVersion + copy(vhash[1:], hash[1:]) + + if vhash != want { + return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, vhash, want) } - // Blob commitments match with the hashes in the transaction, verify the - // blobs themselves via KZG - for i := range blobs { - if err := kzg4844.VerifyBlobProof(blobs[i], commits[i], proofs[i]); err != nil { - return fmt.Errorf("invalid blob %d: %v", i, err) - } + } + // Blob commitments match with the hashes in the transaction, verify the + // blobs themselves via KZG + for i := range sidecar.Blobs { + if err := kzg4844.VerifyBlobProof(sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { + return fmt.Errorf("invalid blob %d: %v", i, err) } } return nil @@ -189,7 +200,7 @@ type ValidationOptionsWithState struct { // be rejected once the number of remaining slots reaches zero. UsedAndLeftSlots func(addr common.Address) (int, int) - // ExistingExpenditure is a mandatory callback to retrieve the cummulative + // ExistingExpenditure is a mandatory callback to retrieve the cumulative // cost of the already pooled transactions to check for overdrafts. ExistingExpenditure func(addr common.Address) *big.Int @@ -253,7 +264,7 @@ func ValidateTransactionWithState(tx *types.Transaction, signer types.Signer, op return fmt.Errorf("%w: balance %v, queued cost %v, tx cost %v, overshot %v", core.ErrInsufficientFunds, balance, spent, cost, new(big.Int).Sub(need, balance)) } // Transaction takes a new nonce value out of the pool. Ensure it doesn't - // overflow the number of permitted transactions from a single accoun + // overflow the number of permitted transactions from a single account // (i.e. max cancellable via out-of-bound transaction). if used, left := opts.UsedAndLeftSlots(from); left <= 0 { return fmt.Errorf("%w: pooled %d txs", ErrAccountLimitExceeded, used) diff --git a/core/types/block.go b/core/types/block.go index a17d4c7422..81393e52ee 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -99,6 +99,9 @@ type Header struct { // ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. ExcessBlobGas *uint64 `json:"excessBlobGas" rlp:"optional"` + + // ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` } // field type overrides for gencodec @@ -253,6 +256,10 @@ func CopyHeader(h *Header) *Header { cpy.BlobGasUsed = new(uint64) *cpy.BlobGasUsed = *h.BlobGasUsed } + if h.ParentBeaconRoot != nil { + cpy.ParentBeaconRoot = new(common.Hash) + *cpy.ParentBeaconRoot = *h.ParentBeaconRoot + } return &cpy } @@ -331,12 +338,7 @@ func (b *Block) BaseFee() *big.Int { return new(big.Int).Set(b.header.BaseFee) } -func (b *Block) BlockGasCost() *big.Int { - if b.header.BlockGasCost == nil { - return nil - } - return new(big.Int).Set(b.header.BlockGasCost) -} +func (b *Block) BeaconRoot() *common.Hash { return b.header.ParentBeaconRoot } func (b *Block) ExcessBlobGas() *uint64 { var excessBlobGas *uint64 @@ -356,6 +358,13 @@ func (b *Block) BlobGasUsed() *uint64 { return blobGasUsed } +func (b *Block) BlockGasCost() *big.Int { + if b.header.BlockGasCost == nil { + return nil + } + return new(big.Int).Set(b.header.BlockGasCost) +} + // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previously cached value. func (b *Block) Size() uint64 { diff --git a/core/types/gen_account_rlp.go b/core/types/gen_account_rlp.go index 9d07200e33..951632fb58 100644 --- a/core/types/gen_account_rlp.go +++ b/core/types/gen_account_rlp.go @@ -1,8 +1,5 @@ // Code generated by rlpgen. DO NOT EDIT. -//go:build !norlpgen -// +build !norlpgen - package types import ( diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index 224c2c3759..0bc26511f0 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -16,26 +16,27 @@ var _ = (*headerMarshaling)(nil) // MarshalJSON marshals as JSON. func (h Header) MarshalJSON() ([]byte, error) { type Header struct { - ParentHash common.Hash `json:"parentHash" gencodec:"required"` - UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"` - Coinbase common.Address `json:"miner" gencodec:"required"` - Root common.Hash `json:"stateRoot" gencodec:"required"` - TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` - Number *hexutil.Big `json:"number" gencodec:"required"` - GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Time hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Extra hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest common.Hash `json:"mixHash"` - Nonce BlockNonce `json:"nonce"` - BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` - BlockGasCost *hexutil.Big `json:"blockGasCost" rlp:"optional"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` - Hash common.Hash `json:"hash"` + ParentHash common.Hash `json:"parentHash" gencodec:"required"` + UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"` + Coinbase common.Address `json:"miner" gencodec:"required"` + Root common.Hash `json:"stateRoot" gencodec:"required"` + TxHash common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom Bloom `json:"logsBloom" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Number *hexutil.Big `json:"number" gencodec:"required"` + GasLimit hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Time hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Extra hexutil.Bytes `json:"extraData" gencodec:"required"` + MixDigest common.Hash `json:"mixHash"` + Nonce BlockNonce `json:"nonce"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + BlockGasCost *hexutil.Big `json:"blockGasCost" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + Hash common.Hash `json:"hash"` } var enc Header enc.ParentHash = h.ParentHash @@ -57,6 +58,7 @@ func (h Header) MarshalJSON() ([]byte, error) { enc.BlockGasCost = (*hexutil.Big)(h.BlockGasCost) enc.BlobGasUsed = (*hexutil.Uint64)(h.BlobGasUsed) enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas) + enc.ParentBeaconRoot = h.ParentBeaconRoot enc.Hash = h.Hash() return json.Marshal(&enc) } @@ -64,25 +66,26 @@ func (h Header) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (h *Header) UnmarshalJSON(input []byte) error { type Header struct { - ParentHash *common.Hash `json:"parentHash" gencodec:"required"` - UncleHash *common.Hash `json:"sha3Uncles" gencodec:"required"` - Coinbase *common.Address `json:"miner" gencodec:"required"` - Root *common.Hash `json:"stateRoot" gencodec:"required"` - TxHash *common.Hash `json:"transactionsRoot" gencodec:"required"` - ReceiptHash *common.Hash `json:"receiptsRoot" gencodec:"required"` - Bloom *Bloom `json:"logsBloom" gencodec:"required"` - Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` - Number *hexutil.Big `json:"number" gencodec:"required"` - GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` - GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` - Time *hexutil.Uint64 `json:"timestamp" gencodec:"required"` - Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` - MixDigest *common.Hash `json:"mixHash"` - Nonce *BlockNonce `json:"nonce"` - BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` - BlockGasCost *hexutil.Big `json:"blockGasCost" rlp:"optional"` - BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` - ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentHash *common.Hash `json:"parentHash" gencodec:"required"` + UncleHash *common.Hash `json:"sha3Uncles" gencodec:"required"` + Coinbase *common.Address `json:"miner" gencodec:"required"` + Root *common.Hash `json:"stateRoot" gencodec:"required"` + TxHash *common.Hash `json:"transactionsRoot" gencodec:"required"` + ReceiptHash *common.Hash `json:"receiptsRoot" gencodec:"required"` + Bloom *Bloom `json:"logsBloom" gencodec:"required"` + Difficulty *hexutil.Big `json:"difficulty" gencodec:"required"` + Number *hexutil.Big `json:"number" gencodec:"required"` + GasLimit *hexutil.Uint64 `json:"gasLimit" gencodec:"required"` + GasUsed *hexutil.Uint64 `json:"gasUsed" gencodec:"required"` + Time *hexutil.Uint64 `json:"timestamp" gencodec:"required"` + Extra *hexutil.Bytes `json:"extraData" gencodec:"required"` + MixDigest *common.Hash `json:"mixHash"` + Nonce *BlockNonce `json:"nonce"` + BaseFee *hexutil.Big `json:"baseFeePerGas" rlp:"optional"` + BlockGasCost *hexutil.Big `json:"blockGasCost" rlp:"optional"` + BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"` + ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` + ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -158,5 +161,8 @@ func (h *Header) UnmarshalJSON(input []byte) error { if dec.ExcessBlobGas != nil { h.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) } + if dec.ParentBeaconRoot != nil { + h.ParentBeaconRoot = dec.ParentBeaconRoot + } return nil } diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go index 1735881553..6553c079ca 100644 --- a/core/types/gen_header_rlp.go +++ b/core/types/gen_header_rlp.go @@ -1,8 +1,5 @@ // Code generated by rlpgen. DO NOT EDIT. -//go:build !norlpgen -// +build !norlpgen - package types import "github.com/ethereum/go-ethereum/rlp" @@ -44,7 +41,8 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { _tmp2 := obj.BlockGasCost != nil _tmp3 := obj.BlobGasUsed != nil _tmp4 := obj.ExcessBlobGas != nil - if _tmp1 || _tmp2 || _tmp3 || _tmp4 { + _tmp5 := obj.ParentBeaconRoot != nil + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 { if obj.BaseFee == nil { w.Write(rlp.EmptyString) } else { @@ -54,7 +52,7 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBigInt(obj.BaseFee) } } - if _tmp2 || _tmp3 || _tmp4 { + if _tmp2 || _tmp3 || _tmp4 || _tmp5 { if obj.BlockGasCost == nil { w.Write(rlp.EmptyString) } else { @@ -64,20 +62,27 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBigInt(obj.BlockGasCost) } } - if _tmp3 || _tmp4 { + if _tmp3 || _tmp4 || _tmp5 { if obj.BlobGasUsed == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.BlobGasUsed)) } } - if _tmp4 { + if _tmp4 || _tmp5 { if obj.ExcessBlobGas == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.ExcessBlobGas)) } } + if _tmp5 { + if obj.ParentBeaconRoot == nil { + w.Write([]byte{0x80}) + } else { + w.WriteBytes(obj.ParentBeaconRoot[:]) + } + } w.ListEnd(_tmp0) return w.Flush() } diff --git a/core/types/gen_log_json.go b/core/types/gen_log_json.go index 90e1c14d90..3ffa9c2feb 100644 --- a/core/types/gen_log_json.go +++ b/core/types/gen_log_json.go @@ -18,12 +18,12 @@ func (l Log) MarshalJSON() ([]byte, error) { Address common.Address `json:"address" gencodec:"required"` Topics []common.Hash `json:"topics" gencodec:"required"` Data hexutil.Bytes `json:"data" gencodec:"required"` - BlockNumber hexutil.Uint64 `json:"blockNumber"` - TxHash common.Hash `json:"transactionHash" gencodec:"required"` - TxIndex hexutil.Uint `json:"transactionIndex"` - BlockHash common.Hash `json:"blockHash"` - Index hexutil.Uint `json:"logIndex"` - Removed bool `json:"removed"` + BlockNumber hexutil.Uint64 `json:"blockNumber" rlp:"-"` + TxHash common.Hash `json:"transactionHash" gencodec:"required" rlp:"-"` + TxIndex hexutil.Uint `json:"transactionIndex" rlp:"-"` + BlockHash common.Hash `json:"blockHash" rlp:"-"` + Index hexutil.Uint `json:"logIndex" rlp:"-"` + Removed bool `json:"removed" rlp:"-"` } var enc Log enc.Address = l.Address @@ -44,12 +44,12 @@ func (l *Log) UnmarshalJSON(input []byte) error { Address *common.Address `json:"address" gencodec:"required"` Topics []common.Hash `json:"topics" gencodec:"required"` Data *hexutil.Bytes `json:"data" gencodec:"required"` - BlockNumber *hexutil.Uint64 `json:"blockNumber"` - TxHash *common.Hash `json:"transactionHash" gencodec:"required"` - TxIndex *hexutil.Uint `json:"transactionIndex"` - BlockHash *common.Hash `json:"blockHash"` - Index *hexutil.Uint `json:"logIndex"` - Removed *bool `json:"removed"` + BlockNumber *hexutil.Uint64 `json:"blockNumber" rlp:"-"` + TxHash *common.Hash `json:"transactionHash" gencodec:"required" rlp:"-"` + TxIndex *hexutil.Uint `json:"transactionIndex" rlp:"-"` + BlockHash *common.Hash `json:"blockHash" rlp:"-"` + Index *hexutil.Uint `json:"logIndex" rlp:"-"` + Removed *bool `json:"removed" rlp:"-"` } var dec Log if err := json.Unmarshal(input, &dec); err != nil { diff --git a/core/types/gen_log_rlp.go b/core/types/gen_log_rlp.go index 78fa783cee..89f2083d4b 100644 --- a/core/types/gen_log_rlp.go +++ b/core/types/gen_log_rlp.go @@ -1,8 +1,5 @@ // Code generated by rlpgen. DO NOT EDIT. -//go:build !norlpgen -// +build !norlpgen - package types import ( @@ -11,7 +8,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -func (obj *rlpLog) EncodeRLP(_w io.Writer) error { +func (obj *Log) EncodeRLP(_w io.Writer) error { w := rlp.NewEncoderBuffer(_w) _tmp0 := w.List() w.WriteBytes(obj.Address[:]) diff --git a/core/types/hashing.go b/core/types/hashing.go index a82c18a9e4..4f38f42156 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -28,6 +28,8 @@ package types import ( "bytes" + "fmt" + "math" "sync" "github.com/ethereum/go-ethereum/common" @@ -46,6 +48,22 @@ var encodeBufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } +// getPooledBuffer retrieves a buffer from the pool and creates a byte slice of the +// requested size from it. +// +// The caller should return the *bytes.Buffer object back into encodeBufferPool after use! +// The returned byte slice must not be used after returning the buffer. +func getPooledBuffer(size uint64) ([]byte, *bytes.Buffer, error) { + if size > math.MaxInt { + return nil, nil, fmt.Errorf("can't get buffer of size %d", size) + } + buf := encodeBufferPool.Get().(*bytes.Buffer) + buf.Reset() + buf.Grow(int(size)) + b := buf.Bytes()[:int(size)] + return b, buf, nil +} + // rlpHash encodes x and hashes the encoded bytes. func rlpHash(x interface{}) (h common.Hash) { sha := hasherPool.Get().(crypto.KeccakState) diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index cc3fdd2c81..230ac97ebf 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -49,7 +49,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -96,7 +96,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) } }) @@ -117,7 +117,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -145,7 +145,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/core/types/log.go b/core/types/log.go index 926327bb08..fc7e42f295 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -27,13 +27,11 @@ package types import ( - "io" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/rlp" ) +//go:generate go run github.com/ethereum/go-ethereum/rlp/rlpgen -type Log -out gen_log_rlp.go //go:generate go run github.com/fjl/gencodec -type Log -field-override logMarshaling -out gen_log_json.go // Log represents a contract log event. These events are generated by the LOG opcode and @@ -50,19 +48,19 @@ type Log struct { // Derived fields. These fields are filled in by the node // but not secured by consensus. // block in which the transaction was included - BlockNumber uint64 `json:"blockNumber"` + BlockNumber uint64 `json:"blockNumber" rlp:"-"` // hash of the transaction - TxHash common.Hash `json:"transactionHash" gencodec:"required"` + TxHash common.Hash `json:"transactionHash" gencodec:"required" rlp:"-"` // index of the transaction in the block - TxIndex uint `json:"transactionIndex"` + TxIndex uint `json:"transactionIndex" rlp:"-"` // hash of the block in which the transaction was included - BlockHash common.Hash `json:"blockHash"` + BlockHash common.Hash `json:"blockHash" rlp:"-"` // index of the log in the block - Index uint `json:"logIndex"` + Index uint `json:"logIndex" rlp:"-"` // The Removed field is true if this log was reverted due to a chain reorganisation. // You must pay attention to this field if you receive logs through a filter query. - Removed bool `json:"removed"` + Removed bool `json:"removed" rlp:"-"` } type logMarshaling struct { @@ -72,31 +70,6 @@ type logMarshaling struct { Index hexutil.Uint } -//go:generate go run github.com/ethereum/go-ethereum/rlp/rlpgen -type rlpLog -out gen_log_rlp.go - -// rlpLog is used to RLP-encode both the consensus and storage formats. -type rlpLog struct { - Address common.Address - Topics []common.Hash - Data []byte -} - -// EncodeRLP implements rlp.Encoder. -func (l *Log) EncodeRLP(w io.Writer) error { - rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data} - return rlp.Encode(w, &rl) -} - -// DecodeRLP implements rlp.Decoder. -func (l *Log) DecodeRLP(s *rlp.Stream) error { - var dec rlpLog - err := s.Decode(&dec) - if err == nil { - l.Address, l.Topics, l.Data = dec.Address, dec.Topics, dec.Data - } - return err -} - // FlattenLogs converts a nested array of logs to a single array of logs. func FlattenLogs(list [][]*Log) []*Log { var flat []*Log diff --git a/core/types/receipt.go b/core/types/receipt.go index 2518d5e49a..d94726ea6c 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -163,7 +163,7 @@ func (r *Receipt) MarshalBinary() ([]byte, error) { // DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt // from an RLP stream. func (r *Receipt) DecodeRLP(s *rlp.Stream) error { - kind, _, err := s.Kind() + kind, size, err := s.Kind() switch { case err != nil: return err @@ -175,12 +175,18 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error { } r.Type = LegacyTxType return r.setFromRLP(dec) + case kind == rlp.Byte: + return errShortTypedReceipt default: // It's an EIP-2718 typed tx receipt. - b, err := s.Bytes() + b, buf, err := getPooledBuffer(size) if err != nil { return err } + defer encodeBufferPool.Put(buf) + if err := s.ReadBytes(b); err != nil { + return err + } return r.decodeTyped(b) } } @@ -274,7 +280,7 @@ func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error { w.WriteUint64(r.CumulativeGasUsed) logList := w.List() for _, log := range r.Logs { - if err := rlp.Encode(w, log); err != nil { + if err := log.EncodeRLP(w); err != nil { return err } } diff --git a/core/types/transaction.go b/core/types/transaction.go index 0dcbf20c00..3c1b638a44 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -92,9 +92,6 @@ type TxData interface { value() *big.Int nonce() uint64 to() *common.Address - blobGas() uint64 - blobGasFeeCap() *big.Int - blobHashes() []common.Hash rawSignatureValues() (v, r, s *big.Int) setSignatureValues(chainID, v, r, s *big.Int) @@ -106,6 +103,9 @@ type TxData interface { // copy of the computed value, i.e. callers are allowed to mutate the result. // Method implementations can use 'dst' to store the result. effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int + + encode(*bytes.Buffer) error + decode([]byte) error } // EncodeRLP implements rlp.Encoder @@ -126,7 +126,7 @@ func (tx *Transaction) EncodeRLP(w io.Writer) error { // encodeTyped writes the canonical encoding of a typed transaction to w. func (tx *Transaction) encodeTyped(w *bytes.Buffer) error { w.WriteByte(tx.Type()) - return rlp.Encode(w, tx.inner) + return tx.inner.encode(w) } // MarshalBinary returns the canonical encoding of the transaction. @@ -155,15 +155,23 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error { tx.setDecoded(&inner, rlp.ListSize(size)) } return err + case kind == rlp.Byte: + return errShortTypedTx default: // It's an EIP-2718 typed TX envelope. - var b []byte - if b, err = s.Bytes(); err != nil { + // First read the tx payload bytes into a temporary buffer. + b, buf, err := getPooledBuffer(size) + if err != nil { return err } + defer encodeBufferPool.Put(buf) + if err := s.ReadBytes(b); err != nil { + return err + } + // Now decode the inner transaction. inner, err := tx.decodeTyped(b) if err == nil { - tx.setDecoded(inner, uint64(len(b))) + tx.setDecoded(inner, size) } return err } @@ -196,22 +204,19 @@ func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { if len(b) <= 1 { return nil, errShortTypedTx } + var inner TxData switch b[0] { case AccessListTxType: - var inner AccessListTx - err := rlp.DecodeBytes(b[1:], &inner) - return &inner, err + inner = new(AccessListTx) case DynamicFeeTxType: - var inner DynamicFeeTx - err := rlp.DecodeBytes(b[1:], &inner) - return &inner, err + inner = new(DynamicFeeTx) case BlobTxType: - var inner BlobTx - err := rlp.DecodeBytes(b[1:], &inner) - return &inner, err + inner = new(BlobTx) default: return nil, ErrTxTypeNotSupported } + err := inner.decode(b[1:]) + return inner, err } // setDecoded sets the inner transaction and size after decoding. @@ -298,15 +303,6 @@ func (tx *Transaction) GasTipCap() *big.Int { return new(big.Int).Set(tx.inner.g // GasFeeCap returns the fee cap per gas of the transaction. func (tx *Transaction) GasFeeCap() *big.Int { return new(big.Int).Set(tx.inner.gasFeeCap()) } -// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise. -func (tx *Transaction) BlobGas() uint64 { return tx.inner.blobGas() } - -// BlobGasFeeCap returns the blob gas fee cap per blob gas of the transaction for blob transactions, nil otherwise. -func (tx *Transaction) BlobGasFeeCap() *big.Int { return tx.inner.blobGasFeeCap() } - -// BlobHashes returns the hases of the blob commitments for blob transactions, nil otherwise. -func (tx *Transaction) BlobHashes() []common.Hash { return tx.inner.blobHashes() } - // Value returns the ether amount of the transaction. func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } @@ -393,14 +389,66 @@ func (tx *Transaction) EffectiveGasTipIntCmp(other *big.Int, baseFee *big.Int) i return tx.EffectiveGasTipValue(baseFee).Cmp(other) } +// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise. +func (tx *Transaction) BlobGas() uint64 { + if blobtx, ok := tx.inner.(*BlobTx); ok { + return blobtx.blobGas() + } + return 0 +} + +// BlobGasFeeCap returns the blob gas fee cap per blob gas of the transaction for blob transactions, nil otherwise. +func (tx *Transaction) BlobGasFeeCap() *big.Int { + if blobtx, ok := tx.inner.(*BlobTx); ok { + return blobtx.BlobFeeCap.ToBig() + } + return nil +} + +// BlobHashes returns the hases of the blob commitments for blob transactions, nil otherwise. +func (tx *Transaction) BlobHashes() []common.Hash { + if blobtx, ok := tx.inner.(*BlobTx); ok { + return blobtx.BlobHashes + } + return nil +} + +// BlobTxSidecar returns the sidecar of a blob transaction, nil otherwise. +func (tx *Transaction) BlobTxSidecar() *BlobTxSidecar { + if blobtx, ok := tx.inner.(*BlobTx); ok { + return blobtx.Sidecar + } + return nil +} + // BlobGasFeeCapCmp compares the blob fee cap of two transactions. func (tx *Transaction) BlobGasFeeCapCmp(other *Transaction) int { - return tx.inner.blobGasFeeCap().Cmp(other.inner.blobGasFeeCap()) + return tx.BlobGasFeeCap().Cmp(other.BlobGasFeeCap()) } // BlobGasFeeCapIntCmp compares the blob fee cap of the transaction against the given blob fee cap. func (tx *Transaction) BlobGasFeeCapIntCmp(other *big.Int) int { - return tx.inner.blobGasFeeCap().Cmp(other) + return tx.BlobGasFeeCap().Cmp(other) +} + +// WithoutBlobTxSidecar returns a copy of tx with the blob sidecar removed. +func (tx *Transaction) WithoutBlobTxSidecar() *Transaction { + blobtx, ok := tx.inner.(*BlobTx) + if !ok { + return tx + } + cpy := &Transaction{ + inner: blobtx.withoutSidecar(), + time: tx.time, + } + // Note: tx.size cache not carried over because the sidecar is included in size! + if h := tx.hash.Load(); h != nil { + cpy.hash.Store(h) + } + if f := tx.from.Load(); f != nil { + cpy.from.Store(f) + } + return cpy } // SetTime sets the decoding time of a transaction. This is used by tests to set @@ -438,13 +486,24 @@ func (tx *Transaction) Size() uint64 { if size := tx.size.Load(); size != nil { return size.(uint64) } + + // Cache miss, encode and cache. + // Note we rely on the assumption that all tx.inner values are RLP-encoded! c := writeCounter(0) rlp.Encode(&c, &tx.inner) - size := uint64(c) + + // For blob transactions, add the size of the blob content and the outer list of the + // tx + sidecar encoding. + if sc := tx.BlobTxSidecar(); sc != nil { + size += rlp.ListSize(sc.encodedSize()) + } + + // For typed transactions, the encoding also includes the leading type byte. if tx.Type() != LegacyTxType { - size += 1 // type byte + size += 1 } + tx.size.Store(size) return size } diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 2437a5b2f3..104ff33e0f 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -383,20 +383,20 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { itx.BlobHashes = dec.BlobVersionedHashes // signature R - var ok bool + var overflow bool if dec.R == nil { return errors.New("missing required field 'r' in transaction") } - itx.R, ok = uint256.FromBig((*big.Int)(dec.R)) - if !ok { + itx.R, overflow = uint256.FromBig((*big.Int)(dec.R)) + if overflow { return errors.New("'r' value overflows uint256") } // signature S if dec.S == nil { return errors.New("missing required field 's' in transaction") } - itx.S, ok = uint256.FromBig((*big.Int)(dec.S)) - if !ok { + itx.S, overflow = uint256.FromBig((*big.Int)(dec.S)) + if overflow { return errors.New("'s' value overflows uint256") } // signature V @@ -404,8 +404,8 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { if err != nil { return err } - itx.V, ok = uint256.FromBig(vbig) - if !ok { + itx.V, overflow = uint256.FromBig(vbig) + if overflow { return errors.New("'v' value overflows uint256") } if itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 25d727a186..327a5be167 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -334,11 +334,7 @@ func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) { V, R, S := tx.RawSignatureValues() switch tx.Type() { case LegacyTxType: - if !tx.Protected() { - return HomesteadSigner{}.Sender(tx) - } - V = new(big.Int).Sub(V, s.chainIdMul) - V.Sub(V, big8) + return s.EIP155Signer.Sender(tx) case AccessListTxType: // AL txs are defined to use 0 and 1 as their recovery // id, add 27 to become equivalent to unprotected Homestead signatures. @@ -375,15 +371,7 @@ func (s eip2930Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *bi func (s eip2930Signer) Hash(tx *Transaction) common.Hash { switch tx.Type() { case LegacyTxType: - return rlpHash([]interface{}{ - tx.Nonce(), - tx.GasPrice(), - tx.Gas(), - tx.To(), - tx.Value(), - tx.Data(), - s.chainId, uint(0), uint(0), - }) + return s.EIP155Signer.Hash(tx) case AccessListTxType: return prefixedRlpHash( tx.Type(), diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 1e583b90f4..ff4360009e 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -224,7 +224,7 @@ func TestEIP2718TransactionEncode(t *testing.T) { func decodeTx(data []byte) (*Transaction, error) { var tx Transaction - t, err := &tx, rlp.Decode(bytes.NewReader(data), &tx) + t, err := &tx, rlp.DecodeBytes(data, &tx) return t, err } diff --git a/core/types/tx_access_list.go b/core/types/tx_access_list.go index e233410d26..c12232d4eb 100644 --- a/core/types/tx_access_list.go +++ b/core/types/tx_access_list.go @@ -27,9 +27,11 @@ package types import ( + "bytes" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" ) //go:generate go run github.com/fjl/gencodec -type AccessTuple -out gen_access_tuple.go @@ -104,20 +106,17 @@ func (tx *AccessListTx) copy() TxData { } // accessors for innerTx. -func (tx *AccessListTx) txType() byte { return AccessListTxType } -func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } -func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } -func (tx *AccessListTx) data() []byte { return tx.Data } -func (tx *AccessListTx) gas() uint64 { return tx.Gas } -func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) value() *big.Int { return tx.Value } -func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } -func (tx *AccessListTx) to() *common.Address { return tx.To } -func (tx *AccessListTx) blobGas() uint64 { return 0 } -func (tx *AccessListTx) blobGasFeeCap() *big.Int { return nil } -func (tx *AccessListTx) blobHashes() []common.Hash { return nil } +func (tx *AccessListTx) txType() byte { return AccessListTxType } +func (tx *AccessListTx) chainID() *big.Int { return tx.ChainID } +func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } +func (tx *AccessListTx) data() []byte { return tx.Data } +func (tx *AccessListTx) gas() uint64 { return tx.Gas } +func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) value() *big.Int { return tx.Value } +func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } +func (tx *AccessListTx) to() *common.Address { return tx.To } func (tx *AccessListTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { return dst.Set(tx.GasPrice) @@ -130,3 +129,11 @@ func (tx *AccessListTx) rawSignatureValues() (v, r, s *big.Int) { func (tx *AccessListTx) setSignatureValues(chainID, v, r, s *big.Int) { tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s } + +func (tx *AccessListTx) encode(b *bytes.Buffer) error { + return rlp.Encode(b, tx) +} + +func (tx *AccessListTx) decode(input []byte) error { + return rlp.DecodeBytes(input, tx) +} diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index f97b1bf3c5..d91845ff57 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -17,10 +17,14 @@ package types import ( + "bytes" + "crypto/sha256" "math/big" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/rlp" "github.com/holiman/uint256" ) @@ -38,12 +42,56 @@ type BlobTx struct { BlobFeeCap *uint256.Int // a.k.a. maxFeePerBlobGas BlobHashes []common.Hash + // A blob transaction can optionally contain blobs. This field must be set when BlobTx + // is used to create a transaction for sigining. + Sidecar *BlobTxSidecar `rlp:"-"` + // Signature values V *uint256.Int `json:"v" gencodec:"required"` R *uint256.Int `json:"r" gencodec:"required"` S *uint256.Int `json:"s" gencodec:"required"` } +// BlobTxSidecar contains the blobs of a blob transaction. +type BlobTxSidecar struct { + Blobs []kzg4844.Blob // Blobs needed by the blob pool + Commitments []kzg4844.Commitment // Commitments needed by the blob pool + Proofs []kzg4844.Proof // Proofs needed by the blob pool +} + +// BlobHashes computes the blob hashes of the given blobs. +func (sc *BlobTxSidecar) BlobHashes() []common.Hash { + h := make([]common.Hash, len(sc.Commitments)) + for i := range sc.Blobs { + h[i] = blobHash(&sc.Commitments[i]) + } + return h +} + +// encodedSize computes the RLP size of the sidecar elements. This does NOT return the +// encoded size of the BlobTxSidecar, it's just a helper for tx.Size(). +func (sc *BlobTxSidecar) encodedSize() uint64 { + var blobs, commitments, proofs uint64 + for i := range sc.Blobs { + blobs += rlp.BytesSize(sc.Blobs[i][:]) + } + for i := range sc.Commitments { + commitments += rlp.BytesSize(sc.Commitments[i][:]) + } + for i := range sc.Proofs { + proofs += rlp.BytesSize(sc.Proofs[i][:]) + } + return rlp.ListSize(blobs) + rlp.ListSize(commitments) + rlp.ListSize(proofs) +} + +// blobTxWithBlobs is used for encoding of transactions when blobs are present. +type blobTxWithBlobs struct { + BlobTx *BlobTx + Blobs []kzg4844.Blob + Commitments []kzg4844.Commitment + Proofs []kzg4844.Proof +} + // copy creates a deep copy of the transaction data and initializes all fields. func (tx *BlobTx) copy() TxData { cpy := &BlobTx{ @@ -90,24 +138,29 @@ func (tx *BlobTx) copy() TxData { if tx.S != nil { cpy.S.Set(tx.S) } + if tx.Sidecar != nil { + cpy.Sidecar = &BlobTxSidecar{ + Blobs: append([]kzg4844.Blob(nil), tx.Sidecar.Blobs...), + Commitments: append([]kzg4844.Commitment(nil), tx.Sidecar.Commitments...), + Proofs: append([]kzg4844.Proof(nil), tx.Sidecar.Proofs...), + } + } return cpy } // accessors for innerTx. -func (tx *BlobTx) txType() byte { return BlobTxType } -func (tx *BlobTx) chainID() *big.Int { return tx.ChainID.ToBig() } -func (tx *BlobTx) accessList() AccessList { return tx.AccessList } -func (tx *BlobTx) data() []byte { return tx.Data } -func (tx *BlobTx) gas() uint64 { return tx.Gas } -func (tx *BlobTx) gasFeeCap() *big.Int { return tx.GasFeeCap.ToBig() } -func (tx *BlobTx) gasTipCap() *big.Int { return tx.GasTipCap.ToBig() } -func (tx *BlobTx) gasPrice() *big.Int { return tx.GasFeeCap.ToBig() } -func (tx *BlobTx) value() *big.Int { return tx.Value.ToBig() } -func (tx *BlobTx) nonce() uint64 { return tx.Nonce } -func (tx *BlobTx) to() *common.Address { tmp := tx.To; return &tmp } -func (tx *BlobTx) blobGas() uint64 { return params.BlobTxBlobGasPerBlob * uint64(len(tx.BlobHashes)) } -func (tx *BlobTx) blobGasFeeCap() *big.Int { return tx.BlobFeeCap.ToBig() } -func (tx *BlobTx) blobHashes() []common.Hash { return tx.BlobHashes } +func (tx *BlobTx) txType() byte { return BlobTxType } +func (tx *BlobTx) chainID() *big.Int { return tx.ChainID.ToBig() } +func (tx *BlobTx) accessList() AccessList { return tx.AccessList } +func (tx *BlobTx) data() []byte { return tx.Data } +func (tx *BlobTx) gas() uint64 { return tx.Gas } +func (tx *BlobTx) gasFeeCap() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *BlobTx) gasTipCap() *big.Int { return tx.GasTipCap.ToBig() } +func (tx *BlobTx) gasPrice() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *BlobTx) value() *big.Int { return tx.Value.ToBig() } +func (tx *BlobTx) nonce() uint64 { return tx.Nonce } +func (tx *BlobTx) to() *common.Address { tmp := tx.To; return &tmp } +func (tx *BlobTx) blobGas() uint64 { return params.BlobTxBlobGasPerBlob * uint64(len(tx.BlobHashes)) } func (tx *BlobTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { if baseFee == nil { @@ -130,3 +183,64 @@ func (tx *BlobTx) setSignatureValues(chainID, v, r, s *big.Int) { tx.R.SetFromBig(r) tx.S.SetFromBig(s) } + +func (tx *BlobTx) withoutSidecar() *BlobTx { + cpy := *tx + cpy.Sidecar = nil + return &cpy +} + +func (tx *BlobTx) encode(b *bytes.Buffer) error { + if tx.Sidecar == nil { + return rlp.Encode(b, tx) + } + inner := &blobTxWithBlobs{ + BlobTx: tx, + Blobs: tx.Sidecar.Blobs, + Commitments: tx.Sidecar.Commitments, + Proofs: tx.Sidecar.Proofs, + } + return rlp.Encode(b, inner) +} + +func (tx *BlobTx) decode(input []byte) error { + // Here we need to support two formats: the network protocol encoding of the tx (with + // blobs) or the canonical encoding without blobs. + // + // The two encodings can be distinguished by checking whether the first element of the + // input list is itself a list. + + outerList, _, err := rlp.SplitList(input) + if err != nil { + return err + } + firstElemKind, _, _, err := rlp.Split(outerList) + if err != nil { + return err + } + + if firstElemKind != rlp.List { + return rlp.DecodeBytes(input, tx) + } + // It's a tx with blobs. + var inner blobTxWithBlobs + if err := rlp.DecodeBytes(input, &inner); err != nil { + return err + } + *tx = *inner.BlobTx + tx.Sidecar = &BlobTxSidecar{ + Blobs: inner.Blobs, + Commitments: inner.Commitments, + Proofs: inner.Proofs, + } + return nil +} + +func blobHash(commit *kzg4844.Commitment) common.Hash { + hasher := sha256.New() + hasher.Write(commit[:]) + var vhash common.Hash + hasher.Sum(vhash[:0]) + vhash[0] = params.BlobTxHashVersion + return vhash +} diff --git a/core/types/tx_blob_test.go b/core/types/tx_blob_test.go new file mode 100644 index 0000000000..44ac48cc6f --- /dev/null +++ b/core/types/tx_blob_test.go @@ -0,0 +1,90 @@ +package types + +import ( + "crypto/ecdsa" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/holiman/uint256" +) + +// This test verifies that tx.Hash() is not affected by presence of a BlobTxSidecar. +func TestBlobTxHashing(t *testing.T) { + key, _ := crypto.GenerateKey() + withBlobs := createEmptyBlobTx(key, true) + withBlobsStripped := withBlobs.WithoutBlobTxSidecar() + withoutBlobs := createEmptyBlobTx(key, false) + + hash := withBlobs.Hash() + t.Log("tx hash:", hash) + + if h := withBlobsStripped.Hash(); h != hash { + t.Fatal("wrong tx hash after WithoutBlobTxSidecar:", h) + } + if h := withoutBlobs.Hash(); h != hash { + t.Fatal("wrong tx hash on tx created without sidecar:", h) + } +} + +// This test verifies that tx.Size() takes BlobTxSidecar into account. +func TestBlobTxSize(t *testing.T) { + key, _ := crypto.GenerateKey() + withBlobs := createEmptyBlobTx(key, true) + withBlobsStripped := withBlobs.WithoutBlobTxSidecar() + withoutBlobs := createEmptyBlobTx(key, false) + + withBlobsEnc, _ := withBlobs.MarshalBinary() + withoutBlobsEnc, _ := withoutBlobs.MarshalBinary() + + size := withBlobs.Size() + t.Log("size with blobs:", size) + + sizeNoBlobs := withoutBlobs.Size() + t.Log("size without blobs:", sizeNoBlobs) + + if size != uint64(len(withBlobsEnc)) { + t.Error("wrong size with blobs:", size, "encoded length:", len(withBlobsEnc)) + } + if sizeNoBlobs != uint64(len(withoutBlobsEnc)) { + t.Error("wrong size without blobs:", sizeNoBlobs, "encoded length:", len(withoutBlobsEnc)) + } + if sizeNoBlobs >= size { + t.Error("size without blobs >= size with blobs") + } + if sz := withBlobsStripped.Size(); sz != sizeNoBlobs { + t.Fatal("wrong size on tx after WithoutBlobTxSidecar:", sz) + } +} + +var ( + emptyBlob = kzg4844.Blob{} + emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) + emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) +) + +func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction { + sidecar := &BlobTxSidecar{ + Blobs: []kzg4844.Blob{emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + } + blobtx := &BlobTx{ + ChainID: uint256.NewInt(1), + Nonce: 5, + GasTipCap: uint256.NewInt(22), + GasFeeCap: uint256.NewInt(5), + Gas: 25000, + To: common.Address{0x03, 0x04, 0x05}, + Value: uint256.NewInt(99), + Data: make([]byte, 50), + BlobFeeCap: uint256.NewInt(15), + BlobHashes: sidecar.BlobHashes(), + } + if withSidecar { + blobtx.Sidecar = sidecar + } + signer := NewCancunSigner(blobtx.ChainID.ToBig()) + return MustSignNewTx(key, signer, blobtx) +} diff --git a/core/types/tx_dynamic_fee.go b/core/types/tx_dynamic_fee.go index bf6dbb748b..d40f1fbd03 100644 --- a/core/types/tx_dynamic_fee.go +++ b/core/types/tx_dynamic_fee.go @@ -27,9 +27,11 @@ package types import ( + "bytes" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" ) // DynamicFeeTx represents an EIP-1559 transaction. @@ -93,20 +95,17 @@ func (tx *DynamicFeeTx) copy() TxData { } // accessors for innerTx. -func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } -func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } -func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } -func (tx *DynamicFeeTx) data() []byte { return tx.Data } -func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } -func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } -func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } -func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } -func (tx *DynamicFeeTx) to() *common.Address { return tx.To } -func (tx *DynamicFeeTx) blobGas() uint64 { return 0 } -func (tx *DynamicFeeTx) blobGasFeeCap() *big.Int { return nil } -func (tx *DynamicFeeTx) blobHashes() []common.Hash { return nil } +func (tx *DynamicFeeTx) txType() byte { return DynamicFeeTxType } +func (tx *DynamicFeeTx) chainID() *big.Int { return tx.ChainID } +func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } +func (tx *DynamicFeeTx) data() []byte { return tx.Data } +func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } +func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } +func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } +func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } +func (tx *DynamicFeeTx) to() *common.Address { return tx.To } func (tx *DynamicFeeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { if baseFee == nil { @@ -126,3 +125,11 @@ func (tx *DynamicFeeTx) rawSignatureValues() (v, r, s *big.Int) { func (tx *DynamicFeeTx) setSignatureValues(chainID, v, r, s *big.Int) { tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s } + +func (tx *DynamicFeeTx) encode(b *bytes.Buffer) error { + return rlp.Encode(b, tx) +} + +func (tx *DynamicFeeTx) decode(input []byte) error { + return rlp.DecodeBytes(input, tx) +} diff --git a/core/types/tx_legacy.go b/core/types/tx_legacy.go index dc05354d6d..b1f12567db 100644 --- a/core/types/tx_legacy.go +++ b/core/types/tx_legacy.go @@ -27,6 +27,7 @@ package types import ( + "bytes" "math/big" "github.com/ethereum/go-ethereum/common" @@ -101,20 +102,17 @@ func (tx *LegacyTx) copy() TxData { } // accessors for innerTx. -func (tx *LegacyTx) txType() byte { return LegacyTxType } -func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } -func (tx *LegacyTx) accessList() AccessList { return nil } -func (tx *LegacyTx) data() []byte { return tx.Data } -func (tx *LegacyTx) gas() uint64 { return tx.Gas } -func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) value() *big.Int { return tx.Value } -func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } -func (tx *LegacyTx) to() *common.Address { return tx.To } -func (tx *LegacyTx) blobGas() uint64 { return 0 } -func (tx *LegacyTx) blobGasFeeCap() *big.Int { return nil } -func (tx *LegacyTx) blobHashes() []common.Hash { return nil } +func (tx *LegacyTx) txType() byte { return LegacyTxType } +func (tx *LegacyTx) chainID() *big.Int { return deriveChainId(tx.V) } +func (tx *LegacyTx) accessList() AccessList { return nil } +func (tx *LegacyTx) data() []byte { return tx.Data } +func (tx *LegacyTx) gas() uint64 { return tx.Gas } +func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) value() *big.Int { return tx.Value } +func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } +func (tx *LegacyTx) to() *common.Address { return tx.To } func (tx *LegacyTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { return dst.Set(tx.GasPrice) @@ -127,3 +125,11 @@ func (tx *LegacyTx) rawSignatureValues() (v, r, s *big.Int) { func (tx *LegacyTx) setSignatureValues(chainID, v, r, s *big.Int) { tx.V, tx.R, tx.S = v, r, s } + +func (tx *LegacyTx) encode(*bytes.Buffer) error { + panic("encode called on LegacyTx") +} + +func (tx *LegacyTx) decode([]byte) error { + panic("decode called on LegacyTx)") +} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index d36de0f053..65f659aa32 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -261,6 +261,7 @@ func opKeccak256(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ( size.SetBytes(interpreter.hasherBuf[:]) return nil, nil } + func opAddress(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Address().Bytes())) return nil, nil @@ -277,6 +278,7 @@ func opOrigin(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b scope.Stack.push(new(uint256.Int).SetBytes(interpreter.evm.Origin.Bytes())) return nil, nil } + func opCaller(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Caller().Bytes())) return nil, nil diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 226e1332e6..21c41de359 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -55,6 +55,7 @@ type Config struct { EVMConfig vm.Config BaseFee *big.Int BlobHashes []common.Hash + Random *common.Hash State *state.StateDB GetHashFn func(n uint64) common.Hash diff --git a/eth/api_admin.go b/eth/api_admin.go index e1fe683c01..4a5237b3a9 100644 --- a/eth/api_admin.go +++ b/eth/api_admin.go @@ -128,6 +128,10 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) { } else if err != nil { return false, fmt.Errorf("block %d: failed to parse: %v", index, err) } + // ignore the genesis block when importing blocks + if block.NumberU64() == 0 { + continue + } blocks = append(blocks, block) index++ } diff --git a/eth/api_backend.go b/eth/api_backend.go index edf78810fd..376c1e7ce0 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -40,7 +40,6 @@ import ( "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" - "github.com/ava-labs/subnet-evm/core/txpool" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/gasprice" @@ -245,7 +244,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.B return nil, nil, errors.New("header not found") } stateDb, err := b.eth.BlockChain().StateAt(header.Root) - return stateDb, header, err + if err != nil { + return nil, nil, err + } + return stateDb, header, nil } func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { @@ -264,7 +266,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN return nil, nil, errors.New("header for hash not found") } stateDb, err := b.eth.BlockChain().StateAt(header.Root) - return stateDb, header, err + if err != nil { + return nil, nil, err + } + return stateDb, header, nil } return nil, nil, errors.New("invalid arguments; neither block nor hash specified") } @@ -337,7 +342,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) if err := ctx.Err(); err != nil { return err } - if err := b.eth.txPool.Add([]*txpool.Transaction{{Tx: signedTx}}, true, false)[0]; err != nil { + if err := b.eth.txPool.Add([]*types.Transaction{signedTx}, true, false)[0]; err != nil { return err } @@ -353,7 +358,7 @@ func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { for _, batch := range pending { for _, lazy := range batch { if tx := lazy.Resolve(); tx != nil { - txs = append(txs, tx.Tx) + txs = append(txs, tx) } } } @@ -361,10 +366,7 @@ func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { } func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { - if tx := b.eth.txPool.Get(hash); tx != nil { - return tx.Tx - } - return nil + return b.eth.txPool.Get(hash) } func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { @@ -494,7 +496,7 @@ func (b *EthAPIBackend) GetMaxBlocksPerRequest() int64 { } func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { - return b.eth.StateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) + return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk) } func (b *EthAPIBackend) StateAtNextBlock(ctx context.Context, parent, nextBlock *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { diff --git a/eth/api_debug.go b/eth/api_debug.go index 50d7b140ec..5bfd982809 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -40,6 +40,7 @@ import ( "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) @@ -170,11 +171,15 @@ type storageEntry struct { } // StorageRangeAt returns the storage at the given block height and transaction index. -func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { - // Retrieve the block - block := api.eth.blockchain.GetBlockByHash(blockHash) +func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { + var block *types.Block + + block, err := api.eth.APIBackend.BlockByNumberOrHash(ctx, blockNrOrHash) + if err != nil { + return StorageRangeResult{}, err + } if block == nil { - return StorageRangeResult{}, fmt.Errorf("block %#x not found", blockHash) + return StorageRangeResult{}, fmt.Errorf("block %v not found", blockNrOrHash) } _, _, statedb, release, err := api.eth.stateAtTransaction(ctx, block, txIndex, 0) if err != nil { @@ -182,18 +187,20 @@ func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, } defer release() - st, err := statedb.StorageTrie(contractAddress) + return storageRangeAt(statedb, block.Root(), contractAddress, keyStart, maxResult) +} + +func storageRangeAt(statedb *state.StateDB, root common.Hash, address common.Address, start []byte, maxResult int) (StorageRangeResult, error) { + storageRoot := statedb.GetStorageRoot(address) + if storageRoot == types.EmptyRootHash || storageRoot == (common.Hash{}) { + return StorageRangeResult{}, nil // empty storage + } + id := trie.StorageTrieID(root, crypto.Keccak256Hash(address.Bytes()), storageRoot) + tr, err := trie.NewStateTrie(id, statedb.Database().TrieDB()) if err != nil { return StorageRangeResult{}, err } - if st == nil { - return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress) - } - return storageRangeAt(st, keyStart, maxResult) -} - -func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeResult, error) { - trieIt, err := st.NodeIterator(start) + trieIt, err := tr.NodeIterator(start) if err != nil { return StorageRangeResult{}, err } @@ -205,7 +212,7 @@ func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeRes return StorageRangeResult{}, err } e := storageEntry{Value: common.BytesToHash(content)} - if preimage := st.GetKey(it.Key); preimage != nil { + if preimage := tr.GetKey(it.Key); preimage != nil { preimage := common.BytesToHash(preimage) e.Key = &preimage } @@ -278,7 +285,7 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } - triedb := api.eth.BlockChain().StateCache().TrieDB() + triedb := api.eth.BlockChain().TrieDB() oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb) if err != nil { @@ -316,6 +323,9 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c // The (from, to) parameters are the sequence of blocks to search, which can go // either forwards or backwards func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) { + if api.eth.blockchain.TrieDB().Scheme() == rawdb.PathScheme { + return 0, errors.New("state history is not yet available in path-based scheme") + } var resolveNum = func(num rpc.BlockNumber) (uint64, error) { // We don't have state for pending (-2), so treat it as latest if num.Int64() < 0 { diff --git a/eth/backend.go b/eth/backend.go index 9871559a09..eb807df98a 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -150,14 +150,17 @@ func New( "snapshot clean", common.StorageSize(config.SnapshotCache)*1024*1024, ) - // Note: RecoverPruning must be called to handle the case that we are midway through offline pruning. - // If the data directory is changed in between runs preventing RecoverPruning from performing its job correctly, - // it may cause DB corruption. - // Since RecoverPruning will only continue a pruning run that already began, we do not need to ensure that - // reprocessState has already been called and completed successfully. To ensure this, we must maintain - // that Prune is only run after reprocessState has finished successfully. - if err := pruner.RecoverPruning(config.OfflinePruningDataDirectory, chainDb); err != nil { - log.Error("Failed to recover state", "error", err) + // Try to recover offline state pruning only in hash-based. + if config.StateScheme == rawdb.HashScheme { + // Note: RecoverPruning must be called to handle the case that we are midway through offline pruning. + // If the data directory is changed in between runs preventing RecoverPruning from performing its job correctly, + // it may cause DB corruption. + // Since RecoverPruning will only continue a pruning run that already began, we do not need to ensure that + // reprocessState has already been called and completed successfully. To ensure this, we must maintain + // that Prune is only run after reprocessState has finished successfully. + if err := pruner.RecoverPruning(config.OfflinePruningDataDirectory, chainDb); err != nil { + log.Error("Failed to recover state", "error", err) + } } eth := &Ethereum{ @@ -175,7 +178,6 @@ func New( settings: settings, shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb), } - bcVersion := rawdb.ReadDatabaseVersion(chainDb) dbVer := "" if bcVersion != nil { @@ -215,6 +217,8 @@ func New( AcceptedCacheSize: config.AcceptedCacheSize, TxLookupLimit: config.TxLookupLimit, SkipTxIndexing: config.SkipTxIndexing, + StateHistory: config.StateHistory, + StateScheme: config.StateScheme, } ) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8115198e4d..8b0d48847a 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -30,10 +30,12 @@ import ( "time" "github.com/ava-labs/subnet-evm/core" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/txpool/blobpool" "github.com/ava-labs/subnet-evm/core/txpool/legacypool" "github.com/ava-labs/subnet-evm/eth/gasprice" "github.com/ava-labs/subnet-evm/miner" + "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" ) @@ -55,6 +57,8 @@ var DefaultConfig = NewDefaultConfig() func NewDefaultConfig() Config { return Config{ NetworkId: 1, + StateHistory: params.FullImmutabilityThreshold, + StateScheme: rawdb.HashScheme, TrieCleanCache: 512, TrieDirtyCache: 256, TrieDirtyCommitTarget: 20, @@ -158,7 +162,11 @@ type Config struct { // are reserved: // * 0: means no limit // * N: means N block limit [HEAD-N+1, HEAD] and delete extra indexes - TxLookupLimit uint64 + // Deprecated, use 'TransactionHistory' instead. + TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. + StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top // SkipTxIndexing skips indexing transactions. // This is useful for validators that don't need to index transactions. diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 15a8bc803b..74e8179a78 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -364,48 +364,47 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ return logs, nil } -func includes(addresses []common.Address, a common.Address) bool { - for _, addr := range addresses { - if addr == a { +// includes returns true if the element is present in the list. +func includes[T comparable](things []T, element T) bool { + for _, thing := range things { + if thing == element { return true } } - return false } // filterLogs creates a slice of logs matching the given criteria. func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*types.Log { - var ret []*types.Log -Logs: - for _, log := range logs { + var check = func(log *types.Log) bool { if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber { - continue + return false } if toBlock != nil && toBlock.Int64() >= 0 && toBlock.Uint64() < log.BlockNumber { - continue + return false } - if len(addresses) > 0 && !includes(addresses, log.Address) { - continue + return false } // If the to filtered topics is greater than the amount of topics in logs, skip. if len(topics) > len(log.Topics) { - continue + return false } for i, sub := range topics { - match := len(sub) == 0 // empty rule set == wildcard - for _, topic := range sub { - if log.Topics[i] == topic { - match = true - break - } + if len(sub) == 0 { + continue // empty rule set == wildcard } - if !match { - continue Logs + if !includes(sub, log.Topics[i]) { + return false } } - ret = append(ret, log) + return true + } + var ret []*types.Log + for _, log := range logs { + if check(log) { + ret = append(ret, log) + } } return ret } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index c44cc6559b..0fa9fc05df 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -98,7 +98,7 @@ func BenchmarkFilters(b *testing.B) { // The test txs are not properly signed, can't simply create a chain // and then import blocks. TODO(rjl493456442) try to get rid of the // manual database writes. - gspec.MustCommit(db) + gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) for i, block := range chain { rawdb.WriteBlock(db, block) @@ -192,7 +192,7 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, trie.NewDatabase(db)) + _, err = gspec.Commit(db, trie.NewDatabase(db, nil)) if err != nil { t.Fatal(err) } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index e435fdb34f..aa03d5fdc7 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -33,6 +33,7 @@ import ( "time" "github.com/ava-labs/subnet-evm/core" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" @@ -46,32 +47,12 @@ import ( // for releasing state. var noopReleaser = tracers.StateReleaseFunc(func() {}) -// StateAtBlock retrieves the state database associated with a certain block. -// If no state is locally available for the given block, a number of blocks -// are attempted to be reexecuted to generate the desired state. The optional -// base layer statedb can be provided which is regarded as the statedb of the -// parent block. -// -// An additional release function will be returned if the requested state is -// available. Release is expected to be invoked when the returned state is no longer needed. -// Its purpose is to prevent resource leaking. Though it can be noop in some cases. -// -// Parameters: -// - block: The block for which we want the state(state = block.Root) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state -// - base: If the caller is tracing multiple blocks, the caller can provide the parent -// state continuously from the callsite. -// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should -// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. -// Otherwise, the trash generated by caller may be persisted permanently. -// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is -// provided, it would be preferable to start from a fresh state, if we have it -// on disk. -func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { +func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { reexec = 0 // Do not support re-executing historical blocks to grab state var ( current *types.Block database state.Database + triedb *trie.Database report = true origin = block.NumberU64() ) @@ -82,9 +63,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe // on top to prevent garbage collection and return a release // function to deref it. if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil { - statedb.Database().TrieDB().Reference(block.Root(), common.Hash{}) + eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{}) return statedb, func() { - statedb.Database().TrieDB().Dereference(block.Root()) + eth.blockchain.TrieDB().Dereference(block.Root()) }, nil } } @@ -95,14 +76,16 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe if preferDisk { // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block - statedb, database, report = base, base.Database(), false + statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) } else { // Otherwise, try to reexec blocks until we find a state or reach our limit @@ -110,7 +93,10 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. - database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) + // TODO(rjl493456442), clean cache is disabled to prevent memory leak, + // please re-enable it for better performance. + triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults) + database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is @@ -188,15 +174,56 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe // Note: In coreth, the state reference is held by passing true to [statedb.Commit]. // Drop the parent state to prevent accumulating too many nodes in memory. if parent != (common.Hash{}) { - database.TrieDB().Dereference(parent) + triedb.Dereference(parent) } parent = root } if report { - nodes, imgs := database.TrieDB().Size() + _, nodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil + return statedb, func() { triedb.Dereference(block.Root()) }, nil +} + +func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) { + // Check if the requested state is available in the live chain. + statedb, err := eth.blockchain.StateAt(block.Root()) + if err == nil { + return statedb, noopReleaser, nil + } + // TODO historic state is not supported in path-based scheme. + // Fully archive node in pbss will be implemented by relying + // on state history, but needs more work on top. + return nil, nil, errors.New("historical state not available in path scheme yet") +} + +// stateAtBlock retrieves the state database associated with a certain block. +// If no state is locally available for the given block, a number of blocks +// are attempted to be reexecuted to generate the desired state. The optional +// base layer statedb can be provided which is regarded as the statedb of the +// parent block. +// +// An additional release function will be returned if the requested state is +// available. Release is expected to be invoked when the returned state is no +// longer needed. Its purpose is to prevent resource leaking. Though it can be +// noop in some cases. +// +// Parameters: +// - block: The block for which we want the state(state = block.Root) +// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state +// - base: If the caller is tracing multiple blocks, the caller can provide the parent +// state continuously from the callsite. +// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should +// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. +// Otherwise, the trash generated by caller may be persisted permanently. +// - preferDisk: This arg can be used by the caller to signal that even though the 'base' is +// provided, it would be preferable to start from a fresh state, if we have it +// on disk. +func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { + if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme { + return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk) + } + return eth.pathState(block) } // stateAtTransaction returns the execution environment of a certain transaction. @@ -249,7 +276,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // apply the upgrades to the [parent] state before returning it. func (eth *Ethereum) StateAtNextBlock(ctx context.Context, parent *types.Block, nextBlock *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { // Get state for [parent] - statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, base, readOnly, preferDisk) + statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, base, readOnly, preferDisk) if err != nil { return nil, nil, err } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 904ce12160..3f7fcec34f 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -28,7 +28,6 @@ package tracers import ( "bufio" - "bytes" "context" "encoding/json" "errors" @@ -401,8 +400,8 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed // if the relevant state is available in disk. var preferDisk bool if statedb != nil { - s1, s2 := statedb.Database().TrieDB().Size() - preferDisk = s1+s2 > defaultTracechainMemLimit + s1, s2, s3 := statedb.Database().TrieDB().Size() + preferDisk = s1+s2+s3 > defaultTracechainMemLimit } statedb, release, err = api.backend.StateAtNextBlock(ctx, block, next, reexec, statedb, false, preferDisk) if err != nil { @@ -485,7 +484,7 @@ func (api *API) TraceBlockByHash(ctx context.Context, hash common.Hash, config * // and returns them as a JSON object. func (api *baseAPI) TraceBlock(ctx context.Context, blob hexutil.Bytes, config *TraceConfig) ([]*txTraceResult, error) { block := new(types.Block) - if err := rlp.Decode(bytes.NewReader(blob), block); err != nil { + if err := rlp.DecodeBytes(blob, block); err != nil { return nil, fmt.Errorf("could not decode block: %v", err) } return api.traceBlock(ctx, block, config) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index b89a50a364..cbcb1fe555 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -147,8 +147,10 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + triedb.Close() + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) @@ -247,7 +249,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() b.ReportAllocs() b.ResetTimer() @@ -373,7 +376,7 @@ func TestInternals(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), core.GenesisAlloc{ to: core.GenesisAccount{ Code: tc.code, @@ -381,7 +384,9 @@ func TestInternals(t *testing.T) { origin: core.GenesisAccount{ Balance: big.NewInt(500000000000000), }, - }, false) + }, false, rawdb.HashScheme) + defer triedb.Close() + evm := vm.NewEVM(context, txContext, statedb, params.TestPreSubnetEVMConfig, vm.Config{Tracer: tc.tracer}) msg := &core.Message{ To: &to, diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 3b02d35326..3afa5d3080 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -110,7 +110,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string Difficulty: (*big.Int)(test.Context.Difficulty), GasLimit: uint64(test.Context.GasLimit), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) + defer triedb.Close() // Create the tracer, the EVM environment and run it tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 7c296f3e6a..98a18aadba 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -118,8 +118,10 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { GasLimit: uint64(test.Context.GasLimit), BaseFee: test.Genesis.BaseFee, } - _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme) ) + defer triedb.Close() + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index cb40e1bec8..14d7826149 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -246,7 +246,12 @@ func (t *jsTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Addr t.ctx["to"] = t.vm.ToValue(to.Bytes()) t.ctx["input"] = t.vm.ToValue(input) t.ctx["gas"] = t.vm.ToValue(t.gasLimit) - t.ctx["gasPrice"] = t.vm.ToValue(env.TxContext.GasPrice) + gasPriceBig, err := t.toBig(t.vm, env.TxContext.GasPrice.String()) + if err != nil { + t.err = err + return + } + t.ctx["gasPrice"] = gasPriceBig valueBig, err := t.toBig(t.vm, value.String()) if err != nil { t.err = err diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 6777649af9..aa0166d38e 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -89,7 +89,9 @@ func BenchmarkTransactionTrace(b *testing.B) { Code: []byte{}, Balance: big.NewInt(500000000000000), } - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) + triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) + defer triedb.Close() + // Create the tracer, the EVM environment and run it tracer := logger.NewStructLogger(&logger.Config{ Debug: false, diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 6741312331..546cfff556 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -714,7 +714,7 @@ func toCallArg(msg interfaces.CallMsg) interface{} { "to": msg.To, } if len(msg.Data) > 0 { - arg["data"] = hexutil.Bytes(msg.Data) + arg["input"] = hexutil.Bytes(msg.Data) } if msg.Value != nil { arg["value"] = (*hexutil.Big)(msg.Value) diff --git a/ethclient/subnetevmclient/subnet_evm_client.go b/ethclient/subnetevmclient/subnet_evm_client.go index 1ce9a50b3f..638a2e0bd3 100644 --- a/ethclient/subnetevmclient/subnet_evm_client.go +++ b/ethclient/subnetevmclient/subnet_evm_client.go @@ -186,7 +186,7 @@ func toCallArg(msg interfaces.CallMsg) interface{} { "to": msg.To, } if len(msg.Data) > 0 { - arg["data"] = hexutil.Bytes(msg.Data) + arg["input"] = hexutil.Bytes(msg.Data) } if msg.Value != nil { arg["value"] = (*hexutil.Big)(msg.Value) diff --git a/go.mod b/go.mod index 9003b3194c..33e83a10d5 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,9 @@ require ( github.com/cespare/cp v0.1.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 - github.com/docker/docker v1.6.2 - github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/go-ethereum v1.12.2 + github.com/docker/docker v24.0.5+incompatible + github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 + github.com/ethereum/go-ethereum v1.13.2 github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 @@ -38,7 +38,7 @@ require ( github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.8.4 github.com/tyler-smith/go-bip39 v1.1.0 - github.com/urfave/cli/v2 v2.24.1 + github.com/urfave/cli/v2 v2.25.7 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.18.0 @@ -54,7 +54,7 @@ require ( require ( github.com/DataDog/zstd v1.5.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/ava-labs/coreth v0.13.2-rc.2 // indirect + github.com/ava-labs/coreth v0.13.3-0.20240326002912-83b1aa1c7a43 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect @@ -63,7 +63,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect + github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect @@ -90,7 +90,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huin/goupnp v1.0.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/gateway v1.0.6 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/klauspost/compress v1.15.15 // indirect @@ -119,8 +119,8 @@ require ( github.com/subosito/gotenv v1.3.0 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect - github.com/tklauser/go-sysconf v0.3.5 // indirect - github.com/tklauser/numcpus v0.2.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opentelemetry.io/otel v1.22.0 // indirect diff --git a/go.sum b/go.sum index e2f0d6178f..2e2e8bbf05 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/ava-labs/avalanchego v1.11.3 h1:Fgf2R46SFsbe3dbaCu0vFPaA8F1zMqdf6Y/NY github.com/ava-labs/avalanchego v1.11.3/go.mod h1:ruzSPKSH8GBFegvNsnKerD8+8oVnkJ5ejRAOUQ4pAZU= github.com/ava-labs/coreth v0.13.2-rc.2 h1:GmXSyDykDUuDyW7933T8lK7Fp6/4k/IcHhLJjkvjUYI= github.com/ava-labs/coreth v0.13.2-rc.2/go.mod h1:jOapwtgvroqZ2U8PJpoaq1PHrUFOrlgshUWQfM3nba0= +github.com/ava-labs/coreth v0.13.3-0.20240326002912-83b1aa1c7a43 h1:CR0HAG6CYakCyxibAmehCDyjvyriWt2pSxhmDR8MrKk= +github.com/ava-labs/coreth v0.13.3-0.20240326002912-83b1aa1c7a43/go.mod h1:n128DRgabYrCAUsGEXOKP0uzBLSV37zGIGs7xTAQZDY= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -107,15 +109,16 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= +github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= +github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= +github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -149,11 +152,11 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/docker/docker v1.6.2 h1:HlFGsy+9/xrgMmhmN+NGhCc5SHGJ7I+kHosRR1xc/aI= -github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= +github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= -github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= +github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -168,8 +171,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= -github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= +github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= +github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= @@ -334,9 +337,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iU github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -545,18 +547,18 @@ github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5f github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/thepudds/fzgen v0.4.2 h1:HlEHl5hk2/cqEomf2uK5SA/FeJc12s/vIHmOG+FbACw= github.com/thepudds/fzgen v0.4.2/go.mod h1:kHCWdsv5tdnt32NIHYDdgq083m6bMtaY0M+ipiO9xWE= -github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= -github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= -github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= -github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -780,7 +782,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -799,6 +800,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1019,6 +1022,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 1453b92049..714811787f 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -95,8 +95,9 @@ var ( Category: flags.LoggingCategory, } logRotateFlag = &cli.BoolFlag{ - Name: "log.rotate", - Usage: "Enables log file rotation", + Name: "log.rotate", + Usage: "Enables log file rotation", + Category: flags.LoggingCategory, } logMaxSizeMBsFlag = &cli.IntFlag{ Name: "log.maxsize", diff --git a/internal/debug/loudpanic.go b/internal/debug/loudpanic.go index 2bf10929ea..44634913cc 100644 --- a/internal/debug/loudpanic.go +++ b/internal/debug/loudpanic.go @@ -24,9 +24,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build go1.6 -// +build go1.6 - package debug import "runtime/debug" diff --git a/internal/debug/loudpanic_fallback.go b/internal/debug/loudpanic_fallback.go deleted file mode 100644 index df21bb948a..0000000000 --- a/internal/debug/loudpanic_fallback.go +++ /dev/null @@ -1,35 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !go1.6 -// +build !go1.6 - -package debug - -// LoudPanic panics in a way that gets all goroutine stacks printed on stderr. -func LoudPanic(x interface{}) { - panic(x) -} diff --git a/internal/debug/trace.go b/internal/debug/trace.go index 7c51228bc1..12f7bfdb05 100644 --- a/internal/debug/trace.go +++ b/internal/debug/trace.go @@ -24,9 +24,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build go1.5 -// +build go1.5 - package debug import ( diff --git a/internal/debug/trace_fallback.go b/internal/debug/trace_fallback.go deleted file mode 100644 index b97a17012d..0000000000 --- a/internal/debug/trace_fallback.go +++ /dev/null @@ -1,42 +0,0 @@ -// (c) 2019-2020, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !go1.5 -// +build !go1.5 - -// no-op implementation of tracing methods for Go < 1.5. - -package debug - -import "errors" - -func (*HandlerT) StartGoTrace(string) error { - return errors.New("tracing is not supported on Go < 1.5") -} - -func (*HandlerT) StopGoTrace() error { - return errors.New("tracing is not supported on Go < 1.5") -} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 37e2be6ae3..10bfd57d71 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -48,6 +48,7 @@ import ( "github.com/ava-labs/subnet-evm/eth/tracers/logger" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" + "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/vmerrs" "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" @@ -710,9 +711,10 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st keys = make([]common.Hash, len(storageKeys)) keyLengths = make([]int, len(storageKeys)) storageProof = make([]StorageResult, len(storageKeys)) - storageTrie state.Trie - storageHash = types.EmptyRootHash - codeHash = types.EmptyCodeHash + + storageTrie state.Trie + storageHash = types.EmptyRootHash + codeHash = types.EmptyCodeHash ) // Deserialize all keys. This prevents state access on invalid input. for i, hexKey := range storageKeys { @@ -722,15 +724,18 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st return nil, err } } - - state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + state, header, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err } - if storageTrie, err = state.StorageTrie(address); err != nil { - return nil, err + if storageRoot := state.GetStorageRoot(address); storageRoot != types.EmptyRootHash && storageRoot != (common.Hash{}) { + id := trie.StorageTrieID(header.Root, crypto.Keccak256Hash(address.Bytes()), storageRoot) + tr, err := trie.NewStateTrie(id, state.Database().TrieDB()) + if err != nil { + return nil, err + } + storageTrie = tr } - // If we have a storageTrie, the account exists and we must update // the storage root hash and the code hash. if storageTrie != nil { @@ -763,14 +768,17 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st } // Create the accountProof. - accountProof, proofErr := state.GetProof(address) - if proofErr != nil { - return nil, proofErr + tr, err := trie.NewStateTrie(trie.StateTrieID(header.Root), state.Database().TrieDB()) + if err != nil { + return nil, err + } + var accountProof proofList + if err := tr.Prove(crypto.Keccak256(address.Bytes()), &accountProof); err != nil { + return nil, err } - return &AccountResult{ Address: address, - AccountProof: toHexSlice(accountProof), + AccountProof: accountProof, Balance: (*hexutil.Big)(state.GetBalance(address)), CodeHash: codeHash, Nonce: hexutil.Uint64(state.GetNonce(address)), @@ -1236,8 +1244,12 @@ func (s *BlockChainAPI) CallDetailed(ctx context.Context, args TransactionArgs, // // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. -func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, blockOverrides *BlockOverrides) (hexutil.Bytes, error) { - result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, blockOverrides, s.b.RPCEVMTimeout(), s.b.RPCGasCap()) +func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, overrides *StateOverride, blockOverrides *BlockOverrides) (hexutil.Bytes, error) { + if blockNrOrHash == nil { + latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + blockNrOrHash = &latest + } + result, err := DoCall(ctx, s.b, args, *blockNrOrHash, overrides, blockOverrides, s.b.RPCEVMTimeout(), s.b.RPCGasCap()) if err != nil { return nil, err } @@ -1248,12 +1260,30 @@ func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrO return result.Return(), result.Err } +// executeEstimate is a helper that executes the transaction under a given gas limit and returns +// true if the transaction fails for a reason that might be related to not enough gas. A non-nil +// error means execution failed due to reasons unrelated to the gas limit. +func executeEstimate(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, gasCap uint64, gasLimit uint64) (bool, *core.ExecutionResult, error) { + args.Gas = (*hexutil.Uint64)(&gasLimit) + result, err := doCall(ctx, b, args, state, header, nil, nil, 0, gasCap) + if err != nil { + if errors.Is(err, core.ErrIntrinsicGas) { + return true, nil, nil // Special case, raise gas limit + } + return true, nil, err // Bail out + } + return result.Failed(), result, nil +} + +// DoEstimateGas returns the lowest possible gas limit that allows the transaction to run +// successfully at block `blockNrOrHash`. It returns error if the transaction would revert, or if +// there are unexpected failures. The gas limit is capped by both `args.Gas` (if non-nil & +// non-zero) and `gasCap` (if non-zero). func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, gasCap uint64) (hexutil.Uint64, error) { - // Binary search the gas requirement, as it may be higher than the amount used + // Binary search the gas limit, as it may need to be higher than the amount used var ( - lo uint64 = params.TxGas - 1 - hi uint64 - cap uint64 + lo uint64 // lowest-known gas limit where tx execution fails + hi uint64 // lowest-known gas limit where tx execution succeeds ) // Use zero address if sender unspecified. if args.From == nil { @@ -1284,16 +1314,17 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } else { feeCap = common.Big0 } + + state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return 0, err + } + if err := overrides.Apply(state); err != nil { + return 0, err + } + // Recap the highest gas limit with account's available balance. if feeCap.BitLen() != 0 { - state, _, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if err != nil { - return 0, err - } - err = overrides.Apply(state) - if err != nil { - return 0, err - } balance := state.GetBalance(*args.From) // from can't be nil available := new(big.Int).Set(balance) if args.Value != nil { @@ -1320,39 +1351,42 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr log.Info("Caller gas above allowance, capping", "requested", hi, "cap", gasCap) hi = gasCap } - cap = hi - // Create a helper to check if a gas allowance results in an executable transaction - executable := func(gas uint64, state *state.StateDB, header *types.Header) (bool, *core.ExecutionResult, error) { - args.Gas = (*hexutil.Uint64)(&gas) - - result, err := doCall(ctx, b, args, state, header, nil, nil, 0, gasCap) - if err != nil { - if errors.Is(err, core.ErrIntrinsicGas) { - return true, nil, nil // Special case, raise gas limit - } - return true, nil, err // Bail out - } - return result.Failed(), result, nil - } - state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return 0, err - } - err = overrides.Apply(state) + // We first execute the transaction at the highest allowable gas limit, since if this fails we + // can return error immediately. + failed, result, err := executeEstimate(ctx, b, args, state.Copy(), header, gasCap, hi) if err != nil { return 0, err } - // Execute the binary search and hone in on an executable gas limit + if failed { + if result != nil && result.Err != vmerrs.ErrOutOfGas { + if len(result.Revert()) > 0 { + return 0, newRevertError(result) + } + return 0, result.Err + } + return 0, fmt.Errorf("gas required exceeds allowance (%d)", hi) + } + // For almost any transaction, the gas consumed by the unconstrained execution above + // lower-bounds the gas limit required for it to succeed. One exception is those txs that + // explicitly check gas remaining in order to successfully execute within a given limit, but we + // probably don't want to return a lowest possible gas limit for these cases anyway. + lo = result.UsedGas - 1 + + // Binary search for the smallest gas limit that allows the tx to execute successfully. for lo+1 < hi { - s := state.Copy() mid := (hi + lo) / 2 - failed, _, err := executable(mid, s, header) - - // If the error is not nil(consensus error), it means the provided message - // call or transaction will never be accepted no matter how much gas it is - // assigned. Return the error directly, don't struggle any more. + if mid > lo*2 { + // Most txs don't need much higher gas limit than their gas used, and most txs don't + // require near the full block limit of gas, so the selection of where to bisect the + // range here is skewed to favor the low side. + mid = lo * 2 + } + failed, _, err = executeEstimate(ctx, b, args, state.Copy(), header, gasCap, mid) if err != nil { + // This should not happen under normal conditions since if we make it this far the + // transaction had run without error at least once before. + log.Error("execution error in estimate gas", "err", err) return 0, err } if failed { @@ -1361,28 +1395,14 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr hi = mid } } - // Reject the transaction as invalid if it still fails at the highest allowance - if hi == cap { - failed, result, err := executable(hi, state, header) - if err != nil { - return 0, err - } - if failed { - if result != nil && result.Err != vmerrs.ErrOutOfGas { - if len(result.Revert()) > 0 { - return 0, newRevertError(result) - } - return 0, result.Err - } - // Otherwise, the specified gas cap is too low - return 0, fmt.Errorf("gas required exceeds allowance (%d)", cap) - } - } return hexutil.Uint64(hi), nil } -// EstimateGas returns an estimate of the amount of gas needed to execute the -// given transaction against the current pending block. +// EstimateGas returns the lowest possible gas limit that allows the transaction to run +// successfully at block `blockNrOrHash`, or the latest block if `blockNrOrHash` is unspecified. It +// returns error if the transaction would revert or if there are unexpected failures. The returned +// value is capped by both `args.Gas` (if non-nil & non-zero) and the backend's RPCGasCap +// configuration (if non-zero). func (s *BlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Uint64, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) if blockNrOrHash != nil { @@ -1411,14 +1431,21 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { "transactionsRoot": head.TxHash, "receiptsRoot": head.ReceiptHash, } - if head.BaseFee != nil { result["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee) } if head.BlockGasCost != nil { result["blockGasCost"] = (*hexutil.Big)(head.BlockGasCost) } - + if head.BlobGasUsed != nil { + result["blobGasUsed"] = hexutil.Uint64(*head.BlobGasUsed) + } + if head.ExcessBlobGas != nil { + result["excessBlobGas"] = hexutil.Uint64(*head.ExcessBlobGas) + } + if head.ParentBeaconRoot != nil { + result["parentBeaconBlockRoot"] = head.ParentBeaconRoot + } return result } @@ -1478,26 +1505,28 @@ func (s *BlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inc // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { - BlockHash *common.Hash `json:"blockHash"` - BlockNumber *hexutil.Big `json:"blockNumber"` - From common.Address `json:"from"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - GasFeeCap *hexutil.Big `json:"maxFeePerGas,omitempty"` - GasTipCap *hexutil.Big `json:"maxPriorityFeePerGas,omitempty"` - Hash common.Hash `json:"hash"` - Input hexutil.Bytes `json:"input"` - Nonce hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` - Value *hexutil.Big `json:"value"` - Type hexutil.Uint64 `json:"type"` - Accesses *types.AccessList `json:"accessList,omitempty"` - ChainID *hexutil.Big `json:"chainId,omitempty"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` - YParity *hexutil.Uint64 `json:"yParity,omitempty"` + BlockHash *common.Hash `json:"blockHash"` + BlockNumber *hexutil.Big `json:"blockNumber"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + GasFeeCap *hexutil.Big `json:"maxFeePerGas,omitempty"` + GasTipCap *hexutil.Big `json:"maxPriorityFeePerGas,omitempty"` + MaxFeePerBlobGas *hexutil.Big `json:"maxFeePerBlobGas,omitempty"` + Hash common.Hash `json:"hash"` + Input hexutil.Bytes `json:"input"` + Nonce hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + TransactionIndex *hexutil.Uint64 `json:"transactionIndex"` + Value *hexutil.Big `json:"value"` + Type hexutil.Uint64 `json:"type"` + Accesses *types.AccessList `json:"accessList,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + YParity *hexutil.Uint64 `json:"yParity,omitempty"` } // newRPCTransaction returns a transaction that will serialize to the RPC @@ -1551,15 +1580,43 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber // if the transaction has been mined, compute the effective gas price if baseFee != nil && blockHash != (common.Hash{}) { // price = min(gasTipCap + baseFee, gasFeeCap) - price := math.BigMin(new(big.Int).Add(tx.GasTipCap(), baseFee), tx.GasFeeCap()) - result.GasPrice = (*hexutil.Big)(price) + result.GasPrice = (*hexutil.Big)(effectiveGasPrice(tx, baseFee)) + } else { + result.GasPrice = (*hexutil.Big)(tx.GasFeeCap()) + } + + case types.BlobTxType: + al := tx.AccessList() + yparity := hexutil.Uint64(v.Sign()) + result.Accesses = &al + result.ChainID = (*hexutil.Big)(tx.ChainId()) + result.YParity = &yparity + result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap()) + result.GasTipCap = (*hexutil.Big)(tx.GasTipCap()) + // if the transaction has been mined, compute the effective gas price + if baseFee != nil && blockHash != (common.Hash{}) { + result.GasPrice = (*hexutil.Big)(effectiveGasPrice(tx, baseFee)) } else { result.GasPrice = (*hexutil.Big)(tx.GasFeeCap()) } + result.MaxFeePerBlobGas = (*hexutil.Big)(tx.BlobGasFeeCap()) + result.BlobVersionedHashes = tx.BlobHashes() } return result } +// effectiveGasPrice computes the transaction gas fee, based on the given basefee value. +// +// price = min(gasTipCap + baseFee, gasFeeCap) +func effectiveGasPrice(tx *types.Transaction, baseFee *big.Int) *big.Int { + fee := tx.GasTipCap() + fee = fee.Add(fee, baseFee) + if tx.GasFeeCapIntCmp(fee) < 0 { + return tx.GasFeeCap() + } + return fee +} + // NewRPCTransaction returns a pending transaction that will serialize to the RPC representation // Note: in go-ethereum this function is called NewRPCPendingTransaction. // In subnet-evm, we have renamed it to NewRPCTransaction as it is used for accepted transactions as well. @@ -1904,6 +1961,11 @@ func marshalReceipt(receipt *types.Receipt, blockHash common.Hash, blockNumber u fields["logs"] = []*types.Log{} } + if tx.Type() == types.BlobTxType { + fields["blobGasUsed"] = hexutil.Uint64(receipt.BlobGasUsed) + fields["blobGasPrice"] = (*hexutil.Big)(receipt.BlobGasPrice) + } + // If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation if receipt.ContractAddress != (common.Address{}) { fields["contractAddress"] = receipt.ContractAddress @@ -2294,12 +2356,3 @@ func checkTxFee(gasPrice *big.Int, gas uint64, cap float64) error { } return nil } - -// toHexSlice creates a slice of hex-strings based on []byte. -func toHexSlice(b [][]byte) []string { - r := make([]string, len(b)) - for i := range b { - r[i] = hexutil.Encode(b[i]) - } - return r -} diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 6b7251266d..db07958d61 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -33,6 +33,8 @@ import ( "errors" "fmt" "math/big" + "os" + "path/filepath" "reflect" "testing" "time" @@ -55,19 +57,18 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/stretchr/testify/assert" + "github.com/holiman/uint256" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" ) -func TestTransaction_RoundTripRpcJSON(t *testing.T) { +func testTransactionMarshal(t *testing.T, tests []txData, config *params.ChainConfig) { + t.Parallel() var ( - config = params.TestChainConfig signer = types.LatestSigner(config) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - tests = allTransactionTypes(common.Address{0xde, 0xad}, config) ) - t.Parallel() + for i, tt := range tests { var tx2 types.Transaction tx, err := types.SignNewTx(key, signer, tt.Tx) @@ -98,6 +99,23 @@ func TestTransaction_RoundTripRpcJSON(t *testing.T) { } } +func TestTransaction_RoundTripRpcJSON(t *testing.T) { + var ( + config = params.TestChainConfig + tests = allTransactionTypes(common.Address{0xde, 0xad}, config) + ) + testTransactionMarshal(t, tests, config) +} + +func TestTransactionBlobTx(t *testing.T) { + config := *params.TestChainConfig + // config.ShanghaiTime = new(uint64) + config.CancunTime = new(uint64) + tests := allBlobTxs(common.Address{0xde, 0xad}, &config) + + testTransactionMarshal(t, tests, &config) +} + type txData struct { Tx types.TxData Want string @@ -348,14 +366,59 @@ func allTransactionTypes(addr common.Address, config *params.ChainConfig) []txDa } } +func allBlobTxs(addr common.Address, config *params.ChainConfig) []txData { + return []txData{ + { + Tx: &types.BlobTx{ + Nonce: 6, + GasTipCap: uint256.NewInt(1), + GasFeeCap: uint256.NewInt(5), + Gas: 6, + To: addr, + BlobFeeCap: uint256.NewInt(1), + BlobHashes: []common.Hash{{1}}, + Value: new(uint256.Int), + V: uint256.NewInt(32), + R: uint256.NewInt(10), + S: uint256.NewInt(11), + }, + Want: `{ + "blockHash": null, + "blockNumber": null, + "from": "0x71562b71999873db5b286df957af199ec94617f7", + "gas": "0x6", + "gasPrice": "0x5", + "maxFeePerGas": "0x5", + "maxPriorityFeePerGas": "0x1", + "maxFeePerBlobGas": "0x1", + "hash": "0x1f2b59a20e61efc615ad0cbe936379d6bbea6f938aafaf35eb1da05d8e7f46a3", + "input": "0x", + "nonce": "0x6", + "to": "0xdead000000000000000000000000000000000000", + "transactionIndex": null, + "value": "0x0", + "type": "0x3", + "accessList": [], + "chainId": "0x1", + "blobVersionedHashes": [ + "0x0100000000000000000000000000000000000000000000000000000000000000" + ], + "v": "0x0", + "r": "0x618be8908e0e5320f8f3b48042a079fe5a335ebd4ed1422a7d2207cd45d872bc", + "s": "0x27b2bc6c80e849a8e8b764d4549d8c2efac3441e73cf37054eb0a9b9f8e89b27", + "yParity": "0x0" + }`, + }, + } +} + type testBackend struct { db ethdb.Database chain *core.BlockChain } -func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i int, b *core.BlockGen)) *testBackend { +func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.Engine, generator func(i int, b *core.BlockGen)) *testBackend { var ( - engine = dummy.NewCoinbaseFaker() cacheConfig = &core.CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, @@ -557,7 +620,7 @@ func TestEstimateGas(t *testing.T) { signer = types.HomesteadSigner{} randomAccounts = newAccounts(2) ) - api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, dummy.NewCoinbaseFaker(), func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei @@ -660,7 +723,7 @@ func TestCall(t *testing.T) { genBlocks = 10 signer = types.HomesteadSigner{} ) - api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + api := NewBlockChainAPI(newTestBackend(t, genBlocks, genesis, dummy.NewCoinbaseFaker(), func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei @@ -789,7 +852,7 @@ func TestCall(t *testing.T) { }, } for i, tc := range testSuite { - result, err := api.Call(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides) + result, err := api.Call(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides) if tc.expectErr != nil { if err == nil { t.Errorf("test %d: want error %v, have nothing", i, tc.expectErr) @@ -1043,7 +1106,7 @@ func TestRPCMarshalBlock(t *testing.T) { t.Errorf("test %d: json marshal error: %v", i, err) continue } - assert.JSONEqf(t, tc.want, string(out), "test %d", i) + require.JSONEqf(t, tc.want, string(out), "test %d", i) } } @@ -1075,7 +1138,7 @@ func TestRPCGetBlockOrHeader(t *testing.T) { }) pending = types.NewBlock(&types.Header{Number: big.NewInt(11), Time: 42}, []*types.Transaction{tx}, nil, nil, blocktest.NewHasher()) ) - backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + backend := newTestBackend(t, genBlocks, genesis, dummy.NewCoinbaseFaker(), func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei @@ -1099,540 +1162,156 @@ func TestRPCGetBlockOrHeader(t *testing.T) { blockHash *common.Hash fullTx bool reqHeader bool - want string + file string expectErr error }{ // 0. latest header { blockNumber: rpc.LatestBlockNumber, reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0xa", - "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", - "timestamp": "0x64", - "totalDifficulty": "0xa", - "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01" - }`, + file: "tag-latest", }, // 1. genesis header { blockNumber: rpc.BlockNumber(0), reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "difficulty": "0x20000", - "extraData": "0x", - "gasLimit": "0x47e7c4", - "gasUsed": "0x0", - "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x0", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", - "timestamp": "0x0", - "totalDifficulty": "0x0", - "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - }`, + file: "number-0", }, // 2. #1 header { blockNumber: rpc.BlockNumber(1), reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x1", - "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", - "timestamp": "0xa", - "totalDifficulty": "0x1", - "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5" - }`, + file: "number-1", }, // 3. latest-1 header { blockNumber: rpc.BlockNumber(9), reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x9", - "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", - "timestamp": "0x5a", - "totalDifficulty": "0x9", - "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb" - }`, + file: "number-latest-1", }, // 4. latest+1 header { blockNumber: rpc.BlockNumber(11), reqHeader: true, - want: "null", + file: "number-latest+1", }, // 5. pending header { blockNumber: rpc.PendingBlockNumber, reqHeader: true, - want: "null", + file: "tag-pending", }, // 6. latest block { blockNumber: rpc.LatestBlockNumber, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0xa", - "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", - "timestamp": "0x64", - "totalDifficulty": "0xa", - "transactions": [ - "0x71be223424ab6e3457513a760b196d43b094414c32a70ff929b2b720a16b832d" - ], - "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01", - "uncles": [] - }`, + file: "tag-latest", }, // 7. genesis block { blockNumber: rpc.BlockNumber(0), - want: `{ - "baseFeePerGas": "0x5d21dba00", - "difficulty": "0x20000", - "extraData": "0x", - "gasLimit": "0x47e7c4", - "gasUsed": "0x0", - "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x0", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x201", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", - "timestamp": "0x0", - "totalDifficulty": "0x0", - "transactions": [], - "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "uncles": [] - }`, + file: "number-0", }, // 8. #1 block { blockNumber: rpc.BlockNumber(1), - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x1", - "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", - "timestamp": "0xa", - "totalDifficulty": "0x1", - "transactions": [ - "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26" - ], - "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5", - "uncles": [] - }`, + file: "number-1", }, // 9. latest-1 block { blockNumber: rpc.BlockNumber(9), fullTx: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x9", - "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", - "timestamp": "0x5a", - "totalDifficulty": "0x9", - "transactions": [ - { - "blockHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "blockNumber": "0x9", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gas": "0x5208", - "gasPrice": "0x5d21dba00", - "hash": "0x237f95840187a93f8aaf8d6f1515f8a8ac9d9359fcb0c220cdb3d642d6b9a19a", - "input": "0x", - "nonce": "0x8", - "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionIndex": "0x0", - "value": "0x3e8", - "type": "0x0", - "v": "0x1c", - "r": "0xd7cdc527490b7ba29c515aae3bbe80c67729cda7f736e6515652cfc40e9da68f", - "s": "0x4d0a4a59bef165b16f910bdadd41efaaad1b73549bacc35eaf6d073eb1fb92b7" - } - ], - "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb", - "uncles": [] - }`, + file: "number-latest-1", }, // 10. latest+1 block { blockNumber: rpc.BlockNumber(11), fullTx: true, - want: "null", + file: "number-latest+1", }, // 11. pending block { blockNumber: rpc.PendingBlockNumber, - want: "null", + file: "tag-pending", }, // 12. pending block + fullTx { blockNumber: rpc.PendingBlockNumber, fullTx: true, - want: "null", + file: "tag-pending-fullTx", }, // 13. latest header by hash { blockHash: &blockHashes[len(blockHashes)-1], reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0xa", - "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", - "timestamp": "0x64", - "totalDifficulty": "0xa", - "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01" - }`, + file: "hash-latest", }, // 14. genesis header by hash { blockHash: &blockHashes[0], reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "difficulty": "0x20000", - "extraData": "0x", - "gasLimit": "0x47e7c4", - "gasUsed": "0x0", - "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x0", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", - "timestamp": "0x0", - "totalDifficulty": "0x0", - "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - }`, + file: "hash-0", }, // 15. #1 header { blockHash: &blockHashes[1], reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x1", - "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", - "timestamp": "0xa", - "totalDifficulty": "0x1", - "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5" - }`, + file: "hash-1", }, // 16. latest-1 header { blockHash: &blockHashes[len(blockHashes)-2], reqHeader: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x9", - "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", - "timestamp": "0x5a", - "totalDifficulty": "0x9", - "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb" - }`, + file: "hash-latest-1", }, // 17. empty hash { blockHash: &common.Hash{}, reqHeader: true, - want: "null", + file: "hash-empty", }, // 18. pending hash { blockHash: &pendingHash, reqHeader: true, - want: `null`, + file: `hash-pending`, }, // 19. latest block { blockHash: &blockHashes[len(blockHashes)-1], - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0xa", - "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", - "timestamp": "0x64", - "totalDifficulty": "0xa", - "transactions": [ - "0x71be223424ab6e3457513a760b196d43b094414c32a70ff929b2b720a16b832d" - ], - "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01", - "uncles": [] - }`, + file: "hash-latest", }, // 20. genesis block { blockHash: &blockHashes[0], - want: `{ - "baseFeePerGas": "0x5d21dba00", - "difficulty": "0x20000", - "extraData": "0x", - "gasLimit": "0x47e7c4", - "gasUsed": "0x0", - "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x0", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x201", - "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", - "timestamp": "0x0", - "totalDifficulty": "0x0", - "transactions": [], - "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "uncles": [] - }`, + file: "hash-genesis", }, // 21. #1 block { blockHash: &blockHashes[1], - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x1", - "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", - "timestamp": "0xa", - "totalDifficulty": "0x1", - "transactions": [ - "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26" - ], - "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5", - "uncles": [] - }`, + file: "hash-1", }, // 22. latest-1 block { blockHash: &blockHashes[len(blockHashes)-2], fullTx: true, - want: `{ - "baseFeePerGas": "0x5d21dba00", - "blockGasCost": "0x0", - "difficulty": "0x1", - "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "gasLimit": "0x7a1200", - "gasUsed": "0x5208", - "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "miner": "0x0000000000000000000000000000000000000000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "nonce": "0x0000000000000000", - "number": "0x9", - "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", - "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", - "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "size": "0x2bb", - "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", - "timestamp": "0x5a", - "totalDifficulty": "0x9", - "transactions": [ - { - "blockHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", - "blockNumber": "0x9", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gas": "0x5208", - "gasPrice": "0x5d21dba00", - "hash": "0x237f95840187a93f8aaf8d6f1515f8a8ac9d9359fcb0c220cdb3d642d6b9a19a", - "input": "0x", - "nonce": "0x8", - "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionIndex": "0x0", - "value": "0x3e8", - "type": "0x0", - "v": "0x1c", - "r": "0xd7cdc527490b7ba29c515aae3bbe80c67729cda7f736e6515652cfc40e9da68f", - "s": "0x4d0a4a59bef165b16f910bdadd41efaaad1b73549bacc35eaf6d073eb1fb92b7" - } - ], - "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb", - "uncles": [] - }`, + file: "hash-latest-1-fullTx", }, // 23. empty hash + body { blockHash: &common.Hash{}, fullTx: true, - want: "null", + file: "hash-empty-fullTx", }, // 24. pending block { blockHash: &pendingHash, - want: `null`, + file: `hash-pending`, }, // 25. pending block + fullTx { blockHash: &pendingHash, fullTx: true, - want: `null`, + file: "hash-pending-fullTx", }, } @@ -1640,18 +1319,23 @@ func TestRPCGetBlockOrHeader(t *testing.T) { var ( result map[string]interface{} err error + rpc string ) if tt.blockHash != nil { if tt.reqHeader { result = api.GetHeaderByHash(context.Background(), *tt.blockHash) + rpc = "eth_getHeaderByHash" } else { result, err = api.GetBlockByHash(context.Background(), *tt.blockHash, tt.fullTx) + rpc = "eth_getBlockByHash" } } else { if tt.reqHeader { result, err = api.GetHeaderByNumber(context.Background(), tt.blockNumber) + rpc = "eth_getHeaderByNumber" } else { result, err = api.GetBlockByNumber(context.Background(), tt.blockNumber, tt.fullTx) + rpc = "eth_getBlockByNumber" } } if tt.expectErr != nil { @@ -1668,20 +1352,15 @@ func TestRPCGetBlockOrHeader(t *testing.T) { t.Errorf("test %d: want no error, have %v", i, err) continue } - data, err := json.Marshal(result) - if err != nil { - t.Errorf("test %d: json marshal error", i) - continue - } - want, have := tt.want, string(data) - require.JSONEqf(t, want, have, "test %d: json not match, want: %s, have: %s", i, want, have) + + testRPCResponseWithFile(t, i, result, rpc, tt.file) } } -func TestRPCGetTransactionReceipt(t *testing.T) { - t.Parallel() - - // Initialize test accounts +func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Hash) { + config := *params.TestChainConfig + // config.ShanghaiTime = new(uint64) + config.CancunTime = new(uint64) var ( acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") @@ -1689,7 +1368,9 @@ func TestRPCGetTransactionReceipt(t *testing.T) { acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) contract = common.HexToAddress("0000000000000000000000000000000000031ec7") genesis = &core.Genesis{ - Config: params.TestSubnetEVMConfig, + Config: &config, + ExcessBlobGas: new(uint64), + BlobGasUsed: new(uint64), Alloc: core.GenesisAlloc{ acc1Addr: {Balance: big.NewInt(params.Ether)}, acc2Addr: {Balance: big.NewInt(params.Ether)}, @@ -1706,11 +1387,14 @@ func TestRPCGetTransactionReceipt(t *testing.T) { contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, }, } - genBlocks = 5 - signer = types.LatestSignerForChainID(params.TestChainConfig.ChainID) - txHashes = make([]common.Hash, genBlocks) + signer = types.LatestSignerForChainID(params.TestChainConfig.ChainID) + txHashes = make([]common.Hash, genBlocks) ) - backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + + // Set the terminal total difficulty in the config + // genesis.Config.TerminalTotalDifficulty = big.NewInt(0) + // genesis.Config.TerminalTotalDifficultyPassed = true + backend := newTestBackend(t, genBlocks, genesis, dummy.NewCoinbaseFaker(), func(i int, b *core.BlockGen) { var ( tx *types.Transaction err error @@ -1741,6 +1425,20 @@ func TestRPCGetTransactionReceipt(t *testing.T) { StorageKeys: []common.Hash{{0}}, }} tx, err = types.SignTx(types.NewTx(&types.AccessListTx{Nonce: uint64(i), To: nil, Gas: 58100, GasPrice: b.BaseFee(), Data: common.FromHex("0x60806040"), AccessList: accessList}), signer, acc1Key) + case 5: + // blob tx + fee := big.NewInt(500) + fee.Add(fee, b.BaseFee()) + tx, err = types.SignTx(types.NewTx(&types.BlobTx{ + Nonce: uint64(i), + GasTipCap: uint256.NewInt(1), + GasFeeCap: uint256.MustFromBig(fee), + Gas: params.TxGas, + To: acc2Addr, + BlobFeeCap: uint256.NewInt(1), + BlobHashes: []common.Hash{{1}}, + Value: new(uint256.Int), + }), signer, acc1Key) } if err != nil { t.Errorf("failed to sign tx: %v", err) @@ -1749,147 +1447,65 @@ func TestRPCGetTransactionReceipt(t *testing.T) { b.AddTx(tx) txHashes[i] = tx.Hash() } - }) - api := NewTransactionAPI(backend, new(AddrLocker)) - blockHashes := make([]common.Hash, genBlocks+1) - ctx := context.Background() - for i := 0; i <= genBlocks; i++ { - header, err := backend.HeaderByNumber(ctx, rpc.BlockNumber(i)) - if err != nil { - t.Errorf("failed to get block: %d err: %v", i, err) + if i == 5 { + b.SetBlobGas(params.BlobTxBlobGasPerBlob) } - blockHashes[i] = header.Hash() - } + // b.SetPoS() + }) + return backend, txHashes +} + +func TestRPCGetTransactionReceipt(t *testing.T) { + t.Parallel() + + var ( + backend, txHashes = setupReceiptBackend(t, 6) + api = NewTransactionAPI(backend, new(AddrLocker)) + ) var testSuite = []struct { txHash common.Hash - want string + file string }{ // 0. normal success { txHash: txHashes[0], - want: `{ - "blockHash": "0xcc27e155b6eadfa892992a8cd8adaf3c929a6ec6d98c4dfbc60258883c73568e", - "blockNumber": "0x1", - "contractAddress": null, - "cumulativeGasUsed": "0x5208", - "effectiveGasPrice": "0x5d21dba00", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0x5208", - "logs": [], - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "status": "0x1", - "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionHash": "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26", - "transactionIndex": "0x0", - "type": "0x0" - }`, + file: "normal-transfer-tx", }, // 1. create contract { txHash: txHashes[1], - want: `{ - "blockHash": "0xbea66b509ec6e5639279ca696def697d47d0c40ecfa00bbcdb5e31a492491c83", - "blockNumber": "0x2", - "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", - "cumulativeGasUsed": "0xcf4e", - "effectiveGasPrice": "0x5d21dba00", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0xcf4e", - "logs": [], - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "status": "0x1", - "to": null, - "transactionHash": "0x22aa617165f83a9f8c191c2b7724ae43eeb1249bee06c98c03c7624c21d27dc8", - "transactionIndex": "0x0", - "type": "0x0" - }`, + file: "create-contract-tx", }, // 2. with logs success { txHash: txHashes[2], - want: `{ - "blockHash": "0x139eee6b02792c6bee20be4d0aa72b3876f22cf8fe8e2bf45e1a0cee94aa3cf1", - "blockNumber": "0x3", - "contractAddress": null, - "cumulativeGasUsed": "0x5e28", - "effectiveGasPrice": "0x5d21dba00", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0x5e28", - "logs": [ - { - "address": "0x0000000000000000000000000000000000031ec7", - "topics": [ - "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", - "0x000000000000000000000000703c4b2bd70c169f5717101caee543299fc946c7", - "0x0000000000000000000000000000000000000000000000000000000000000003" - ], - "data": "0x000000000000000000000000000000000000000000000000000000000000000d", - "blockNumber": "0x3", - "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", - "transactionIndex": "0x0", - "blockHash": "0x139eee6b02792c6bee20be4d0aa72b3876f22cf8fe8e2bf45e1a0cee94aa3cf1", - "logIndex": "0x0", - "removed": false - } - ], - "logsBloom": "0x00000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000800000000000000008000000000000000000000000000000000020000000080000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000400000000002000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000", - "status": "0x1", - "to": "0x0000000000000000000000000000000000031ec7", - "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", - "transactionIndex": "0x0", - "type": "0x0" - }`, + file: "with-logs", }, // 3. dynamic tx with logs success { txHash: txHashes[3], - want: `{ - "blockHash": "0xba48c351b0aa848ab2ec889f5794f0db779e0840af80472d1c29df54b22288c8", - "blockNumber": "0x4", - "contractAddress": null, - "cumulativeGasUsed": "0x538d", - "effectiveGasPrice": "0x5d21dbbf4", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0x538d", - "logs": [], - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "status": "0x0", - "to": "0x0000000000000000000000000000000000031ec7", - "transactionHash": "0x4e1e9194ca6f9d4e1736e9e441f66104f273548ed6d91b236a5f9c2ea10fa06d", - "transactionIndex": "0x0", - "type": "0x2" - }`, + file: `dynamic-tx-with-logs`, }, // 4. access list tx with create contract { txHash: txHashes[4], - want: `{ - "blockHash": "0x83f2712dfaeab6ab6239bf060bccfc49652e4afdc3b80b22a8373816a2047bd3", - "blockNumber": "0x5", - "contractAddress": "0xfdaa97661a584d977b4d3abb5370766ff5b86a18", - "cumulativeGasUsed": "0xe01a", - "effectiveGasPrice": "0x5d21dba00", - "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0xe01a", - "logs": [], - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "status": "0x1", - "to": null, - "transactionHash": "0x8afe030574f663fe5096371d6f58a6287bfb3e0c73a5050220f5775a08e7abc9", - "transactionIndex": "0x0", - "type": "0x1" - }`, + file: "create-contract-with-access-list", }, // 5. txhash empty { txHash: common.Hash{}, - want: `null`, + file: "txhash-empty", }, // 6. txhash not found { txHash: common.HexToHash("deadbeef"), - want: `null`, + file: "txhash-notfound", + }, + // 7. blob tx + { + txHash: txHashes[5], + file: "blob-tx", }, } @@ -1903,93 +1519,15 @@ func TestRPCGetTransactionReceipt(t *testing.T) { t.Errorf("test %d: want no error, have %v", i, err) continue } - data, err := json.Marshal(result) - if err != nil { - t.Errorf("test %d: json marshal error", i) - continue - } - want, have := tt.want, string(data) - require.JSONEqf(t, want, have, "test %d: json not match, want: %s, have: %s", i, want, have) + testRPCResponseWithFile(t, i, result, "eth_getTransactionReceipt", tt.file) } } -func setupReceiptBackend(t *testing.T, genBlocks int) (*testBackend, []common.Hash) { - // Initialize test accounts - var ( - acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) - acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) - contract = common.HexToAddress("0000000000000000000000000000000000031ec7") - genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{ - acc1Addr: {Balance: big.NewInt(params.Ether)}, - acc2Addr: {Balance: big.NewInt(params.Ether)}, - // // SPDX-License-Identifier: GPL-3.0 - // pragma solidity >=0.7.0 <0.9.0; - // - // contract Token { - // event Transfer(address indexed from, address indexed to, uint256 value); - // function transfer(address to, uint256 value) public returns (bool) { - // emit Transfer(msg.sender, to, value); - // return true; - // } - // } - contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, - }, - } - signer = types.LatestSignerForChainID(params.TestChainConfig.ChainID) - txHashes = make([]common.Hash, genBlocks) - ) - backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { - var ( - tx *types.Transaction - err error - ) - switch i { - case 0: - // transfer 1000wei - tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &acc2Addr, Value: big.NewInt(1000), Gas: params.TxGas, GasPrice: b.BaseFee(), Data: nil}), types.HomesteadSigner{}, acc1Key) - case 1: - // create contract - tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: nil, Gas: 53100, GasPrice: b.BaseFee(), Data: common.FromHex("0x60806040")}), signer, acc1Key) - case 2: - // with logs - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Gas: 60000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, acc1Key) - case 3: - // dynamic fee with logs - // transfer(address to, uint256 value) - data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) - fee := big.NewInt(500) - fee.Add(fee, b.BaseFee()) - tx, err = types.SignTx(types.NewTx(&types.DynamicFeeTx{Nonce: uint64(i), To: &contract, Gas: 60000, Value: big.NewInt(1), GasTipCap: big.NewInt(500), GasFeeCap: fee, Data: common.FromHex(data)}), signer, acc1Key) - case 4: - // access list with contract create - accessList := types.AccessList{{ - Address: contract, - StorageKeys: []common.Hash{{0}}, - }} - tx, err = types.SignTx(types.NewTx(&types.AccessListTx{Nonce: uint64(i), To: nil, Gas: 58100, GasPrice: b.BaseFee(), Data: common.FromHex("0x60806040"), AccessList: accessList}), signer, acc1Key) - } - if err != nil { - t.Errorf("failed to sign tx: %v", err) - } - if tx != nil { - b.AddTx(tx) - txHashes[i] = tx.Hash() - } - }) - return backend, txHashes -} - func TestRPCGetBlockReceipts(t *testing.T) { t.Parallel() var ( - genBlocks = 5 + genBlocks = 6 backend, _ = setupReceiptBackend(t, genBlocks) api = NewBlockChainAPI(backend) ) @@ -2005,62 +1543,67 @@ func TestRPCGetBlockReceipts(t *testing.T) { var testSuite = []struct { test rpc.BlockNumberOrHash - want string + file string }{ // 0. block without any txs(hash) { test: rpc.BlockNumberOrHashWithHash(blockHashes[0], false), - want: `[]`, + file: "number-0", }, // 1. block without any txs(number) { test: rpc.BlockNumberOrHashWithNumber(0), - want: `[]`, + file: "number-1", }, // 2. earliest tag { test: rpc.BlockNumberOrHashWithNumber(rpc.EarliestBlockNumber), - want: `[]`, + file: "tag-earliest", }, // 3. latest tag { test: rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), - want: `[{"blockHash":"0x581862f55d194bd8945820d0d5f28c3a53a72fffa17c0e0376e43478c2e2a75e","blockNumber":"0x5","contractAddress":"0xfdaa97661a584d977b4d3abb5370766ff5b86a18","cumulativeGasUsed":"0xe01c","effectiveGasPrice":"0x5d21dba00","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0xe01c","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":null,"transactionHash":"0x8afe030574f663fe5096371d6f58a6287bfb3e0c73a5050220f5775a08e7abc9","transactionIndex":"0x0","type":"0x1"}]`, + file: "tag-latest", }, // 4. block with legacy transfer tx(hash) { test: rpc.BlockNumberOrHashWithHash(blockHashes[1], false), - want: `[{"blockHash":"0xcc27e155b6eadfa892992a8cd8adaf3c929a6ec6d98c4dfbc60258883c73568e","blockNumber":"0x1","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x5d21dba00","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e","transactionHash":"0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26","transactionIndex":"0x0","type":"0x0"}]`, + file: "block-with-legacy-transfer-tx", }, // 5. block with contract create tx(number) { test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(2)), - want: `[{"blockHash":"0x5dcac7b6d680226ef9429ad15f726aa051d422365adaae65247cec38b42788cd","blockNumber":"0x2","contractAddress":"0xae9bea628c4ce503dcfd7e305cab4e29e7476592","cumulativeGasUsed":"0xcf50","effectiveGasPrice":"0x5d21dba00","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0xcf50","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":null,"transactionHash":"0x22aa617165f83a9f8c191c2b7724ae43eeb1249bee06c98c03c7624c21d27dc8","transactionIndex":"0x0","type":"0x0"}]`, + file: "block-with-contract-create-tx", }, // 6. block with legacy contract call tx(hash) { test: rpc.BlockNumberOrHashWithHash(blockHashes[3], false), - want: `[{"blockHash":"0x2517532d72847621eb37804ae02257e8b3b9bf9668d3e6f7d20dbd24d7dfe863","blockNumber":"0x3","contractAddress":null,"cumulativeGasUsed":"0x5e28","effectiveGasPrice":"0x5d21dba00","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5e28","logs":[{"address":"0x0000000000000000000000000000000000031ec7","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x000000000000000000000000703c4b2bd70c169f5717101caee543299fc946c7","0x0000000000000000000000000000000000000000000000000000000000000003"],"data":"0x000000000000000000000000000000000000000000000000000000000000000d","blockNumber":"0x3","transactionHash":"0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf","transactionIndex":"0x0","blockHash":"0x2517532d72847621eb37804ae02257e8b3b9bf9668d3e6f7d20dbd24d7dfe863","logIndex":"0x0","removed":false}],"logsBloom":"0x00000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000800000000000000008000000000000000000000000000000000020000000080000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000400000000002000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000","status":"0x1","to":"0x0000000000000000000000000000000000031ec7","transactionHash":"0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf","transactionIndex":"0x0","type":"0x0"}]`, + file: "block-with-legacy-contract-call-tx", }, // 7. block with dynamic fee tx(number) { test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(4)), - want: `[{"blockHash":"0xf3ec30848127f600b396b9cf7ead62b3baacbdf0fbf32a419cc4837b362d4e33","blockNumber":"0x4","contractAddress":null,"cumulativeGasUsed":"0x538d","effectiveGasPrice":"0x5d21dbbf4","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x538d","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x0","to":"0x0000000000000000000000000000000000031ec7","transactionHash":"0x4e1e9194ca6f9d4e1736e9e441f66104f273548ed6d91b236a5f9c2ea10fa06d","transactionIndex":"0x0","type":"0x2"}]`, + file: "block-with-dynamic-fee-tx", }, // 8. block is empty { test: rpc.BlockNumberOrHashWithHash(common.Hash{}, false), - want: `null`, + file: "hash-empty", }, // 9. block is not found { test: rpc.BlockNumberOrHashWithHash(common.HexToHash("deadbeef"), false), - want: `null`, + file: "hash-notfound", }, // 10. block is not found { test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(genBlocks + 1)), - want: `null`, + file: "block-notfound", + }, + // 11. block with blob tx + { + test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(6)), + file: "block-with-blob-tx", }, } @@ -2074,12 +1617,23 @@ func TestRPCGetBlockReceipts(t *testing.T) { t.Errorf("test %d: want no error, have %v", i, err) continue } - data, err := json.Marshal(result) - if err != nil { - t.Errorf("test %d: json marshal error", i) - continue - } - want, have := tt.want, string(data) - require.JSONEqf(t, want, have, "test %d: json not match, want: %s, have: %s", i, want, have) + testRPCResponseWithFile(t, i, result, "eth_getBlockReceipts", tt.file) + } +} + +func testRPCResponseWithFile(t *testing.T, testid int, result interface{}, rpc string, file string) { + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + t.Errorf("test %d: json marshal error", testid) + return + } + outputFile := filepath.Join("testdata", fmt.Sprintf("%s-%s.json", rpc, file)) + if os.Getenv("WRITE_TEST_FILES") != "" { + os.WriteFile(outputFile, data, 0644) + } + want, err := os.ReadFile(outputFile) + if err != nil { + t.Fatalf("error reading expected test file: %s output: %v", outputFile, err) } + require.JSONEqf(t, string(want), string(data), "test %d: json not match, want: %s, have: %s", testid, string(want), string(data)) } diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json new file mode 100644 index 0000000000..a21af855b8 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-1.json @@ -0,0 +1,26 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x1", + "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", + "timestamp": "0xa", + "totalDifficulty": "0x1", + "transactions": [ + "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26" + ], + "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-empty-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-empty-fullTx.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-empty-fullTx.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json new file mode 100644 index 0000000000..446f5db6ab --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-genesis.json @@ -0,0 +1,23 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "difficulty": "0x20000", + "extraData": "0x", + "gasLimit": "0x47e7c4", + "gasUsed": "0x0", + "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x201", + "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "timestamp": "0x0", + "totalDifficulty": "0x0", + "transactions": [], + "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json new file mode 100644 index 0000000000..4d54e3f10b --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest-1-fullTx.json @@ -0,0 +1,42 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x9", + "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", + "timestamp": "0x5a", + "totalDifficulty": "0x9", + "transactions": [ + { + "blockHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "blockNumber": "0x9", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gas": "0x5208", + "gasPrice": "0x5d21dba00", + "hash": "0x237f95840187a93f8aaf8d6f1515f8a8ac9d9359fcb0c220cdb3d642d6b9a19a", + "input": "0x", + "nonce": "0x8", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionIndex": "0x0", + "value": "0x3e8", + "type": "0x0", + "v": "0x1c", + "r": "0xd7cdc527490b7ba29c515aae3bbe80c67729cda7f736e6515652cfc40e9da68f", + "s": "0x4d0a4a59bef165b16f910bdadd41efaaad1b73549bacc35eaf6d073eb1fb92b7" + } + ], + "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json new file mode 100644 index 0000000000..7917a2acc7 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-latest.json @@ -0,0 +1,26 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0xa", + "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", + "timestamp": "0x64", + "totalDifficulty": "0xa", + "transactions": [ + "0x71be223424ab6e3457513a760b196d43b094414c32a70ff929b2b720a16b832d" + ], + "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-pending-fullTx.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-pending-fullTx.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-pending-fullTx.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByHash-hash-pending.json b/internal/ethapi/testdata/eth_getBlockByHash-hash-pending.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByHash-hash-pending.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json new file mode 100644 index 0000000000..446f5db6ab --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-0.json @@ -0,0 +1,23 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "difficulty": "0x20000", + "extraData": "0x", + "gasLimit": "0x47e7c4", + "gasUsed": "0x0", + "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x201", + "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "timestamp": "0x0", + "totalDifficulty": "0x0", + "transactions": [], + "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json new file mode 100644 index 0000000000..a21af855b8 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-1.json @@ -0,0 +1,26 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x1", + "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", + "timestamp": "0xa", + "totalDifficulty": "0x1", + "transactions": [ + "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26" + ], + "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest+1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest+1.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest+1.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json new file mode 100644 index 0000000000..4d54e3f10b --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-number-latest-1.json @@ -0,0 +1,42 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x9", + "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", + "timestamp": "0x5a", + "totalDifficulty": "0x9", + "transactions": [ + { + "blockHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "blockNumber": "0x9", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gas": "0x5208", + "gasPrice": "0x5d21dba00", + "hash": "0x237f95840187a93f8aaf8d6f1515f8a8ac9d9359fcb0c220cdb3d642d6b9a19a", + "input": "0x", + "nonce": "0x8", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionIndex": "0x0", + "value": "0x3e8", + "type": "0x0", + "v": "0x1c", + "r": "0xd7cdc527490b7ba29c515aae3bbe80c67729cda7f736e6515652cfc40e9da68f", + "s": "0x4d0a4a59bef165b16f910bdadd41efaaad1b73549bacc35eaf6d073eb1fb92b7" + } + ], + "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json new file mode 100644 index 0000000000..7917a2acc7 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-latest.json @@ -0,0 +1,26 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0xa", + "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2bb", + "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", + "timestamp": "0x64", + "totalDifficulty": "0xa", + "transactions": [ + "0x71be223424ab6e3457513a760b196d43b094414c32a70ff929b2b720a16b832d" + ], + "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01", + "uncles": [] +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending-fullTx.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending.json b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockByNumber-tag-pending.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-notfound.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-notfound.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-notfound.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json new file mode 100644 index 0000000000..552cc3a7f4 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json @@ -0,0 +1,20 @@ +[ + { + "blobGasPrice": "0x1", + "blobGasUsed": "0x20000", + "blockHash": "0x77d5e4ffb415ca53742a37e4bf7bb1522d99dcce1c90e9957eb6d267a0c7e5a1", + "blockNumber": "0x6", + "contractAddress": null, + "cumulativeGasUsed": "0x5208", + "effectiveGasPrice": "0x5d21dba01", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5208", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionHash": "0x7e71344129674f4bbfdaa86313d005a96581993d93ae3a30d81b13fa25579eb2", + "transactionIndex": "0x0", + "type": "0x3" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json new file mode 100644 index 0000000000..8122dbfbf5 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json @@ -0,0 +1,18 @@ +[ + { + "blockHash": "0xa4eb72463332ae8ad98317466a6cc3c61c5ca4d884dd488c44edec166ff1b495", + "blockNumber": "0x2", + "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", + "cumulativeGasUsed": "0xcf50", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0xcf50", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": null, + "transactionHash": "0x22aa617165f83a9f8c191c2b7724ae43eeb1249bee06c98c03c7624c21d27dc8", + "transactionIndex": "0x0", + "type": "0x0" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json new file mode 100644 index 0000000000..1ec360a284 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json @@ -0,0 +1,18 @@ +[ + { + "blockHash": "0x2dcd37090523d8d1ec308edec7d25eced1e26b96c4784f5f362ac58f9b33b2a1", + "blockNumber": "0x4", + "contractAddress": null, + "cumulativeGasUsed": "0x538d", + "effectiveGasPrice": "0x5d21dbbf4", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x538d", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x0", + "to": "0x0000000000000000000000000000000000031ec7", + "transactionHash": "0x4e1e9194ca6f9d4e1736e9e441f66104f273548ed6d91b236a5f9c2ea10fa06d", + "transactionIndex": "0x0", + "type": "0x2" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json new file mode 100644 index 0000000000..02c89dfe10 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json @@ -0,0 +1,34 @@ +[ + { + "blockHash": "0xbeee747a45ccfe51c6da7ac72f585e813a74669528a59ff590d9ce7433938c9d", + "blockNumber": "0x3", + "contractAddress": null, + "cumulativeGasUsed": "0x5e28", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5e28", + "logs": [ + { + "address": "0x0000000000000000000000000000000000031ec7", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000703c4b2bd70c169f5717101caee543299fc946c7", + "0x0000000000000000000000000000000000000000000000000000000000000003" + ], + "data": "0x000000000000000000000000000000000000000000000000000000000000000d", + "blockNumber": "0x3", + "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", + "transactionIndex": "0x0", + "blockHash": "0xbeee747a45ccfe51c6da7ac72f585e813a74669528a59ff590d9ce7433938c9d", + "logIndex": "0x0", + "removed": false + } + ], + "logsBloom": "0x00000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000800000000000000008000000000000000000000000000000000020000000080000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000400000000002000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000", + "status": "0x1", + "to": "0x0000000000000000000000000000000000031ec7", + "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", + "transactionIndex": "0x0", + "type": "0x0" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json new file mode 100644 index 0000000000..32517dd481 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json @@ -0,0 +1,18 @@ +[ + { + "blockHash": "0xe387aa64524bf1712ce0640d1fbdab0992005cfa0d8cd124c526c66a87786359", + "blockNumber": "0x1", + "contractAddress": null, + "cumulativeGasUsed": "0x5208", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5208", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionHash": "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26", + "transactionIndex": "0x0", + "type": "0x0" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-hash-empty.json b/internal/ethapi/testdata/eth_getBlockReceipts-hash-empty.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-hash-empty.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-hash-notfound.json b/internal/ethapi/testdata/eth_getBlockReceipts-hash-notfound.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-hash-notfound.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-number-0.json b/internal/ethapi/testdata/eth_getBlockReceipts-number-0.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-number-0.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-number-1.json b/internal/ethapi/testdata/eth_getBlockReceipts-number-1.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-number-1.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-tag-earliest.json b/internal/ethapi/testdata/eth_getBlockReceipts-tag-earliest.json new file mode 100644 index 0000000000..0637a088a0 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-tag-earliest.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json new file mode 100644 index 0000000000..552cc3a7f4 --- /dev/null +++ b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json @@ -0,0 +1,20 @@ +[ + { + "blobGasPrice": "0x1", + "blobGasUsed": "0x20000", + "blockHash": "0x77d5e4ffb415ca53742a37e4bf7bb1522d99dcce1c90e9957eb6d267a0c7e5a1", + "blockNumber": "0x6", + "contractAddress": null, + "cumulativeGasUsed": "0x5208", + "effectiveGasPrice": "0x5d21dba01", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5208", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionHash": "0x7e71344129674f4bbfdaa86313d005a96581993d93ae3a30d81b13fa25579eb2", + "transactionIndex": "0x0", + "type": "0x3" + } +] \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json new file mode 100644 index 0000000000..eced05ccfc --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-0.json @@ -0,0 +1,20 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "difficulty": "0x20000", + "extraData": "0x", + "gasLimit": "0x47e7c4", + "gasUsed": "0x0", + "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "timestamp": "0x0", + "totalDifficulty": "0x0", + "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json new file mode 100644 index 0000000000..d14b952ff6 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-1.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x1", + "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", + "timestamp": "0xa", + "totalDifficulty": "0x1", + "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-empty.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-empty.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-empty.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json new file mode 100644 index 0000000000..c90f186501 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest-1.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x9", + "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", + "timestamp": "0x5a", + "totalDifficulty": "0x9", + "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json new file mode 100644 index 0000000000..ce691fa435 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-latest.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0xa", + "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", + "timestamp": "0x64", + "totalDifficulty": "0xa", + "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByHash-hash-pending.json b/internal/ethapi/testdata/eth_getHeaderByHash-hash-pending.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByHash-hash-pending.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json new file mode 100644 index 0000000000..eced05ccfc --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-0.json @@ -0,0 +1,20 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "difficulty": "0x20000", + "extraData": "0x", + "gasLimit": "0x47e7c4", + "gasUsed": "0x0", + "hash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x0", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0xfe168c5e9584a85927212e5bea5304bb7d0d8a893453b4b2c52176a72f585ae2", + "timestamp": "0x0", + "totalDifficulty": "0x0", + "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json new file mode 100644 index 0000000000..d14b952ff6 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-1.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0f67ad1fc8052afad4c24551748600c164091cf37e068adef76315025d3c78e7", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x1", + "parentHash": "0x3ead7923676a44500c46ad2192a0fc084aa42063b1703e6866f138a47fb1a9ca", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x6b830601767ac4968163193facbe20123435180e325910b2c50efa21f778c697", + "timestamp": "0xa", + "totalDifficulty": "0x1", + "transactionsRoot": "0x87c65a3f1a98dafe282ace11eaf88b8f31bf41fe6794d401d2f986c1af84bcd5" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest+1.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest+1.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest+1.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json new file mode 100644 index 0000000000..c90f186501 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-number-latest-1.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x9", + "parentHash": "0x2fab5c6892c66668842683ced6b384c2ee83bfd6096a58f451290cabaf57a63e", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x3703d70c6443e809ce035c2a8212dbf9813f6b7d1b0f597766e9023867a852f5", + "timestamp": "0x5a", + "totalDifficulty": "0x9", + "transactionsRoot": "0xe16929d9c7efab0f962c1ed8c1295ddff42d3026779ed1318ea079ca580ee4cb" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json new file mode 100644 index 0000000000..ce691fa435 --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-latest.json @@ -0,0 +1,21 @@ +{ + "baseFeePerGas": "0x5d21dba00", + "blockGasCost": "0x0", + "difficulty": "0x1", + "extraData": "0x0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit": "0x7a1200", + "gasUsed": "0x5208", + "hash": "0x1ec39e7ec46f8df1fb31cfca53fbf71a01869af8bd8f9a1bccbffc16ffa1461d", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x0000000000000000000000000000000000000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0xa", + "parentHash": "0x0583a9d630632001771b4ecc7d62574aec3825aff47e2a680b0ea4ddb79e7365", + "receiptsRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "stateRoot": "0x7e06187d15d50badf60930290fb292ebe43e79553ad8b7d8f1b614316631def7", + "timestamp": "0x64", + "totalDifficulty": "0xa", + "transactionsRoot": "0x69ff8003291e1cd08f75d174f070618f7291e4540b2e33f60b3375743e3fda01" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getHeaderByNumber-tag-pending.json b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-pending.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getHeaderByNumber-tag-pending.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json new file mode 100644 index 0000000000..7f905f1610 --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json @@ -0,0 +1,18 @@ +{ + "blobGasPrice": "0x1", + "blobGasUsed": "0x20000", + "blockHash": "0x77d5e4ffb415ca53742a37e4bf7bb1522d99dcce1c90e9957eb6d267a0c7e5a1", + "blockNumber": "0x6", + "contractAddress": null, + "cumulativeGasUsed": "0x5208", + "effectiveGasPrice": "0x5d21dba01", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5208", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionHash": "0x7e71344129674f4bbfdaa86313d005a96581993d93ae3a30d81b13fa25579eb2", + "transactionIndex": "0x0", + "type": "0x3" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json new file mode 100644 index 0000000000..911fed2357 --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json @@ -0,0 +1,16 @@ +{ + "blockHash": "0xa4eb72463332ae8ad98317466a6cc3c61c5ca4d884dd488c44edec166ff1b495", + "blockNumber": "0x2", + "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", + "cumulativeGasUsed": "0xcf50", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0xcf50", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": null, + "transactionHash": "0x22aa617165f83a9f8c191c2b7724ae43eeb1249bee06c98c03c7624c21d27dc8", + "transactionIndex": "0x0", + "type": "0x0" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json new file mode 100644 index 0000000000..ed4c84c8f2 --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json @@ -0,0 +1,16 @@ +{ + "blockHash": "0x9acfc909e24be0eed69e55e826935be38c9546eea779e0105f465eb6d8d493ad", + "blockNumber": "0x5", + "contractAddress": "0xfdaa97661a584d977b4d3abb5370766ff5b86a18", + "cumulativeGasUsed": "0xe01c", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0xe01c", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": null, + "transactionHash": "0x8afe030574f663fe5096371d6f58a6287bfb3e0c73a5050220f5775a08e7abc9", + "transactionIndex": "0x0", + "type": "0x1" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json new file mode 100644 index 0000000000..51303ecc43 --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json @@ -0,0 +1,16 @@ +{ + "blockHash": "0x2dcd37090523d8d1ec308edec7d25eced1e26b96c4784f5f362ac58f9b33b2a1", + "blockNumber": "0x4", + "contractAddress": null, + "cumulativeGasUsed": "0x538d", + "effectiveGasPrice": "0x5d21dbbf4", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x538d", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x0", + "to": "0x0000000000000000000000000000000000031ec7", + "transactionHash": "0x4e1e9194ca6f9d4e1736e9e441f66104f273548ed6d91b236a5f9c2ea10fa06d", + "transactionIndex": "0x0", + "type": "0x2" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json new file mode 100644 index 0000000000..45fb068ba5 --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json @@ -0,0 +1,16 @@ +{ + "blockHash": "0xe387aa64524bf1712ce0640d1fbdab0992005cfa0d8cd124c526c66a87786359", + "blockNumber": "0x1", + "contractAddress": null, + "cumulativeGasUsed": "0x5208", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5208", + "logs": [], + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", + "transactionHash": "0xdf92bc7c4c0341ecbdcd2a3ca7011fe9e21df4b8553bf0c8caabe6cb4a1aee26", + "transactionIndex": "0x0", + "type": "0x0" +} \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-empty.json b/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-empty.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-empty.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-notfound.json b/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-notfound.json new file mode 100644 index 0000000000..ec747fa47d --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-txhash-notfound.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json new file mode 100644 index 0000000000..2b30d4e44e --- /dev/null +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json @@ -0,0 +1,32 @@ +{ + "blockHash": "0xbeee747a45ccfe51c6da7ac72f585e813a74669528a59ff590d9ce7433938c9d", + "blockNumber": "0x3", + "contractAddress": null, + "cumulativeGasUsed": "0x5e28", + "effectiveGasPrice": "0x5d21dba00", + "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", + "gasUsed": "0x5e28", + "logs": [ + { + "address": "0x0000000000000000000000000000000000031ec7", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x000000000000000000000000703c4b2bd70c169f5717101caee543299fc946c7", + "0x0000000000000000000000000000000000000000000000000000000000000003" + ], + "data": "0x000000000000000000000000000000000000000000000000000000000000000d", + "blockNumber": "0x3", + "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", + "transactionIndex": "0x0", + "blockHash": "0xbeee747a45ccfe51c6da7ac72f585e813a74669528a59ff590d9ce7433938c9d", + "logIndex": "0x0", + "removed": false + } + ], + "logsBloom": "0x00000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000800000000000000008000000000000000000000000000000000020000000080000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000400000000002000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000", + "status": "0x1", + "to": "0x0000000000000000000000000000000000031ec7", + "transactionHash": "0x7366a7738f47e32f5b6d292ca064b6b66f295d3931533a3745975be1191fccdf", + "transactionIndex": "0x0", + "type": "0x0" +} \ No newline at end of file diff --git a/internal/flags/categories.go b/internal/flags/categories.go index d7500157e3..7f631ddb2e 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -32,7 +32,7 @@ const ( EthCategory = "ETHEREUM" LightCategory = "LIGHT CLIENT" DevCategory = "DEVELOPER CHAIN" - EthashCategory = "ETHASH" + StateCategory = "STATE HISTORY MANAGEMENT" TxPoolCategory = "TRANSACTION POOL (EVM)" BlobPoolCategory = "TRANSACTION POOL (BLOB)" PerfCategory = "PERFORMANCE TUNING" diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 9ab6b3e1fd..ed3a14bcf4 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -30,11 +30,13 @@ import ( "encoding" "errors" "flag" + "fmt" "math/big" "os" "os/user" "path/filepath" "strings" + "syscall" "github.com/ethereum/go-ethereum/common/math" "github.com/urfave/cli/v2" @@ -78,6 +80,7 @@ type DirectoryFlag struct { Value DirectoryString Aliases []string + EnvVars []string } // For cli.Flag: @@ -89,6 +92,14 @@ func (f *DirectoryFlag) String() string { return cli.FlagStringer(f) } // Apply called by cli library, grabs variable from environment (if in env) // and adds variable to flag set for parsing. func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { + for _, envVar := range f.EnvVars { + envVar = strings.TrimSpace(envVar) + if value, found := syscall.Getenv(envVar); found { + f.Value.Set(value) + f.HasBeenSet = true + break + } + } eachName(f, func(name string) { set.Var(&f.Value, f.Name, f.Usage) }) @@ -112,7 +123,7 @@ func (f *DirectoryFlag) GetCategory() string { return f.Category } func (f *DirectoryFlag) TakesValue() bool { return true } func (f *DirectoryFlag) GetUsage() string { return f.Usage } func (f *DirectoryFlag) GetValue() string { return f.Value.String() } -func (f *DirectoryFlag) GetEnvVars() []string { return nil } // env not supported +func (f *DirectoryFlag) GetEnvVars() []string { return f.EnvVars } func (f *DirectoryFlag) GetDefaultText() string { if f.DefaultText != "" { @@ -166,6 +177,7 @@ type TextMarshalerFlag struct { Value TextMarshaler Aliases []string + EnvVars []string } // For cli.Flag: @@ -175,6 +187,16 @@ func (f *TextMarshalerFlag) IsSet() bool { return f.HasBeenSet } func (f *TextMarshalerFlag) String() string { return cli.FlagStringer(f) } func (f *TextMarshalerFlag) Apply(set *flag.FlagSet) error { + for _, envVar := range f.EnvVars { + envVar = strings.TrimSpace(envVar) + if value, found := syscall.Getenv(envVar); found { + if err := f.Value.UnmarshalText([]byte(value)); err != nil { + return fmt.Errorf("could not parse %q from environment variable %q for flag %s: %s", value, envVar, f.Name, err) + } + f.HasBeenSet = true + break + } + } eachName(f, func(name string) { set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage) }) @@ -197,7 +219,7 @@ func (f *TextMarshalerFlag) GetCategory() string { return f.Category } func (f *TextMarshalerFlag) TakesValue() bool { return true } func (f *TextMarshalerFlag) GetUsage() string { return f.Usage } -func (f *TextMarshalerFlag) GetEnvVars() []string { return nil } // env not supported +func (f *TextMarshalerFlag) GetEnvVars() []string { return f.EnvVars } func (f *TextMarshalerFlag) GetValue() string { t, err := f.Value.MarshalText() @@ -247,6 +269,7 @@ type BigFlag struct { Value *big.Int Aliases []string + EnvVars []string } // For cli.Flag: @@ -256,6 +279,16 @@ func (f *BigFlag) IsSet() bool { return f.HasBeenSet } func (f *BigFlag) String() string { return cli.FlagStringer(f) } func (f *BigFlag) Apply(set *flag.FlagSet) error { + for _, envVar := range f.EnvVars { + envVar = strings.TrimSpace(envVar) + if value, found := syscall.Getenv(envVar); found { + if _, ok := f.Value.SetString(value, 10); !ok { + return fmt.Errorf("could not parse %q from environment variable %q for flag %s", value, envVar, f.Name) + } + f.HasBeenSet = true + break + } + } eachName(f, func(name string) { f.Value = new(big.Int) set.Var((*bigValue)(f.Value), f.Name, f.Usage) @@ -281,7 +314,7 @@ func (f *BigFlag) GetCategory() string { return f.Category } func (f *BigFlag) TakesValue() bool { return true } func (f *BigFlag) GetUsage() string { return f.Usage } func (f *BigFlag) GetValue() string { return f.Value.String() } -func (f *BigFlag) GetEnvVars() []string { return nil } // env not supported +func (f *BigFlag) GetEnvVars() []string { return f.EnvVars } func (f *BigFlag) GetDefaultText() string { if f.DefaultText != "" { diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index 6241d05ef5..796afc124d 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -28,13 +28,22 @@ package flags import ( "fmt" + "os" + "regexp" + "sort" "strings" "github.com/ava-labs/subnet-evm/internal/version" "github.com/ava-labs/subnet-evm/params" + "github.com/ethereum/go-ethereum/log" + "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" ) +// usecolor defines whether the CLI help should use colored output or normal dumb +// colorless terminal formatting. +var usecolor = (isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())) && os.Getenv("TERM") != "dumb" + // NewApp creates an app with sane defaults. func NewApp(usage string) *cli.App { git, _ := version.VCS() @@ -139,6 +148,14 @@ func doMigrateFlags(ctx *cli.Context) { } func init() { + if usecolor { + // Annotate all help categories with colors + cli.AppHelpTemplate = regexp.MustCompile("[A-Z ]+:").ReplaceAllString(cli.AppHelpTemplate, "\u001B[33m$0\u001B[0m") + + // Annotate flag categories with colors (private template, so need to + // copy-paste the entire thing here...) + cli.AppHelpTemplate = strings.ReplaceAll(cli.AppHelpTemplate, "{{template \"visibleFlagCategoryTemplate\" .}}", "{{range .VisibleFlagCategories}}\n {{if .Name}}\u001B[33m{{.Name}}\u001B[0m\n\n {{end}}{{$flglen := len .Flags}}{{range $i, $e := .Flags}}{{if eq (subtract $flglen $i) 1}}{{$e}}\n{{else}}{{$e}}\n {{end}}{{end}}{{end}}") + } cli.FlagStringer = FlagString } @@ -148,37 +165,31 @@ func FlagString(f cli.Flag) string { if !ok { return "" } - needsPlaceholder := df.TakesValue() placeholder := "" if needsPlaceholder { placeholder = "value" } - namesText := pad(cli.FlagNamePrefixer(df.Names(), placeholder), 30) + namesText := cli.FlagNamePrefixer(df.Names(), placeholder) defaultValueString := "" if s := df.GetDefaultText(); s != "" { defaultValueString = " (default: " + s + ")" } - - usage := strings.TrimSpace(df.GetUsage()) envHint := strings.TrimSpace(cli.FlagEnvHinter(df.GetEnvVars(), "")) - if len(envHint) > 0 { - usage += " " + envHint + if envHint != "" { + envHint = " (" + envHint[1:len(envHint)-1] + ")" } - + usage := strings.TrimSpace(df.GetUsage()) usage = wordWrap(usage, 80) usage = indent(usage, 10) - return fmt.Sprintf("\n %s%s\n%s", namesText, defaultValueString, usage) -} - -func pad(s string, length int) string { - if len(s) < length { - s += strings.Repeat(" ", length-len(s)) + if usecolor { + return fmt.Sprintf("\n \u001B[32m%-35s%-35s\u001B[0m%s\n%s", namesText, defaultValueString, envHint, usage) + } else { + return fmt.Sprintf("\n %-35s%-35s%s\n%s", namesText, defaultValueString, envHint, usage) } - return s } func indent(s string, nspace int) string { @@ -223,3 +234,78 @@ func wordWrap(s string, width int) string { return output.String() } + +// AutoEnvVars extends all the specific CLI flags with automatically generated +// env vars by capitalizing the flag, replacing . with _ and prefixing it with +// the specified string. +// +// Note, the prefix should *not* contain the separator underscore, that will be +// added automatically. +func AutoEnvVars(flags []cli.Flag, prefix string) { + for _, flag := range flags { + envvar := strings.ToUpper(prefix + "_" + strings.ReplaceAll(strings.ReplaceAll(flag.Names()[0], ".", "_"), "-", "_")) + + switch flag := flag.(type) { + case *cli.StringFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *cli.BoolFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *cli.IntFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *cli.Uint64Flag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *cli.DurationFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *cli.PathFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *BigFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *TextMarshalerFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + + case *DirectoryFlag: + flag.EnvVars = append(flag.EnvVars, envvar) + } + } +} + +// CheckEnvVars iterates over all the environment variables and checks if any of +// them look like a CLI flag but is not consumed. This can be used to detect old +// or mistyped names. +func CheckEnvVars(ctx *cli.Context, flags []cli.Flag, prefix string) { + known := make(map[string]string) + for _, flag := range flags { + docflag, ok := flag.(cli.DocGenerationFlag) + if !ok { + continue + } + for _, envvar := range docflag.GetEnvVars() { + known[envvar] = flag.Names()[0] + } + } + keyvals := os.Environ() + sort.Strings(keyvals) + + for _, keyval := range keyvals { + key := strings.Split(keyval, "=")[0] + if !strings.HasPrefix(key, prefix) { + continue + } + if flag, ok := known[key]; ok { + if ctx.Count(flag) > 0 { + log.Info("Config environment variable found", "envvar", key, "shadowedby", "--"+flag) + } else { + log.Info("Config environment variable found", "envvar", key) + } + } else { + log.Warn("Unknown config environment variable", "envvar", key) + } + } +} diff --git a/log/format.go b/log/format.go new file mode 100644 index 0000000000..669911acdd --- /dev/null +++ b/log/format.go @@ -0,0 +1,569 @@ +// (c) 2024, Ava Labs, Inc. +// +// This file is a derived work, based on the go-ethereum library whose original +// notices appear below. +// +// It is distributed under a license compatible with the licensing terms of the +// original code from which it is derived. +// +// Much love to the original authors for their work. +// ********** + +package log + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/ethereum/go-ethereum/log" + "github.com/holiman/uint256" +) + +const errorKey = "LOG15_ERROR" + +const ( + timeFormat = "2006-01-02T15:04:05-0700" + termTimeFormat = "01-02|15:04:05.000" + floatFormat = 'f' + termMsgJust = 40 + termCtxMaxPadding = 40 +) + +// locationTrims are trimmed for display to avoid unwieldy log lines. +var locationTrims = []string{ + "github.com/ava-labs/", +} + +// PrintOrigins sets or unsets log location (file:line) printing for terminal +// format output. +func PrintOrigins(print bool) { + locationEnabled.Store(print) + if print { + stackEnabled.Store(true) + } +} + +// stackEnabled is an atomic flag controlling whether the log handler needs +// to store the callsite stack. This is needed in case any handler wants to +// print locations (locationEnabled), use vmodule, or print full stacks (BacktraceAt). +var stackEnabled atomic.Bool + +// locationEnabled is an atomic flag controlling whether the terminal formatter +// should append the log locations too when printing entries. +var locationEnabled atomic.Bool + +// locationLength is the maxmimum path length encountered, which all logs are +// padded to to aid in alignment. +var locationLength atomic.Uint32 + +// fieldPadding is a global map with maximum field value lengths seen until now +// to allow padding log contexts in a bit smarter way. +var fieldPadding = make(map[string]int) + +// fieldPaddingLock is a global mutex protecting the field padding map. +var fieldPaddingLock sync.RWMutex + +type Format interface { + Format(r *log.Record) []byte +} + +// FormatFunc returns a new Format object which uses +// the given function to perform log.Record formatting. +func FormatFunc(f func(*log.Record) []byte) Format { + return formatFunc(f) +} + +type formatFunc func(*log.Record) []byte + +func (f formatFunc) Format(r *log.Record) []byte { + return f(r) +} + +// TerminalStringer is an analogous interface to the stdlib stringer, allowing +// own types to have custom shortened serialization formats when printed to the +// screen. +type TerminalStringer interface { + TerminalString() string +} + +// TerminalFormat formats log log.Records optimized for human readability on +// a terminal with color-coded level output and terser human friendly timestamp. +// This format should only be used for interactive programs or while developing. +// +// [LEVEL] [TIME] MESSAGE key=value key=value ... +// +// Example: +// +// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 +func TerminalFormat(usecolor bool) Format { + return FormatFunc(func(r *log.Record) []byte { + msg := escapeMessage(r.Msg) + color := 0 + if usecolor { + switch r.Lvl { + case log.LvlCrit: + color = 35 + case log.LvlError: + color = 31 + case log.LvlWarn: + color = 33 + case log.LvlInfo: + color = 32 + case log.LvlDebug: + color = 36 + case log.LvlTrace: + color = 34 + } + } + + b := &bytes.Buffer{} + lvl := r.Lvl.AlignedString() + if locationEnabled.Load() { + // Log origin printing was requested, format the location path and line number + location := fmt.Sprintf("%+v", r.Call) + for _, prefix := range locationTrims { + location = strings.TrimPrefix(location, prefix) + } + // Maintain the maximum location length for fancyer alignment + align := int(locationLength.Load()) + if align < len(location) { + align = len(location) + locationLength.Store(uint32(align)) + } + padding := strings.Repeat(" ", align-len(location)) + + // Assemble and print the log heading + if color > 0 { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s|%s]%s %s ", color, lvl, r.Time.Format(termTimeFormat), location, padding, msg) + } else { + fmt.Fprintf(b, "%s[%s|%s]%s %s ", lvl, r.Time.Format(termTimeFormat), location, padding, msg) + } + } else { + if color > 0 { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", color, lvl, r.Time.Format(termTimeFormat), msg) + } else { + fmt.Fprintf(b, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), msg) + } + } + // try to justify the log output for short messages + length := utf8.RuneCountInString(msg) + if len(r.Ctx) > 0 && length < termMsgJust { + b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length)) + } + // print the keys logfmt style + logfmt(b, r.Ctx, color, true) + return b.Bytes() + }) +} + +// LogfmtFormat prints log.Records in logfmt format, an easy machine-parseable but human-readable +// format for key/value pairs. +// +// For more details see: http://godoc.org/github.com/kr/logfmt +func LogfmtFormat() Format { + return FormatFunc(func(r *log.Record) []byte { + common := []interface{}{r.KeyNames.Time, r.Time, r.KeyNames.Lvl, r.Lvl, r.KeyNames.Msg, r.Msg} + buf := &bytes.Buffer{} + logfmt(buf, append(common, r.Ctx...), 0, false) + return buf.Bytes() + }) +} + +func logfmt(buf *bytes.Buffer, ctx []interface{}, color int, term bool) { + for i := 0; i < len(ctx); i += 2 { + if i != 0 { + buf.WriteByte(' ') + } + + k, ok := ctx[i].(string) + v := formatLogfmtValue(ctx[i+1], term) + if !ok { + k, v = errorKey, fmt.Sprintf("%+T is not a string key", ctx[i]) + } else { + k = escapeString(k) + } + + // XXX: we should probably check that all of your key bytes aren't invalid + fieldPaddingLock.RLock() + padding := fieldPadding[k] + fieldPaddingLock.RUnlock() + + length := utf8.RuneCountInString(v) + if padding < length && length <= termCtxMaxPadding { + padding = length + + fieldPaddingLock.Lock() + fieldPadding[k] = padding + fieldPaddingLock.Unlock() + } + if color > 0 { + fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=", color, k) + } else { + buf.WriteString(k) + buf.WriteByte('=') + } + buf.WriteString(v) + if i < len(ctx)-2 && padding > length { + buf.Write(bytes.Repeat([]byte{' '}, padding-length)) + } + } + buf.WriteByte('\n') +} + +// JSONFormat formats log log.Records as JSON objects separated by newlines. +// It is the equivalent of JSONFormatEx(false, true). +func JSONFormat() Format { + return JSONFormatEx(false, true) +} + +// JSONFormatOrderedEx formats log log.Records as JSON arrays. If pretty is true, +// log.Records will be pretty-printed. If lineSeparated is true, log.Records +// will be logged with a new line between each log.Record. +func JSONFormatOrderedEx(pretty, lineSeparated bool) Format { + jsonMarshal := json.Marshal + if pretty { + jsonMarshal = func(v interface{}) ([]byte, error) { + return json.MarshalIndent(v, "", " ") + } + } + return FormatFunc(func(r *log.Record) []byte { + props := map[string]interface{}{ + r.KeyNames.Time: r.Time, + r.KeyNames.Lvl: r.Lvl.String(), + r.KeyNames.Msg: r.Msg, + } + + ctx := make([]string, len(r.Ctx)) + for i := 0; i < len(r.Ctx); i += 2 { + if k, ok := r.Ctx[i].(string); ok { + ctx[i] = k + ctx[i+1] = formatLogfmtValue(r.Ctx[i+1], true) + } else { + props[errorKey] = fmt.Sprintf("%+T is not a string key,", r.Ctx[i]) + } + } + props[r.KeyNames.Ctx] = ctx + + b, err := jsonMarshal(props) + if err != nil { + b, _ = jsonMarshal(map[string]string{ + errorKey: err.Error(), + }) + return b + } + if lineSeparated { + b = append(b, '\n') + } + return b + }) +} + +// JSONFormatEx formats log log.Records as JSON objects. If pretty is true, +// log.Records will be pretty-printed. If lineSeparated is true, log.Records +// will be logged with a new line between each log.Record. +func JSONFormatEx(pretty, lineSeparated bool) Format { + jsonMarshal := json.Marshal + if pretty { + jsonMarshal = func(v interface{}) ([]byte, error) { + return json.MarshalIndent(v, "", " ") + } + } + + return FormatFunc(func(r *log.Record) []byte { + props := map[string]interface{}{ + r.KeyNames.Time: r.Time, + r.KeyNames.Lvl: r.Lvl.String(), + r.KeyNames.Msg: r.Msg, + } + + for i := 0; i < len(r.Ctx); i += 2 { + k, ok := r.Ctx[i].(string) + if !ok { + props[errorKey] = fmt.Sprintf("%+T is not a string key", r.Ctx[i]) + } else { + props[k] = formatJSONValue(r.Ctx[i+1]) + } + } + + b, err := jsonMarshal(props) + if err != nil { + b, _ = jsonMarshal(map[string]string{ + errorKey: err.Error(), + }) + return b + } + + if lineSeparated { + b = append(b, '\n') + } + + return b + }) +} + +func formatShared(value interface{}) (result interface{}) { + defer func() { + if err := recover(); err != nil { + if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() { + result = "nil" + } else { + panic(err) + } + } + }() + + switch v := value.(type) { + case time.Time: + return v.Format(timeFormat) + + case error: + return v.Error() + + case fmt.Stringer: + return v.String() + + default: + return v + } +} + +func formatJSONValue(value interface{}) interface{} { + value = formatShared(value) + switch value.(type) { + case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64, string: + return value + default: + return fmt.Sprintf("%+v", value) + } +} + +// formatValue formats a value for serialization +func formatLogfmtValue(value interface{}, term bool) string { + if value == nil { + return "nil" + } + + switch v := value.(type) { + case time.Time: + // Performance optimization: No need for escaping since the provided + // timeFormat doesn't have any escape characters, and escaping is + // expensive. + return v.Format(timeFormat) + + case *big.Int: + // Big ints get consumed by the Stringer clause, so we need to handle + // them earlier on. + if v == nil { + return "" + } + return formatLogfmtBigInt(v) + + case *uint256.Int: + // Uint256s get consumed by the Stringer clause, so we need to handle + // them earlier on. + if v == nil { + return "" + } + return formatLogfmtUint256(v) + } + if term { + if s, ok := value.(TerminalStringer); ok { + // Custom terminal stringer provided, use that + return escapeString(s.TerminalString()) + } + } + value = formatShared(value) + switch v := value.(type) { + case bool: + return strconv.FormatBool(v) + case float32: + return strconv.FormatFloat(float64(v), floatFormat, 3, 64) + case float64: + return strconv.FormatFloat(v, floatFormat, 3, 64) + case int8: + return strconv.FormatInt(int64(v), 10) + case uint8: + return strconv.FormatInt(int64(v), 10) + case int16: + return strconv.FormatInt(int64(v), 10) + case uint16: + return strconv.FormatInt(int64(v), 10) + // Larger integers get thousands separators. + case int: + return FormatLogfmtInt64(int64(v)) + case int32: + return FormatLogfmtInt64(int64(v)) + case int64: + return FormatLogfmtInt64(v) + case uint: + return FormatLogfmtUint64(uint64(v)) + case uint32: + return FormatLogfmtUint64(uint64(v)) + case uint64: + return FormatLogfmtUint64(v) + case string: + return escapeString(v) + default: + return escapeString(fmt.Sprintf("%+v", value)) + } +} + +// FormatLogfmtInt64 formats n with thousand separators. +func FormatLogfmtInt64(n int64) string { + if n < 0 { + return formatLogfmtUint64(uint64(-n), true) + } + return formatLogfmtUint64(uint64(n), false) +} + +// FormatLogfmtUint64 formats n with thousand separators. +func FormatLogfmtUint64(n uint64) string { + return formatLogfmtUint64(n, false) +} + +func formatLogfmtUint64(n uint64, neg bool) string { + // Small numbers are fine as is + if n < 100000 { + if neg { + return strconv.Itoa(-int(n)) + } else { + return strconv.Itoa(int(n)) + } + } + // Large numbers should be split + const maxLength = 26 + + var ( + out = make([]byte, maxLength) + i = maxLength - 1 + comma = 0 + ) + for ; n > 0; i-- { + if comma == 3 { + comma = 0 + out[i] = ',' + } else { + comma++ + out[i] = '0' + byte(n%10) + n /= 10 + } + } + if neg { + out[i] = '-' + i-- + } + return string(out[i+1:]) +} + +// formatLogfmtBigInt formats n with thousand separators. +func formatLogfmtBigInt(n *big.Int) string { + if n.IsUint64() { + return FormatLogfmtUint64(n.Uint64()) + } + if n.IsInt64() { + return FormatLogfmtInt64(n.Int64()) + } + + var ( + text = n.String() + buf = make([]byte, len(text)+len(text)/3) + comma = 0 + i = len(buf) - 1 + ) + for j := len(text) - 1; j >= 0; j, i = j-1, i-1 { + c := text[j] + + switch { + case c == '-': + buf[i] = c + case comma == 3: + buf[i] = ',' + i-- + comma = 0 + fallthrough + default: + buf[i] = c + comma++ + } + } + return string(buf[i+1:]) +} + +// formatLogfmtUint256 formats n with thousand separators. +func formatLogfmtUint256(n *uint256.Int) string { + if n.IsUint64() { + return FormatLogfmtUint64(n.Uint64()) + } + var ( + text = n.Dec() + buf = make([]byte, len(text)+len(text)/3) + comma = 0 + i = len(buf) - 1 + ) + for j := len(text) - 1; j >= 0; j, i = j-1, i-1 { + c := text[j] + + switch { + case c == '-': + buf[i] = c + case comma == 3: + buf[i] = ',' + i-- + comma = 0 + fallthrough + default: + buf[i] = c + comma++ + } + } + return string(buf[i+1:]) +} + +// escapeString checks if the provided string needs escaping/quoting, and +// calls strconv.Quote if needed +func escapeString(s string) string { + needsQuoting := false + for _, r := range s { + // We quote everything below " (0x22) and above~ (0x7E), plus equal-sign + if r <= '"' || r > '~' || r == '=' { + needsQuoting = true + break + } + } + if !needsQuoting { + return s + } + return strconv.Quote(s) +} + +// escapeMessage checks if the provided string needs escaping/quoting, similarly +// to escapeString. The difference is that this method is more lenient: it allows +// for spaces and linebreaks to occur without needing quoting. +func escapeMessage(s string) string { + needsQuoting := false + for _, r := range s { + // Allow CR/LF/TAB. This is to make multi-line messages work. + if r == '\r' || r == '\n' || r == '\t' { + continue + } + // We quote everything below (0x20) and above~ (0x7E), + // plus equal-sign + if r < ' ' || r > '~' || r == '=' { + needsQuoting = true + break + } + } + if !needsQuoting { + return s + } + return strconv.Quote(s) +} diff --git a/log/format_extra.go b/log/format_extra.go new file mode 100644 index 0000000000..663b3796a4 --- /dev/null +++ b/log/format_extra.go @@ -0,0 +1,71 @@ +// (c) 2024 Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package log + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + "unicode/utf8" + + "github.com/ethereum/go-ethereum/log" +) + +func SubnetEVMTermFormat(alias string) log.Format { + prefix := fmt.Sprintf("<%s Chain>", alias) + return log.FormatFunc(func(r *log.Record) []byte { + msg := escapeMessage(r.Msg) + b := &bytes.Buffer{} + lvl := r.Lvl.AlignedString() + + location := fmt.Sprintf("%+v", r.Call) + for _, prefix := range locationTrims { + location = strings.TrimPrefix(location, prefix) + } + + fmt.Fprintf(b, "[%s] %s %s %s %s ", r.Time.Format(termTimeFormat), lvl, prefix, location, msg) + // try to justify the log output for short messages + length := utf8.RuneCountInString(msg) + if len(r.Ctx) > 0 && length < termMsgJust { + b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length)) + } + // print the keys logfmt style + logfmt(b, r.Ctx, 0, true) + return b.Bytes() + }) +} + +func SubnetEVMJSONFormat(alias string) log.Format { + prefix := fmt.Sprintf("%s Chain", alias) + return log.FormatFunc(func(r *log.Record) []byte { + props := make(map[string]interface{}, 5+len(r.Ctx)/2) + props["timestamp"] = r.Time + props["level"] = r.Lvl.String() + props[r.KeyNames.Msg] = r.Msg + props["logger"] = prefix + props["caller"] = fmt.Sprintf("%+v", r.Call) + for i := 0; i < len(r.Ctx); i += 2 { + k, ok := r.Ctx[i].(string) + if !ok { + props[errorKey] = fmt.Sprintf("%+v is not a string key", r.Ctx[i]) + } else { + // The number of arguments is normalized from the geth logger + // to ensure that this will not cause an index out of bounds error + props[k] = formatJSONValue(r.Ctx[i+1]) + } + } + + b, err := json.Marshal(props) + if err != nil { + b, _ = json.Marshal(map[string]string{ + errorKey: err.Error(), + }) + return b + } + + b = append(b, '\n') + return b + }) +} diff --git a/metrics/counter.go b/metrics/counter.go index 55e1c59540..cb81599c21 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -4,13 +4,16 @@ import ( "sync/atomic" ) +type CounterSnapshot interface { + Count() int64 +} + // Counters hold an int64 value that can be incremented and decremented. type Counter interface { Clear() - Count() int64 Dec(int64) Inc(int64) - Snapshot() Counter + Snapshot() CounterSnapshot } // GetOrRegisterCounter returns an existing Counter or constructs and registers @@ -38,13 +41,13 @@ func NewCounter() Counter { if !Enabled { return NilCounter{} } - return &StandardCounter{} + return new(StandardCounter) } // NewCounterForced constructs a new StandardCounter and returns it no matter if // the global switch is enabled or not. func NewCounterForced() Counter { - return &StandardCounter{} + return new(StandardCounter) } // NewRegisteredCounter constructs and registers a new StandardCounter. @@ -70,75 +73,40 @@ func NewRegisteredCounterForced(name string, r Registry) Counter { return c } -// CounterSnapshot is a read-only copy of another Counter. -type CounterSnapshot int64 - -// Clear panics. -func (CounterSnapshot) Clear() { - panic("Clear called on a CounterSnapshot") -} +// counterSnapshot is a read-only copy of another Counter. +type counterSnapshot int64 // Count returns the count at the time the snapshot was taken. -func (c CounterSnapshot) Count() int64 { return int64(c) } - -// Dec panics. -func (CounterSnapshot) Dec(int64) { - panic("Dec called on a CounterSnapshot") -} - -// Inc panics. -func (CounterSnapshot) Inc(int64) { - panic("Inc called on a CounterSnapshot") -} - -// Snapshot returns the snapshot. -func (c CounterSnapshot) Snapshot() Counter { return c } +func (c counterSnapshot) Count() int64 { return int64(c) } // NilCounter is a no-op Counter. type NilCounter struct{} -// Clear is a no-op. -func (NilCounter) Clear() {} - -// Count is a no-op. -func (NilCounter) Count() int64 { return 0 } - -// Dec is a no-op. -func (NilCounter) Dec(i int64) {} - -// Inc is a no-op. -func (NilCounter) Inc(i int64) {} - -// Snapshot is a no-op. -func (NilCounter) Snapshot() Counter { return NilCounter{} } +func (NilCounter) Clear() {} +func (NilCounter) Dec(i int64) {} +func (NilCounter) Inc(i int64) {} +func (NilCounter) Snapshot() CounterSnapshot { return (*emptySnapshot)(nil) } // StandardCounter is the standard implementation of a Counter and uses the // sync/atomic package to manage a single int64 value. -type StandardCounter struct { - count atomic.Int64 -} +type StandardCounter atomic.Int64 // Clear sets the counter to zero. func (c *StandardCounter) Clear() { - c.count.Store(0) -} - -// Count returns the current count. -func (c *StandardCounter) Count() int64 { - return c.count.Load() + (*atomic.Int64)(c).Store(0) } // Dec decrements the counter by the given amount. func (c *StandardCounter) Dec(i int64) { - c.count.Add(-i) + (*atomic.Int64)(c).Add(-i) } // Inc increments the counter by the given amount. func (c *StandardCounter) Inc(i int64) { - c.count.Add(i) + (*atomic.Int64)(c).Add(i) } // Snapshot returns a read-only copy of the counter. -func (c *StandardCounter) Snapshot() Counter { - return CounterSnapshot(c.Count()) +func (c *StandardCounter) Snapshot() CounterSnapshot { + return counterSnapshot((*atomic.Int64)(c).Load()) } diff --git a/metrics/counter_float64.go b/metrics/counter_float64.go index d1197bb8e0..15c81494ef 100644 --- a/metrics/counter_float64.go +++ b/metrics/counter_float64.go @@ -5,13 +5,16 @@ import ( "sync/atomic" ) +type CounterFloat64Snapshot interface { + Count() float64 +} + // CounterFloat64 holds a float64 value that can be incremented and decremented. type CounterFloat64 interface { Clear() - Count() float64 Dec(float64) Inc(float64) - Snapshot() CounterFloat64 + Snapshot() CounterFloat64Snapshot } // GetOrRegisterCounterFloat64 returns an existing CounterFloat64 or constructs and registers @@ -71,47 +74,19 @@ func NewRegisteredCounterFloat64Forced(name string, r Registry) CounterFloat64 { return c } -// CounterFloat64Snapshot is a read-only copy of another CounterFloat64. -type CounterFloat64Snapshot float64 - -// Clear panics. -func (CounterFloat64Snapshot) Clear() { - panic("Clear called on a CounterFloat64Snapshot") -} +// counterFloat64Snapshot is a read-only copy of another CounterFloat64. +type counterFloat64Snapshot float64 // Count returns the value at the time the snapshot was taken. -func (c CounterFloat64Snapshot) Count() float64 { return float64(c) } - -// Dec panics. -func (CounterFloat64Snapshot) Dec(float64) { - panic("Dec called on a CounterFloat64Snapshot") -} +func (c counterFloat64Snapshot) Count() float64 { return float64(c) } -// Inc panics. -func (CounterFloat64Snapshot) Inc(float64) { - panic("Inc called on a CounterFloat64Snapshot") -} - -// Snapshot returns the snapshot. -func (c CounterFloat64Snapshot) Snapshot() CounterFloat64 { return c } - -// NilCounterFloat64 is a no-op CounterFloat64. type NilCounterFloat64 struct{} -// Clear is a no-op. -func (NilCounterFloat64) Clear() {} - -// Count is a no-op. -func (NilCounterFloat64) Count() float64 { return 0.0 } - -// Dec is a no-op. -func (NilCounterFloat64) Dec(i float64) {} - -// Inc is a no-op. -func (NilCounterFloat64) Inc(i float64) {} - -// Snapshot is a no-op. -func (NilCounterFloat64) Snapshot() CounterFloat64 { return NilCounterFloat64{} } +func (NilCounterFloat64) Clear() {} +func (NilCounterFloat64) Count() float64 { return 0.0 } +func (NilCounterFloat64) Dec(i float64) {} +func (NilCounterFloat64) Inc(i float64) {} +func (NilCounterFloat64) Snapshot() CounterFloat64Snapshot { return NilCounterFloat64{} } // StandardCounterFloat64 is the standard implementation of a CounterFloat64 and uses the // atomic to manage a single float64 value. @@ -124,11 +99,6 @@ func (c *StandardCounterFloat64) Clear() { c.floatBits.Store(0) } -// Count returns the current value. -func (c *StandardCounterFloat64) Count() float64 { - return math.Float64frombits(c.floatBits.Load()) -} - // Dec decrements the counter by the given amount. func (c *StandardCounterFloat64) Dec(v float64) { atomicAddFloat(&c.floatBits, -v) @@ -140,8 +110,9 @@ func (c *StandardCounterFloat64) Inc(v float64) { } // Snapshot returns a read-only copy of the counter. -func (c *StandardCounterFloat64) Snapshot() CounterFloat64 { - return CounterFloat64Snapshot(c.Count()) +func (c *StandardCounterFloat64) Snapshot() CounterFloat64Snapshot { + v := math.Float64frombits(c.floatBits.Load()) + return counterFloat64Snapshot(v) } func atomicAddFloat(fbits *atomic.Uint64, v float64) { diff --git a/metrics/counter_float_64_test.go b/metrics/counter_float_64_test.go index f17aca330c..c21bd3307f 100644 --- a/metrics/counter_float_64_test.go +++ b/metrics/counter_float_64_test.go @@ -27,7 +27,7 @@ func BenchmarkCounterFloat64Parallel(b *testing.B) { }() } wg.Wait() - if have, want := c.Count(), 10.0*float64(b.N); have != want { + if have, want := c.Snapshot().Count(), 10.0*float64(b.N); have != want { b.Fatalf("have %f want %f", have, want) } } @@ -36,7 +36,7 @@ func TestCounterFloat64Clear(t *testing.T) { c := NewCounterFloat64() c.Inc(1.0) c.Clear() - if count := c.Count(); count != 0 { + if count := c.Snapshot().Count(); count != 0 { t.Errorf("c.Count(): 0 != %v\n", count) } } @@ -44,7 +44,7 @@ func TestCounterFloat64Clear(t *testing.T) { func TestCounterFloat64Dec1(t *testing.T) { c := NewCounterFloat64() c.Dec(1.0) - if count := c.Count(); count != -1.0 { + if count := c.Snapshot().Count(); count != -1.0 { t.Errorf("c.Count(): -1.0 != %v\n", count) } } @@ -52,7 +52,7 @@ func TestCounterFloat64Dec1(t *testing.T) { func TestCounterFloat64Dec2(t *testing.T) { c := NewCounterFloat64() c.Dec(2.0) - if count := c.Count(); count != -2.0 { + if count := c.Snapshot().Count(); count != -2.0 { t.Errorf("c.Count(): -2.0 != %v\n", count) } } @@ -60,7 +60,7 @@ func TestCounterFloat64Dec2(t *testing.T) { func TestCounterFloat64Inc1(t *testing.T) { c := NewCounterFloat64() c.Inc(1.0) - if count := c.Count(); count != 1.0 { + if count := c.Snapshot().Count(); count != 1.0 { t.Errorf("c.Count(): 1.0 != %v\n", count) } } @@ -68,7 +68,7 @@ func TestCounterFloat64Inc1(t *testing.T) { func TestCounterFloat64Inc2(t *testing.T) { c := NewCounterFloat64() c.Inc(2.0) - if count := c.Count(); count != 2.0 { + if count := c.Snapshot().Count(); count != 2.0 { t.Errorf("c.Count(): 2.0 != %v\n", count) } } @@ -85,7 +85,7 @@ func TestCounterFloat64Snapshot(t *testing.T) { func TestCounterFloat64Zero(t *testing.T) { c := NewCounterFloat64() - if count := c.Count(); count != 0 { + if count := c.Snapshot().Count(); count != 0 { t.Errorf("c.Count(): 0 != %v\n", count) } } @@ -93,7 +93,7 @@ func TestCounterFloat64Zero(t *testing.T) { func TestGetOrRegisterCounterFloat64(t *testing.T) { r := NewRegistry() NewRegisteredCounterFloat64("foo", r).Inc(47.0) - if c := GetOrRegisterCounterFloat64("foo", r); c.Count() != 47.0 { + if c := GetOrRegisterCounterFloat64("foo", r).Snapshot(); c.Count() != 47.0 { t.Fatal(c) } } diff --git a/metrics/counter_test.go b/metrics/counter_test.go index af26ef1548..1b15b23f21 100644 --- a/metrics/counter_test.go +++ b/metrics/counter_test.go @@ -14,7 +14,7 @@ func TestCounterClear(t *testing.T) { c := NewCounter() c.Inc(1) c.Clear() - if count := c.Count(); count != 0 { + if count := c.Snapshot().Count(); count != 0 { t.Errorf("c.Count(): 0 != %v\n", count) } } @@ -22,7 +22,7 @@ func TestCounterClear(t *testing.T) { func TestCounterDec1(t *testing.T) { c := NewCounter() c.Dec(1) - if count := c.Count(); count != -1 { + if count := c.Snapshot().Count(); count != -1 { t.Errorf("c.Count(): -1 != %v\n", count) } } @@ -30,7 +30,7 @@ func TestCounterDec1(t *testing.T) { func TestCounterDec2(t *testing.T) { c := NewCounter() c.Dec(2) - if count := c.Count(); count != -2 { + if count := c.Snapshot().Count(); count != -2 { t.Errorf("c.Count(): -2 != %v\n", count) } } @@ -38,7 +38,7 @@ func TestCounterDec2(t *testing.T) { func TestCounterInc1(t *testing.T) { c := NewCounter() c.Inc(1) - if count := c.Count(); count != 1 { + if count := c.Snapshot().Count(); count != 1 { t.Errorf("c.Count(): 1 != %v\n", count) } } @@ -46,7 +46,7 @@ func TestCounterInc1(t *testing.T) { func TestCounterInc2(t *testing.T) { c := NewCounter() c.Inc(2) - if count := c.Count(); count != 2 { + if count := c.Snapshot().Count(); count != 2 { t.Errorf("c.Count(): 2 != %v\n", count) } } @@ -63,7 +63,7 @@ func TestCounterSnapshot(t *testing.T) { func TestCounterZero(t *testing.T) { c := NewCounter() - if count := c.Count(); count != 0 { + if count := c.Snapshot().Count(); count != 0 { t.Errorf("c.Count(): 0 != %v\n", count) } } @@ -71,7 +71,7 @@ func TestCounterZero(t *testing.T) { func TestGetOrRegisterCounter(t *testing.T) { r := NewRegistry() NewRegisteredCounter("foo", r).Inc(47) - if c := GetOrRegisterCounter("foo", r); c.Count() != 47 { + if c := GetOrRegisterCounter("foo", r).Snapshot(); c.Count() != 47 { t.Fatal(c) } } diff --git a/metrics/doc.go b/metrics/doc.go deleted file mode 100644 index 13f429c168..0000000000 --- a/metrics/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -package metrics - -const epsilon = 0.0000000000000001 -const epsilonPercentile = .00000000001 diff --git a/metrics/ewma.go b/metrics/ewma.go index ed95cba19b..1d7a4f00cf 100644 --- a/metrics/ewma.go +++ b/metrics/ewma.go @@ -7,11 +7,14 @@ import ( "time" ) +type EWMASnapshot interface { + Rate() float64 +} + // EWMAs continuously calculate an exponentially-weighted moving average // based on an outside source of clock ticks. type EWMA interface { - Rate() float64 - Snapshot() EWMA + Snapshot() EWMASnapshot Tick() Update(int64) } @@ -36,40 +39,19 @@ func NewEWMA15() EWMA { return NewEWMA(1 - math.Exp(-5.0/60.0/15)) } -// EWMASnapshot is a read-only copy of another EWMA. -type EWMASnapshot float64 +// ewmaSnapshot is a read-only copy of another EWMA. +type ewmaSnapshot float64 // Rate returns the rate of events per second at the time the snapshot was // taken. -func (a EWMASnapshot) Rate() float64 { return float64(a) } - -// Snapshot returns the snapshot. -func (a EWMASnapshot) Snapshot() EWMA { return a } - -// Tick panics. -func (EWMASnapshot) Tick() { - panic("Tick called on an EWMASnapshot") -} - -// Update panics. -func (EWMASnapshot) Update(int64) { - panic("Update called on an EWMASnapshot") -} +func (a ewmaSnapshot) Rate() float64 { return float64(a) } // NilEWMA is a no-op EWMA. type NilEWMA struct{} -// Rate is a no-op. -func (NilEWMA) Rate() float64 { return 0.0 } - -// Snapshot is a no-op. -func (NilEWMA) Snapshot() EWMA { return NilEWMA{} } - -// Tick is a no-op. -func (NilEWMA) Tick() {} - -// Update is a no-op. -func (NilEWMA) Update(n int64) {} +func (NilEWMA) Snapshot() EWMASnapshot { return (*emptySnapshot)(nil) } +func (NilEWMA) Tick() {} +func (NilEWMA) Update(n int64) {} // StandardEWMA is the standard implementation of an EWMA and tracks the number // of uncounted events and processes them on each tick. It uses the @@ -77,37 +59,50 @@ func (NilEWMA) Update(n int64) {} type StandardEWMA struct { uncounted atomic.Int64 alpha float64 - rate float64 - init bool + rate atomic.Uint64 + init atomic.Bool mutex sync.Mutex } -// Rate returns the moving average rate of events per second. -func (a *StandardEWMA) Rate() float64 { - a.mutex.Lock() - defer a.mutex.Unlock() - return a.rate * float64(time.Second) -} - // Snapshot returns a read-only copy of the EWMA. -func (a *StandardEWMA) Snapshot() EWMA { - return EWMASnapshot(a.Rate()) +func (a *StandardEWMA) Snapshot() EWMASnapshot { + r := math.Float64frombits(a.rate.Load()) * float64(time.Second) + return ewmaSnapshot(r) } // Tick ticks the clock to update the moving average. It assumes it is called // every five seconds. func (a *StandardEWMA) Tick() { - count := a.uncounted.Load() - a.uncounted.Add(-count) - instantRate := float64(count) / float64(5*time.Second) + // Optimization to avoid mutex locking in the hot-path. + if a.init.Load() { + a.updateRate(a.fetchInstantRate()) + return + } + // Slow-path: this is only needed on the first Tick() and preserves transactional updating + // of init and rate in the else block. The first conditional is needed below because + // a different thread could have set a.init = 1 between the time of the first atomic load and when + // the lock was acquired. a.mutex.Lock() - defer a.mutex.Unlock() - if a.init { - a.rate += a.alpha * (instantRate - a.rate) + if a.init.Load() { + // The fetchInstantRate() uses atomic loading, which is unnecessary in this critical section + // but again, this section is only invoked on the first successful Tick() operation. + a.updateRate(a.fetchInstantRate()) } else { - a.init = true - a.rate = instantRate + a.init.Store(true) + a.rate.Store(math.Float64bits(a.fetchInstantRate())) } + a.mutex.Unlock() +} + +func (a *StandardEWMA) fetchInstantRate() float64 { + count := a.uncounted.Swap(0) + return float64(count) / float64(5*time.Second) +} + +func (a *StandardEWMA) updateRate(instantRate float64) { + currentRate := math.Float64frombits(a.rate.Load()) + currentRate += a.alpha * (instantRate - currentRate) + a.rate.Store(math.Float64bits(currentRate)) } // Update adds n uncounted events. diff --git a/metrics/ewma_test.go b/metrics/ewma_test.go index 5b24419161..9a91b43db8 100644 --- a/metrics/ewma_test.go +++ b/metrics/ewma_test.go @@ -5,6 +5,8 @@ import ( "testing" ) +const epsilon = 0.0000000000000001 + func BenchmarkEWMA(b *testing.B) { a := NewEWMA1() b.ResetTimer() @@ -14,72 +16,33 @@ func BenchmarkEWMA(b *testing.B) { } } +func BenchmarkEWMAParallel(b *testing.B) { + a := NewEWMA1() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + a.Update(1) + a.Tick() + } + }) +} + func TestEWMA1(t *testing.T) { a := NewEWMA1() a.Update(3) a.Tick() - if rate := a.Rate(); math.Abs(0.6-rate) > epsilon { - t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.22072766470286553-rate) > epsilon { - t.Errorf("1 minute a.Rate(): 0.22072766470286553 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.08120116994196772-rate) > epsilon { - t.Errorf("2 minute a.Rate(): 0.08120116994196772 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.029872241020718428-rate) > epsilon { - t.Errorf("3 minute a.Rate(): 0.029872241020718428 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.01098938333324054-rate) > epsilon { - t.Errorf("4 minute a.Rate(): 0.01098938333324054 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.004042768199451294-rate) > epsilon { - t.Errorf("5 minute a.Rate(): 0.004042768199451294 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.0014872513059998212-rate) > epsilon { - t.Errorf("6 minute a.Rate(): 0.0014872513059998212 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.0005471291793327122-rate) > epsilon { - t.Errorf("7 minute a.Rate(): 0.0005471291793327122 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.00020127757674150815-rate) > epsilon { - t.Errorf("8 minute a.Rate(): 0.00020127757674150815 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(7.404588245200814e-05-rate) > epsilon { - t.Errorf("9 minute a.Rate(): 7.404588245200814e-05 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(2.7239957857491083e-05-rate) > epsilon { - t.Errorf("10 minute a.Rate(): 2.7239957857491083e-05 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(1.0021020474147462e-05-rate) > epsilon { - t.Errorf("11 minute a.Rate(): 1.0021020474147462e-05 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(3.6865274119969525e-06-rate) > epsilon { - t.Errorf("12 minute a.Rate(): 3.6865274119969525e-06 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(1.3561976441886433e-06-rate) > epsilon { - t.Errorf("13 minute a.Rate(): 1.3561976441886433e-06 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(4.989172314621449e-07-rate) > epsilon { - t.Errorf("14 minute a.Rate(): 4.989172314621449e-07 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(1.8354139230109722e-07-rate) > epsilon { - t.Errorf("15 minute a.Rate(): 1.8354139230109722e-07 != %v\n", rate) + for i, want := range []float64{0.6, + 0.22072766470286553, 0.08120116994196772, 0.029872241020718428, + 0.01098938333324054, 0.004042768199451294, 0.0014872513059998212, + 0.0005471291793327122, 0.00020127757674150815, 7.404588245200814e-05, + 2.7239957857491083e-05, 1.0021020474147462e-05, 3.6865274119969525e-06, + 1.3561976441886433e-06, 4.989172314621449e-07, 1.8354139230109722e-07, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) } } @@ -87,68 +50,17 @@ func TestEWMA5(t *testing.T) { a := NewEWMA5() a.Update(3) a.Tick() - if rate := a.Rate(); math.Abs(0.6-rate) > epsilon { - t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.49123845184678905-rate) > epsilon { - t.Errorf("1 minute a.Rate(): 0.49123845184678905 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.4021920276213837-rate) > epsilon { - t.Errorf("2 minute a.Rate(): 0.4021920276213837 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.32928698165641596-rate) > epsilon { - t.Errorf("3 minute a.Rate(): 0.32928698165641596 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.269597378470333-rate) > epsilon { - t.Errorf("4 minute a.Rate(): 0.269597378470333 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.2207276647028654-rate) > epsilon { - t.Errorf("5 minute a.Rate(): 0.2207276647028654 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.18071652714732128-rate) > epsilon { - t.Errorf("6 minute a.Rate(): 0.18071652714732128 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.14795817836496392-rate) > epsilon { - t.Errorf("7 minute a.Rate(): 0.14795817836496392 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.12113791079679326-rate) > epsilon { - t.Errorf("8 minute a.Rate(): 0.12113791079679326 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.09917933293295193-rate) > epsilon { - t.Errorf("9 minute a.Rate(): 0.09917933293295193 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.08120116994196763-rate) > epsilon { - t.Errorf("10 minute a.Rate(): 0.08120116994196763 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.06648189501740036-rate) > epsilon { - t.Errorf("11 minute a.Rate(): 0.06648189501740036 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.05443077197364752-rate) > epsilon { - t.Errorf("12 minute a.Rate(): 0.05443077197364752 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.04456414692860035-rate) > epsilon { - t.Errorf("13 minute a.Rate(): 0.04456414692860035 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.03648603757513079-rate) > epsilon { - t.Errorf("14 minute a.Rate(): 0.03648603757513079 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.0298722410207183831020718428-rate) > epsilon { - t.Errorf("15 minute a.Rate(): 0.0298722410207183831020718428 != %v\n", rate) + for i, want := range []float64{ + 0.6, 0.49123845184678905, 0.4021920276213837, 0.32928698165641596, + 0.269597378470333, 0.2207276647028654, 0.18071652714732128, + 0.14795817836496392, 0.12113791079679326, 0.09917933293295193, + 0.08120116994196763, 0.06648189501740036, 0.05443077197364752, + 0.04456414692860035, 0.03648603757513079, 0.0298722410207183831020718428, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) } } @@ -156,68 +68,17 @@ func TestEWMA15(t *testing.T) { a := NewEWMA15() a.Update(3) a.Tick() - if rate := a.Rate(); math.Abs(0.6-rate) > epsilon { - t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.5613041910189706-rate) > epsilon { - t.Errorf("1 minute a.Rate(): 0.5613041910189706 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.5251039914257684-rate) > epsilon { - t.Errorf("2 minute a.Rate(): 0.5251039914257684 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.4912384518467888184678905-rate) > epsilon { - t.Errorf("3 minute a.Rate(): 0.4912384518467888184678905 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.459557003018789-rate) > epsilon { - t.Errorf("4 minute a.Rate(): 0.459557003018789 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.4299187863442732-rate) > epsilon { - t.Errorf("5 minute a.Rate(): 0.4299187863442732 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.4021920276213831-rate) > epsilon { - t.Errorf("6 minute a.Rate(): 0.4021920276213831 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.37625345116383313-rate) > epsilon { - t.Errorf("7 minute a.Rate(): 0.37625345116383313 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.3519877317060185-rate) > epsilon { - t.Errorf("8 minute a.Rate(): 0.3519877317060185 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.3292869816564153165641596-rate) > epsilon { - t.Errorf("9 minute a.Rate(): 0.3292869816564153165641596 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.3080502714195546-rate) > epsilon { - t.Errorf("10 minute a.Rate(): 0.3080502714195546 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.2881831806538789-rate) > epsilon { - t.Errorf("11 minute a.Rate(): 0.2881831806538789 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.26959737847033216-rate) > epsilon { - t.Errorf("12 minute a.Rate(): 0.26959737847033216 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.2522102307052083-rate) > epsilon { - t.Errorf("13 minute a.Rate(): 0.2522102307052083 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.23594443252115815-rate) > epsilon { - t.Errorf("14 minute a.Rate(): 0.23594443252115815 != %v\n", rate) - } - elapseMinute(a) - if rate := a.Rate(); math.Abs(0.2207276647028646247028654470286553-rate) > epsilon { - t.Errorf("15 minute a.Rate(): 0.2207276647028646247028654470286553 != %v\n", rate) + for i, want := range []float64{ + 0.6, 0.5613041910189706, 0.5251039914257684, 0.4912384518467888184678905, + 0.459557003018789, 0.4299187863442732, 0.4021920276213831, + 0.37625345116383313, 0.3519877317060185, 0.3292869816564153165641596, + 0.3080502714195546, 0.2881831806538789, 0.26959737847033216, + 0.2522102307052083, 0.23594443252115815, 0.2207276647028646247028654470286553, + } { + if rate := a.Snapshot().Rate(); math.Abs(want-rate) > epsilon { + t.Errorf("%d minute a.Snapshot().Rate(): %f != %v\n", i, want, rate) + } + elapseMinute(a) } } diff --git a/metrics/gauge.go b/metrics/gauge.go index 81137d7f7c..68f8f11abc 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -2,13 +2,18 @@ package metrics import "sync/atomic" +// gaugeSnapshot contains a readonly int64. +type GaugeSnapshot interface { + Value() int64 +} + // Gauges hold an int64 value that can be set arbitrarily. type Gauge interface { - Snapshot() Gauge + Snapshot() GaugeSnapshot Update(int64) + UpdateIfGt(int64) Dec(int64) Inc(int64) - Value() int64 } // GetOrRegisterGauge returns an existing Gauge or constructs and registers a @@ -38,65 +43,20 @@ func NewRegisteredGauge(name string, r Registry) Gauge { return c } -// NewFunctionalGauge constructs a new FunctionalGauge. -func NewFunctionalGauge(f func() int64) Gauge { - if !Enabled { - return NilGauge{} - } - return &FunctionalGauge{value: f} -} - -// NewRegisteredFunctionalGauge constructs and registers a new StandardGauge. -func NewRegisteredFunctionalGauge(name string, r Registry, f func() int64) Gauge { - c := NewFunctionalGauge(f) - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// GaugeSnapshot is a read-only copy of another Gauge. -type GaugeSnapshot int64 - -// Snapshot returns the snapshot. -func (g GaugeSnapshot) Snapshot() Gauge { return g } - -// Update panics. -func (GaugeSnapshot) Update(int64) { - panic("Update called on a GaugeSnapshot") -} - -// Dec panics. -func (GaugeSnapshot) Dec(int64) { - panic("Dec called on a GaugeSnapshot") -} - -// Inc panics. -func (GaugeSnapshot) Inc(int64) { - panic("Inc called on a GaugeSnapshot") -} +// gaugeSnapshot is a read-only copy of another Gauge. +type gaugeSnapshot int64 // Value returns the value at the time the snapshot was taken. -func (g GaugeSnapshot) Value() int64 { return int64(g) } +func (g gaugeSnapshot) Value() int64 { return int64(g) } // NilGauge is a no-op Gauge. type NilGauge struct{} -// Snapshot is a no-op. -func (NilGauge) Snapshot() Gauge { return NilGauge{} } - -// Update is a no-op. -func (NilGauge) Update(v int64) {} - -// Dec is a no-op. -func (NilGauge) Dec(i int64) {} - -// Inc is a no-op. -func (NilGauge) Inc(i int64) {} - -// Value is a no-op. -func (NilGauge) Value() int64 { return 0 } +func (NilGauge) Snapshot() GaugeSnapshot { return (*emptySnapshot)(nil) } +func (NilGauge) Update(v int64) {} +func (NilGauge) UpdateIfGt(v int64) {} +func (NilGauge) Dec(i int64) {} +func (NilGauge) Inc(i int64) {} // StandardGauge is the standard implementation of a Gauge and uses the // sync/atomic package to manage a single int64 value. @@ -105,8 +65,8 @@ type StandardGauge struct { } // Snapshot returns a read-only copy of the gauge. -func (g *StandardGauge) Snapshot() Gauge { - return GaugeSnapshot(g.Value()) +func (g *StandardGauge) Snapshot() GaugeSnapshot { + return gaugeSnapshot(g.value.Load()) } // Update updates the gauge's value. @@ -114,9 +74,17 @@ func (g *StandardGauge) Update(v int64) { g.value.Store(v) } -// Value returns the gauge's current value. -func (g *StandardGauge) Value() int64 { - return g.value.Load() +// Update updates the gauge's value if v is larger then the current valie. +func (g *StandardGauge) UpdateIfGt(v int64) { + for { + exist := g.value.Load() + if exist >= v { + break + } + if g.value.CompareAndSwap(exist, v) { + break + } + } } // Dec decrements the gauge's current value by the given amount. @@ -128,31 +96,3 @@ func (g *StandardGauge) Dec(i int64) { func (g *StandardGauge) Inc(i int64) { g.value.Add(i) } - -// FunctionalGauge returns value from given function -type FunctionalGauge struct { - value func() int64 -} - -// Value returns the gauge's current value. -func (g FunctionalGauge) Value() int64 { - return g.value() -} - -// Snapshot returns the snapshot. -func (g FunctionalGauge) Snapshot() Gauge { return GaugeSnapshot(g.Value()) } - -// Update panics. -func (FunctionalGauge) Update(int64) { - panic("Update called on a FunctionalGauge") -} - -// Dec panics. -func (FunctionalGauge) Dec(int64) { - panic("Dec called on a FunctionalGauge") -} - -// Inc panics. -func (FunctionalGauge) Inc(int64) { - panic("Inc called on a FunctionalGauge") -} diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 237ff8036e..967f2bc60e 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -5,11 +5,14 @@ import ( "sync/atomic" ) -// GaugeFloat64s hold a float64 value that can be set arbitrarily. +type GaugeFloat64Snapshot interface { + Value() float64 +} + +// GaugeFloat64 hold a float64 value that can be set arbitrarily. type GaugeFloat64 interface { - Snapshot() GaugeFloat64 + Snapshot() GaugeFloat64Snapshot Update(float64) - Value() float64 } // GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a @@ -39,49 +42,18 @@ func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 { return c } -// NewFunctionalGauge constructs a new FunctionalGauge. -func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 { - if !Enabled { - return NilGaugeFloat64{} - } - return &FunctionalGaugeFloat64{value: f} -} - -// NewRegisteredFunctionalGauge constructs and registers a new StandardGauge. -func NewRegisteredFunctionalGaugeFloat64(name string, r Registry, f func() float64) GaugeFloat64 { - c := NewFunctionalGaugeFloat64(f) - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// GaugeFloat64Snapshot is a read-only copy of another GaugeFloat64. -type GaugeFloat64Snapshot float64 - -// Snapshot returns the snapshot. -func (g GaugeFloat64Snapshot) Snapshot() GaugeFloat64 { return g } - -// Update panics. -func (GaugeFloat64Snapshot) Update(float64) { - panic("Update called on a GaugeFloat64Snapshot") -} +// gaugeFloat64Snapshot is a read-only copy of another GaugeFloat64. +type gaugeFloat64Snapshot float64 // Value returns the value at the time the snapshot was taken. -func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) } +func (g gaugeFloat64Snapshot) Value() float64 { return float64(g) } // NilGauge is a no-op Gauge. type NilGaugeFloat64 struct{} -// Snapshot is a no-op. -func (NilGaugeFloat64) Snapshot() GaugeFloat64 { return NilGaugeFloat64{} } - -// Update is a no-op. -func (NilGaugeFloat64) Update(v float64) {} - -// Value is a no-op. -func (NilGaugeFloat64) Value() float64 { return 0.0 } +func (NilGaugeFloat64) Snapshot() GaugeFloat64Snapshot { return NilGaugeFloat64{} } +func (NilGaugeFloat64) Update(v float64) {} +func (NilGaugeFloat64) Value() float64 { return 0.0 } // StandardGaugeFloat64 is the standard implementation of a GaugeFloat64 and uses // atomic to manage a single float64 value. @@ -90,34 +62,12 @@ type StandardGaugeFloat64 struct { } // Snapshot returns a read-only copy of the gauge. -func (g *StandardGaugeFloat64) Snapshot() GaugeFloat64 { - return GaugeFloat64Snapshot(g.Value()) +func (g *StandardGaugeFloat64) Snapshot() GaugeFloat64Snapshot { + v := math.Float64frombits(g.floatBits.Load()) + return gaugeFloat64Snapshot(v) } // Update updates the gauge's value. func (g *StandardGaugeFloat64) Update(v float64) { g.floatBits.Store(math.Float64bits(v)) } - -// Value returns the gauge's current value. -func (g *StandardGaugeFloat64) Value() float64 { - return math.Float64frombits(g.floatBits.Load()) -} - -// FunctionalGaugeFloat64 returns value from given function -type FunctionalGaugeFloat64 struct { - value func() float64 -} - -// Value returns the gauge's current value. -func (g FunctionalGaugeFloat64) Value() float64 { - return g.value() -} - -// Snapshot returns the snapshot. -func (g FunctionalGaugeFloat64) Snapshot() GaugeFloat64 { return GaugeFloat64Snapshot(g.Value()) } - -// Update panics. -func (FunctionalGaugeFloat64) Update(float64) { - panic("Update called on a FunctionalGaugeFloat64") -} diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 647d090009..f0ac7ea5e7 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -26,19 +26,11 @@ func BenchmarkGaugeFloat64Parallel(b *testing.B) { }() } wg.Wait() - if have, want := c.Value(), float64(b.N-1); have != want { + if have, want := c.Snapshot().Value(), float64(b.N-1); have != want { b.Fatalf("have %f want %f", have, want) } } -func TestGaugeFloat64(t *testing.T) { - g := NewGaugeFloat64() - g.Update(47.0) - if v := g.Value(); 47.0 != v { - t.Errorf("g.Value(): 47.0 != %v\n", v) - } -} - func TestGaugeFloat64Snapshot(t *testing.T) { g := NewGaugeFloat64() g.Update(47.0) @@ -53,28 +45,7 @@ func TestGetOrRegisterGaugeFloat64(t *testing.T) { r := NewRegistry() NewRegisteredGaugeFloat64("foo", r).Update(47.0) t.Logf("registry: %v", r) - if g := GetOrRegisterGaugeFloat64("foo", r); 47.0 != g.Value() { - t.Fatal(g) - } -} - -func TestFunctionalGaugeFloat64(t *testing.T) { - var counter float64 - fg := NewFunctionalGaugeFloat64(func() float64 { - counter++ - return counter - }) - fg.Value() - fg.Value() - if counter != 2 { - t.Error("counter != 2") - } -} - -func TestGetOrRegisterFunctionalGaugeFloat64(t *testing.T) { - r := NewRegistry() - NewRegisteredFunctionalGaugeFloat64("foo", r, func() float64 { return 47 }) - if g := GetOrRegisterGaugeFloat64("foo", r); g.Value() != 47 { + if g := GetOrRegisterGaugeFloat64("foo", r).Snapshot(); 47.0 != g.Value() { t.Fatal(g) } } diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go new file mode 100644 index 0000000000..c44b2d85f3 --- /dev/null +++ b/metrics/gauge_info.go @@ -0,0 +1,84 @@ +package metrics + +import ( + "encoding/json" + "sync" +) + +type GaugeInfoSnapshot interface { + Value() GaugeInfoValue +} + +// GaugeInfos hold a GaugeInfoValue value that can be set arbitrarily. +type GaugeInfo interface { + Update(GaugeInfoValue) + Snapshot() GaugeInfoSnapshot +} + +// GaugeInfoValue is a mapping of keys to values +type GaugeInfoValue map[string]string + +func (val GaugeInfoValue) String() string { + data, _ := json.Marshal(val) + return string(data) +} + +// GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a +// new StandardGaugeInfo. +func GetOrRegisterGaugeInfo(name string, r Registry) GaugeInfo { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewGaugeInfo()).(GaugeInfo) +} + +// NewGaugeInfo constructs a new StandardGaugeInfo. +func NewGaugeInfo() GaugeInfo { + if !Enabled { + return NilGaugeInfo{} + } + return &StandardGaugeInfo{ + value: GaugeInfoValue{}, + } +} + +// NewRegisteredGaugeInfo constructs and registers a new StandardGaugeInfo. +func NewRegisteredGaugeInfo(name string, r Registry) GaugeInfo { + c := NewGaugeInfo() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// gaugeInfoSnapshot is a read-only copy of another GaugeInfo. +type gaugeInfoSnapshot GaugeInfoValue + +// Value returns the value at the time the snapshot was taken. +func (g gaugeInfoSnapshot) Value() GaugeInfoValue { return GaugeInfoValue(g) } + +type NilGaugeInfo struct{} + +func (NilGaugeInfo) Snapshot() GaugeInfoSnapshot { return NilGaugeInfo{} } +func (NilGaugeInfo) Update(v GaugeInfoValue) {} +func (NilGaugeInfo) Value() GaugeInfoValue { return GaugeInfoValue{} } + +// StandardGaugeInfo is the standard implementation of a GaugeInfo and uses +// sync.Mutex to manage a single string value. +type StandardGaugeInfo struct { + mutex sync.Mutex + value GaugeInfoValue +} + +// Snapshot returns a read-only copy of the gauge. +func (g *StandardGaugeInfo) Snapshot() GaugeInfoSnapshot { + return gaugeInfoSnapshot(g.value) +} + +// Update updates the gauge's value. +func (g *StandardGaugeInfo) Update(v GaugeInfoValue) { + g.mutex.Lock() + defer g.mutex.Unlock() + g.value = v +} diff --git a/metrics/gauge_info_test.go b/metrics/gauge_info_test.go new file mode 100644 index 0000000000..319afbf92e --- /dev/null +++ b/metrics/gauge_info_test.go @@ -0,0 +1,36 @@ +package metrics + +import ( + "testing" +) + +func TestGaugeInfoJsonString(t *testing.T) { + g := NewGaugeInfo() + g.Update(GaugeInfoValue{ + "chain_id": "5", + "anotherKey": "any_string_value", + "third_key": "anything", + }, + ) + want := `{"anotherKey":"any_string_value","chain_id":"5","third_key":"anything"}` + + original := g.Snapshot() + g.Update(GaugeInfoValue{"value": "updated"}) + + if have := original.Value().String(); have != want { + t.Errorf("\nhave: %v\nwant: %v\n", have, want) + } + if have, want := g.Snapshot().Value().String(), `{"value":"updated"}`; have != want { + t.Errorf("\nhave: %v\nwant: %v\n", have, want) + } +} + +func TestGetOrRegisterGaugeInfo(t *testing.T) { + r := NewRegistry() + NewRegisteredGaugeInfo("foo", r).Update( + GaugeInfoValue{"chain_id": "5"}) + g := GetOrRegisterGaugeInfo("foo", r).Snapshot() + if have, want := g.Value().String(), `{"chain_id":"5"}`; have != want { + t.Errorf("have\n%v\nwant\n%v\n", have, want) + } +} diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index a98fe985d8..f2ba930bc4 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -1,7 +1,6 @@ package metrics import ( - "fmt" "testing" ) @@ -13,14 +12,6 @@ func BenchmarkGauge(b *testing.B) { } } -func TestGauge(t *testing.T) { - g := NewGauge() - g.Update(int64(47)) - if v := g.Value(); v != 47 { - t.Errorf("g.Value(): 47 != %v\n", v) - } -} - func TestGaugeSnapshot(t *testing.T) { g := NewGauge() g.Update(int64(47)) @@ -34,35 +25,7 @@ func TestGaugeSnapshot(t *testing.T) { func TestGetOrRegisterGauge(t *testing.T) { r := NewRegistry() NewRegisteredGauge("foo", r).Update(47) - if g := GetOrRegisterGauge("foo", r); g.Value() != 47 { - t.Fatal(g) - } -} - -func TestFunctionalGauge(t *testing.T) { - var counter int64 - fg := NewFunctionalGauge(func() int64 { - counter++ - return counter - }) - fg.Value() - fg.Value() - if counter != 2 { - t.Error("counter != 2") - } -} - -func TestGetOrRegisterFunctionalGauge(t *testing.T) { - r := NewRegistry() - NewRegisteredFunctionalGauge("foo", r, func() int64 { return 47 }) - if g := GetOrRegisterGauge("foo", r); g.Value() != 47 { + if g := GetOrRegisterGauge("foo", r); g.Snapshot().Value() != 47 { t.Fatal(g) } } - -func ExampleGetOrRegisterGauge() { - m := "server.bytes_sent" - g := GetOrRegisterGauge(m, nil) - g.Update(47) - fmt.Println(g.Value()) // Output: 47 -} diff --git a/metrics/graphite.go b/metrics/graphite.go index 29f72b0c41..aba752e0ed 100644 --- a/metrics/graphite.go +++ b/metrics/graphite.go @@ -66,13 +66,15 @@ func graphite(c *GraphiteConfig) error { c.Registry.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: - fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, metric.Count(), now) + fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, metric.Snapshot().Count(), now) case CounterFloat64: - fmt.Fprintf(w, "%s.%s.count %f %d\n", c.Prefix, name, metric.Count(), now) + fmt.Fprintf(w, "%s.%s.count %f %d\n", c.Prefix, name, metric.Snapshot().Count(), now) case Gauge: - fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now) + fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Snapshot().Value(), now) case GaugeFloat64: - fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now) + fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Snapshot().Value(), now) + case GaugeInfo: + fmt.Fprintf(w, "%s.%s.value %s %d\n", c.Prefix, name, metric.Snapshot().Value().String(), now) case Histogram: h := metric.Snapshot() ps := h.Percentiles(c.Percentiles) diff --git a/metrics/histogram.go b/metrics/histogram.go index 2c54ce8b40..44de588bc1 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -1,20 +1,14 @@ package metrics +type HistogramSnapshot interface { + SampleSnapshot +} + // Histograms calculate distribution statistics from a series of int64 values. type Histogram interface { Clear() - Count() int64 - Max() int64 - Mean() float64 - Min() int64 - Percentile(float64) float64 - Percentiles([]float64) []float64 - Sample() Sample - Snapshot() Histogram - StdDev() float64 - Sum() int64 Update(int64) - Variance() float64 + Snapshot() HistogramSnapshot } // GetOrRegisterHistogram returns an existing Histogram or constructs and @@ -54,108 +48,12 @@ func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram { return c } -// HistogramSnapshot is a read-only copy of another Histogram. -type HistogramSnapshot struct { - sample *SampleSnapshot -} - -// Clear panics. -func (*HistogramSnapshot) Clear() { - panic("Clear called on a HistogramSnapshot") -} - -// Count returns the number of samples recorded at the time the snapshot was -// taken. -func (h *HistogramSnapshot) Count() int64 { return h.sample.Count() } - -// Max returns the maximum value in the sample at the time the snapshot was -// taken. -func (h *HistogramSnapshot) Max() int64 { return h.sample.Max() } - -// Mean returns the mean of the values in the sample at the time the snapshot -// was taken. -func (h *HistogramSnapshot) Mean() float64 { return h.sample.Mean() } - -// Min returns the minimum value in the sample at the time the snapshot was -// taken. -func (h *HistogramSnapshot) Min() int64 { return h.sample.Min() } - -// Percentile returns an arbitrary percentile of values in the sample at the -// time the snapshot was taken. -func (h *HistogramSnapshot) Percentile(p float64) float64 { - return h.sample.Percentile(p) -} - -// Percentiles returns a slice of arbitrary percentiles of values in the sample -// at the time the snapshot was taken. -func (h *HistogramSnapshot) Percentiles(ps []float64) []float64 { - return h.sample.Percentiles(ps) -} - -// Sample returns the Sample underlying the histogram. -func (h *HistogramSnapshot) Sample() Sample { return h.sample } - -// Snapshot returns the snapshot. -func (h *HistogramSnapshot) Snapshot() Histogram { return h } - -// StdDev returns the standard deviation of the values in the sample at the -// time the snapshot was taken. -func (h *HistogramSnapshot) StdDev() float64 { return h.sample.StdDev() } - -// Sum returns the sum in the sample at the time the snapshot was taken. -func (h *HistogramSnapshot) Sum() int64 { return h.sample.Sum() } - -// Update panics. -func (*HistogramSnapshot) Update(int64) { - panic("Update called on a HistogramSnapshot") -} - -// Variance returns the variance of inputs at the time the snapshot was taken. -func (h *HistogramSnapshot) Variance() float64 { return h.sample.Variance() } - // NilHistogram is a no-op Histogram. type NilHistogram struct{} -// Clear is a no-op. -func (NilHistogram) Clear() {} - -// Count is a no-op. -func (NilHistogram) Count() int64 { return 0 } - -// Max is a no-op. -func (NilHistogram) Max() int64 { return 0 } - -// Mean is a no-op. -func (NilHistogram) Mean() float64 { return 0.0 } - -// Min is a no-op. -func (NilHistogram) Min() int64 { return 0 } - -// Percentile is a no-op. -func (NilHistogram) Percentile(p float64) float64 { return 0.0 } - -// Percentiles is a no-op. -func (NilHistogram) Percentiles(ps []float64) []float64 { - return make([]float64, len(ps)) -} - -// Sample is a no-op. -func (NilHistogram) Sample() Sample { return NilSample{} } - -// Snapshot is a no-op. -func (NilHistogram) Snapshot() Histogram { return NilHistogram{} } - -// StdDev is a no-op. -func (NilHistogram) StdDev() float64 { return 0.0 } - -// Sum is a no-op. -func (NilHistogram) Sum() int64 { return 0 } - -// Update is a no-op. -func (NilHistogram) Update(v int64) {} - -// Variance is a no-op. -func (NilHistogram) Variance() float64 { return 0.0 } +func (NilHistogram) Clear() {} +func (NilHistogram) Snapshot() HistogramSnapshot { return (*emptySnapshot)(nil) } +func (NilHistogram) Update(v int64) {} // StandardHistogram is the standard implementation of a Histogram and uses a // Sample to bound its memory use. @@ -166,46 +64,10 @@ type StandardHistogram struct { // Clear clears the histogram and its sample. func (h *StandardHistogram) Clear() { h.sample.Clear() } -// Count returns the number of samples recorded since the histogram was last -// cleared. -func (h *StandardHistogram) Count() int64 { return h.sample.Count() } - -// Max returns the maximum value in the sample. -func (h *StandardHistogram) Max() int64 { return h.sample.Max() } - -// Mean returns the mean of the values in the sample. -func (h *StandardHistogram) Mean() float64 { return h.sample.Mean() } - -// Min returns the minimum value in the sample. -func (h *StandardHistogram) Min() int64 { return h.sample.Min() } - -// Percentile returns an arbitrary percentile of the values in the sample. -func (h *StandardHistogram) Percentile(p float64) float64 { - return h.sample.Percentile(p) -} - -// Percentiles returns a slice of arbitrary percentiles of the values in the -// sample. -func (h *StandardHistogram) Percentiles(ps []float64) []float64 { - return h.sample.Percentiles(ps) -} - -// Sample returns the Sample underlying the histogram. -func (h *StandardHistogram) Sample() Sample { return h.sample } - // Snapshot returns a read-only copy of the histogram. -func (h *StandardHistogram) Snapshot() Histogram { - return &HistogramSnapshot{sample: h.sample.Snapshot().(*SampleSnapshot)} +func (h *StandardHistogram) Snapshot() HistogramSnapshot { + return h.sample.Snapshot() } -// StdDev returns the standard deviation of the values in the sample. -func (h *StandardHistogram) StdDev() float64 { return h.sample.StdDev() } - -// Sum returns the sum in the sample. -func (h *StandardHistogram) Sum() int64 { return h.sample.Sum() } - // Update samples a new value. func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) } - -// Variance returns the variance of the values in the sample. -func (h *StandardHistogram) Variance() float64 { return h.sample.Variance() } diff --git a/metrics/histogram_test.go b/metrics/histogram_test.go index 7c9f42fcec..22fc5468b0 100644 --- a/metrics/histogram_test.go +++ b/metrics/histogram_test.go @@ -14,7 +14,7 @@ func TestGetOrRegisterHistogram(t *testing.T) { r := NewRegistry() s := NewUniformSample(100) NewRegisteredHistogram("foo", r, s).Update(47) - if h := GetOrRegisterHistogram("foo", r, s); h.Count() != 1 { + if h := GetOrRegisterHistogram("foo", r, s).Snapshot(); h.Count() != 1 { t.Fatal(h) } } @@ -24,11 +24,11 @@ func TestHistogram10000(t *testing.T) { for i := 1; i <= 10000; i++ { h.Update(int64(i)) } - testHistogram10000(t, h) + testHistogram10000(t, h.Snapshot()) } func TestHistogramEmpty(t *testing.T) { - h := NewHistogram(NewUniformSample(100)) + h := NewHistogram(NewUniformSample(100)).Snapshot() if count := h.Count(); count != 0 { t.Errorf("h.Count(): 0 != %v\n", count) } @@ -66,7 +66,7 @@ func TestHistogramSnapshot(t *testing.T) { testHistogram10000(t, snapshot) } -func testHistogram10000(t *testing.T, h Histogram) { +func testHistogram10000(t *testing.T, h HistogramSnapshot) { if count := h.Count(); count != 10000 { t.Errorf("h.Count(): 10000 != %v\n", count) } diff --git a/metrics/inactive.go b/metrics/inactive.go new file mode 100644 index 0000000000..1f47f0210a --- /dev/null +++ b/metrics/inactive.go @@ -0,0 +1,48 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package metrics + +// compile-time checks that interfaces are implemented. +var ( + _ SampleSnapshot = (*emptySnapshot)(nil) + _ HistogramSnapshot = (*emptySnapshot)(nil) + _ CounterSnapshot = (*emptySnapshot)(nil) + _ GaugeSnapshot = (*emptySnapshot)(nil) + _ MeterSnapshot = (*emptySnapshot)(nil) + _ EWMASnapshot = (*emptySnapshot)(nil) + _ TimerSnapshot = (*emptySnapshot)(nil) +) + +type emptySnapshot struct{} + +func (*emptySnapshot) Count() int64 { return 0 } +func (*emptySnapshot) Max() int64 { return 0 } +func (*emptySnapshot) Mean() float64 { return 0.0 } +func (*emptySnapshot) Min() int64 { return 0 } +func (*emptySnapshot) Percentile(p float64) float64 { return 0.0 } +func (*emptySnapshot) Percentiles(ps []float64) []float64 { return make([]float64, len(ps)) } +func (*emptySnapshot) Size() int { return 0 } +func (*emptySnapshot) StdDev() float64 { return 0.0 } +func (*emptySnapshot) Sum() int64 { return 0 } +func (*emptySnapshot) Values() []int64 { return []int64{} } +func (*emptySnapshot) Variance() float64 { return 0.0 } +func (*emptySnapshot) Value() int64 { return 0 } +func (*emptySnapshot) Rate() float64 { return 0.0 } +func (*emptySnapshot) Rate1() float64 { return 0.0 } +func (*emptySnapshot) Rate5() float64 { return 0.0 } +func (*emptySnapshot) Rate15() float64 { return 0.0 } +func (*emptySnapshot) RateMean() float64 { return 0.0 } diff --git a/metrics/log.go b/metrics/log.go index d1ce627a83..3b9773faa7 100644 --- a/metrics/log.go +++ b/metrics/log.go @@ -23,16 +23,19 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { switch metric := i.(type) { case Counter: l.Printf("counter %s\n", name) - l.Printf(" count: %9d\n", metric.Count()) + l.Printf(" count: %9d\n", metric.Snapshot().Count()) case CounterFloat64: l.Printf("counter %s\n", name) - l.Printf(" count: %f\n", metric.Count()) + l.Printf(" count: %f\n", metric.Snapshot().Count()) case Gauge: l.Printf("gauge %s\n", name) - l.Printf(" value: %9d\n", metric.Value()) + l.Printf(" value: %9d\n", metric.Snapshot().Value()) case GaugeFloat64: l.Printf("gauge %s\n", name) - l.Printf(" value: %f\n", metric.Value()) + l.Printf(" value: %f\n", metric.Snapshot().Value()) + case GaugeInfo: + l.Printf("gauge %s\n", name) + l.Printf(" value: %s\n", metric.Snapshot().Value()) case Healthcheck: metric.Check() l.Printf("healthcheck %s\n", name) diff --git a/metrics/meter.go b/metrics/meter.go index e8564d6a5e..22475ef6eb 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -1,21 +1,25 @@ package metrics import ( + "math" "sync" "sync/atomic" "time" ) -// Meters count events to produce exponentially-weighted moving average rates -// at one-, five-, and fifteen-minutes and a mean rate. -type Meter interface { +type MeterSnapshot interface { Count() int64 - Mark(int64) Rate1() float64 Rate5() float64 Rate15() float64 RateMean() float64 - Snapshot() Meter +} + +// Meters count events to produce exponentially-weighted moving average rates +// at one-, five-, and fifteen-minutes and a mean rate. +type Meter interface { + Mark(int64) + Snapshot() MeterSnapshot Stop() } @@ -30,17 +34,6 @@ func GetOrRegisterMeter(name string, r Registry) Meter { return r.GetOrRegister(name, NewMeter).(Meter) } -// GetOrRegisterMeterForced returns an existing Meter or constructs and registers a -// new StandardMeter no matter the global switch is enabled or not. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func GetOrRegisterMeterForced(name string, r Registry) Meter { - if nil == r { - r = DefaultRegistry - } - return r.GetOrRegister(name, NewMeterForced).(Meter) -} - // NewMeter constructs a new StandardMeter and launches a goroutine. // Be sure to call Stop() once the meter is of no use to allow for garbage collection. func NewMeter() Meter { @@ -58,18 +51,13 @@ func NewMeter() Meter { return m } -// NewMeterForced constructs a new StandardMeter and launches a goroutine no matter -// the global switch is enabled or not. -// Be sure to call Stop() once the meter is of no use to allow for garbage collection. -func NewMeterForced() Meter { - m := newStandardMeter() - arbiter.Lock() - defer arbiter.Unlock() - arbiter.meters[m] = struct{}{} - if !arbiter.started { - arbiter.started = true - go arbiter.tick() +// NewInactiveMeter returns a meter but does not start any goroutines. This +// method is mainly intended for testing. +func NewInactiveMeter() Meter { + if !Enabled { + return NilMeter{} } + m := newStandardMeter() return m } @@ -78,95 +66,48 @@ func NewMeterForced() Meter { // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func NewRegisteredMeter(name string, r Registry) Meter { - c := NewMeter() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c -} - -// NewRegisteredMeterForced constructs and registers a new StandardMeter -// and launches a goroutine no matter the global switch is enabled or not. -// Be sure to unregister the meter from the registry once it is of no use to -// allow for garbage collection. -func NewRegisteredMeterForced(name string, r Registry) Meter { - c := NewMeterForced() - if nil == r { - r = DefaultRegistry - } - r.Register(name, c) - return c + return GetOrRegisterMeter(name, r) } -// MeterSnapshot is a read-only copy of another Meter. -type MeterSnapshot struct { - temp atomic.Int64 +// meterSnapshot is a read-only copy of the meter's internal values. +type meterSnapshot struct { count int64 rate1, rate5, rate15, rateMean float64 } // Count returns the count of events at the time the snapshot was taken. -func (m *MeterSnapshot) Count() int64 { return m.count } - -// Mark panics. -func (*MeterSnapshot) Mark(n int64) { - panic("Mark called on a MeterSnapshot") -} +func (m *meterSnapshot) Count() int64 { return m.count } // Rate1 returns the one-minute moving average rate of events per second at the // time the snapshot was taken. -func (m *MeterSnapshot) Rate1() float64 { return m.rate1 } +func (m *meterSnapshot) Rate1() float64 { return m.rate1 } // Rate5 returns the five-minute moving average rate of events per second at // the time the snapshot was taken. -func (m *MeterSnapshot) Rate5() float64 { return m.rate5 } +func (m *meterSnapshot) Rate5() float64 { return m.rate5 } // Rate15 returns the fifteen-minute moving average rate of events per second // at the time the snapshot was taken. -func (m *MeterSnapshot) Rate15() float64 { return m.rate15 } +func (m *meterSnapshot) Rate15() float64 { return m.rate15 } // RateMean returns the meter's mean rate of events per second at the time the // snapshot was taken. -func (m *MeterSnapshot) RateMean() float64 { return m.rateMean } - -// Snapshot returns the snapshot. -func (m *MeterSnapshot) Snapshot() Meter { return m } - -// Stop is a no-op. -func (m *MeterSnapshot) Stop() {} +func (m *meterSnapshot) RateMean() float64 { return m.rateMean } // NilMeter is a no-op Meter. type NilMeter struct{} -// Count is a no-op. -func (NilMeter) Count() int64 { return 0 } - -// Mark is a no-op. -func (NilMeter) Mark(n int64) {} - -// Rate1 is a no-op. -func (NilMeter) Rate1() float64 { return 0.0 } - -// Rate5 is a no-op. -func (NilMeter) Rate5() float64 { return 0.0 } - -// Rate15 is a no-op. -func (NilMeter) Rate15() float64 { return 0.0 } - -// RateMean is a no-op. -func (NilMeter) RateMean() float64 { return 0.0 } - -// Snapshot is a no-op. -func (NilMeter) Snapshot() Meter { return NilMeter{} } - -// Stop is a no-op. -func (NilMeter) Stop() {} +func (NilMeter) Count() int64 { return 0 } +func (NilMeter) Mark(n int64) {} +func (NilMeter) Snapshot() MeterSnapshot { return (*emptySnapshot)(nil) } +func (NilMeter) Stop() {} // StandardMeter is the standard implementation of a Meter. type StandardMeter struct { - lock sync.RWMutex - snapshot *MeterSnapshot + count atomic.Int64 + uncounted atomic.Int64 // not yet added to the EWMAs + rateMean atomic.Uint64 + a1, a5, a15 EWMA startTime time.Time stopped atomic.Bool @@ -174,7 +115,6 @@ type StandardMeter struct { func newStandardMeter() *StandardMeter { return &StandardMeter{ - snapshot: &MeterSnapshot{}, a1: NewEWMA1(), a5: NewEWMA5(), a15: NewEWMA15(), @@ -184,97 +124,42 @@ func newStandardMeter() *StandardMeter { // Stop stops the meter, Mark() will be a no-op if you use it after being stopped. func (m *StandardMeter) Stop() { - stopped := m.stopped.Swap(true) - if !stopped { + if stopped := m.stopped.Swap(true); !stopped { arbiter.Lock() delete(arbiter.meters, m) arbiter.Unlock() } } -// Count returns the number of events recorded. -// It updates the meter to be as accurate as possible -func (m *StandardMeter) Count() int64 { - m.lock.Lock() - defer m.lock.Unlock() - m.updateMeter() - return m.snapshot.count -} - // Mark records the occurrence of n events. func (m *StandardMeter) Mark(n int64) { - m.snapshot.temp.Add(n) -} - -// Rate1 returns the one-minute moving average rate of events per second. -func (m *StandardMeter) Rate1() float64 { - m.lock.RLock() - defer m.lock.RUnlock() - return m.snapshot.rate1 -} - -// Rate5 returns the five-minute moving average rate of events per second. -func (m *StandardMeter) Rate5() float64 { - m.lock.RLock() - defer m.lock.RUnlock() - return m.snapshot.rate5 -} - -// Rate15 returns the fifteen-minute moving average rate of events per second. -func (m *StandardMeter) Rate15() float64 { - m.lock.RLock() - defer m.lock.RUnlock() - return m.snapshot.rate15 -} - -// RateMean returns the meter's mean rate of events per second. -func (m *StandardMeter) RateMean() float64 { - m.lock.RLock() - defer m.lock.RUnlock() - return m.snapshot.rateMean + m.uncounted.Add(n) } // Snapshot returns a read-only copy of the meter. -func (m *StandardMeter) Snapshot() Meter { - m.lock.RLock() - snapshot := MeterSnapshot{ - count: m.snapshot.count, - rate1: m.snapshot.rate1, - rate5: m.snapshot.rate5, - rate15: m.snapshot.rate15, - rateMean: m.snapshot.rateMean, +func (m *StandardMeter) Snapshot() MeterSnapshot { + return &meterSnapshot{ + count: m.count.Load() + m.uncounted.Load(), + rate1: m.a1.Snapshot().Rate(), + rate5: m.a5.Snapshot().Rate(), + rate15: m.a15.Snapshot().Rate(), + rateMean: math.Float64frombits(m.rateMean.Load()), } - snapshot.temp.Store(m.snapshot.temp.Load()) - m.lock.RUnlock() - return &snapshot -} - -func (m *StandardMeter) updateSnapshot() { - // should run with write lock held on m.lock - snapshot := m.snapshot - snapshot.rate1 = m.a1.Rate() - snapshot.rate5 = m.a5.Rate() - snapshot.rate15 = m.a15.Rate() - snapshot.rateMean = float64(snapshot.count) / time.Since(m.startTime).Seconds() } -func (m *StandardMeter) updateMeter() { - // should only run with write lock held on m.lock - n := m.snapshot.temp.Swap(0) - m.snapshot.count += n +func (m *StandardMeter) tick() { + // Take the uncounted values, add to count + n := m.uncounted.Swap(0) + count := m.count.Add(n) + m.rateMean.Store(math.Float64bits(float64(count) / time.Since(m.startTime).Seconds())) + // Update the EWMA's internal state m.a1.Update(n) m.a5.Update(n) m.a15.Update(n) -} - -func (m *StandardMeter) tick() { - m.lock.Lock() - defer m.lock.Unlock() - m.updateMeter() + // And trigger them to calculate the rates m.a1.Tick() m.a5.Tick() m.a15.Tick() - m.updateSnapshot() } // meterArbiter ticks meters every 5s from a single goroutine. diff --git a/metrics/meter_test.go b/metrics/meter_test.go index b3f6cb8c0c..019c4d765b 100644 --- a/metrics/meter_test.go +++ b/metrics/meter_test.go @@ -12,11 +12,17 @@ func BenchmarkMeter(b *testing.B) { m.Mark(1) } } - +func TestMeter(t *testing.T) { + m := NewMeter() + m.Mark(47) + if v := m.Snapshot().Count(); v != 47 { + t.Fatalf("have %d want %d", v, 47) + } +} func TestGetOrRegisterMeter(t *testing.T) { r := NewRegistry() NewRegisteredMeter("foo", r).Mark(47) - if m := GetOrRegisterMeter("foo", r); m.Count() != 47 { + if m := GetOrRegisterMeter("foo", r).Snapshot(); m.Count() != 47 { t.Fatal(m.Count()) } } @@ -31,10 +37,10 @@ func TestMeterDecay(t *testing.T) { ma.meters[m] = struct{}{} m.Mark(1) ma.tickMeters() - rateMean := m.RateMean() + rateMean := m.Snapshot().RateMean() time.Sleep(100 * time.Millisecond) ma.tickMeters() - if m.RateMean() >= rateMean { + if m.Snapshot().RateMean() >= rateMean { t.Error("m.RateMean() didn't decrease") } } @@ -42,7 +48,7 @@ func TestMeterDecay(t *testing.T) { func TestMeterNonzero(t *testing.T) { m := NewMeter() m.Mark(3) - if count := m.Count(); count != 3 { + if count := m.Snapshot().Count(); count != 3 { t.Errorf("m.Count(): 3 != %v\n", count) } } @@ -59,16 +65,8 @@ func TestMeterStop(t *testing.T) { } } -func TestMeterSnapshot(t *testing.T) { - m := NewMeter() - m.Mark(1) - if snapshot := m.Snapshot(); m.RateMean() != snapshot.RateMean() { - t.Fatal(snapshot) - } -} - func TestMeterZero(t *testing.T) { - m := NewMeter() + m := NewMeter().Snapshot() if count := m.Count(); count != 0 { t.Errorf("m.Count(): 0 != %v\n", count) } @@ -79,13 +77,13 @@ func TestMeterRepeat(t *testing.T) { for i := 0; i < 101; i++ { m.Mark(int64(i)) } - if count := m.Count(); count != 5050 { + if count := m.Snapshot().Count(); count != 5050 { t.Errorf("m.Count(): 5050 != %v\n", count) } for i := 0; i < 101; i++ { m.Mark(int64(i)) } - if count := m.Count(); count != 10100 { + if count := m.Snapshot().Count(); count != 10100 { t.Errorf("m.Count(): 10100 != %v\n", count) } } diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go index faa74ee080..775b247185 100644 --- a/metrics/metrics_test.go +++ b/metrics/metrics_test.go @@ -92,8 +92,8 @@ func Example() { t.Time(func() { time.Sleep(10 * time.Millisecond) }) t.Update(1) - fmt.Println(c.Count()) - fmt.Println(t.Min()) + fmt.Println(c.Snapshot().Count()) + fmt.Println(t.Snapshot().Min()) // Output: 17 // 1 } diff --git a/metrics/opentsdb.go b/metrics/opentsdb.go index c9fd2e75d5..e81690f943 100644 --- a/metrics/opentsdb.go +++ b/metrics/opentsdb.go @@ -3,6 +3,7 @@ package metrics import ( "bufio" "fmt" + "io" "log" "net" "os" @@ -57,26 +58,22 @@ func getShortHostname() string { return shortHostName } -func openTSDB(c *OpenTSDBConfig) error { - shortHostname := getShortHostname() - now := time.Now().Unix() +// writeRegistry writes the registry-metrics on the opentsb format. +func (c *OpenTSDBConfig) writeRegistry(w io.Writer, now int64, shortHostname string) { du := float64(c.DurationUnit) - conn, err := net.DialTCP("tcp", nil, c.Addr) - if nil != err { - return err - } - defer conn.Close() - w := bufio.NewWriter(conn) + c.Registry.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: - fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) case CounterFloat64: - fmt.Fprintf(w, "put %s.%s.count %d %f host=%s\n", c.Prefix, name, now, metric.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.count %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Count(), shortHostname) case Gauge: - fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname) + fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) case GaugeFloat64: - fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname) + fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Snapshot().Value(), shortHostname) + case GaugeInfo: + fmt.Fprintf(w, "put %s.%s.value %d %s host=%s\n", c.Prefix, name, now, metric.Snapshot().Value().String(), shortHostname) case Histogram: h := metric.Snapshot() ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) @@ -115,7 +112,17 @@ func openTSDB(c *OpenTSDBConfig) error { fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname) fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname) } - w.Flush() }) +} + +func openTSDB(c *OpenTSDBConfig) error { + conn, err := net.DialTCP("tcp", nil, c.Addr) + if nil != err { + return err + } + defer conn.Close() + w := bufio.NewWriter(conn) + c.writeRegistry(w, time.Now().Unix(), getShortHostname()) + w.Flush() return nil } diff --git a/metrics/opentsdb_test.go b/metrics/opentsdb_test.go index c43728960e..4548309f9c 100644 --- a/metrics/opentsdb_test.go +++ b/metrics/opentsdb_test.go @@ -1,7 +1,11 @@ package metrics import ( + "fmt" "net" + "os" + "strings" + "testing" "time" ) @@ -19,3 +23,44 @@ func ExampleOpenTSDBWithConfig() { DurationUnit: time.Millisecond, }) } + +func TestExampleOpenTSB(t *testing.T) { + r := NewOrderedRegistry() + NewRegisteredGaugeInfo("foo", r).Update(GaugeInfoValue{"chain_id": "5"}) + NewRegisteredGaugeFloat64("pi", r).Update(3.14) + NewRegisteredCounter("months", r).Inc(12) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + NewRegisteredMeter("elite", r).Mark(1337) + NewRegisteredTimer("second", r).Update(time.Second) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + NewRegisteredCounterFloat64("tau", r).Inc(1.57) + + w := new(strings.Builder) + (&OpenTSDBConfig{ + Registry: r, + DurationUnit: time.Millisecond, + Prefix: "pre", + }).writeRegistry(w, 978307200, "hal9000") + + wantB, err := os.ReadFile("./testdata/opentsb.want") + if err != nil { + t.Fatal(err) + } + if have, want := w.String(), string(wantB); have != want { + t.Errorf("\nhave:\n%v\nwant:\n%v\n", have, want) + t.Logf("have vs want:\n%v", findFirstDiffPos(have, want)) + } +} + +func findFirstDiffPos(a, b string) string { + yy := strings.Split(b, "\n") + for i, x := range strings.Split(a, "\n") { + if i >= len(yy) { + return fmt.Sprintf("have:%d: %s\nwant:%d: ", i, x, i) + } + if y := yy[i]; x != y { + return fmt.Sprintf("have:%d: %s\nwant:%d: %s", i, x, i, y) + } + } + return "" +} diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index b0f4a01a85..c07a38ca5f 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -157,8 +157,7 @@ func (g gatherer) Gather() ([]*dto.MetricFamily, error) { case metrics.ResettingTimer: snapshot := m.Snapshot() - vals := snapshot.Values() - count := uint64(len(vals)) + count := uint64(snapshot.Count()) if count == 0 { continue } @@ -167,7 +166,7 @@ func (g gatherer) Gather() ([]*dto.MetricFamily, error) { qs := make([]*dto.Quantile, len(pv)) for i := range pvShort { v := pv[i] - s := float64(ps[i]) + s := ps[i] qs[i] = &dto.Quantile{ Quantile: &v, Value: &s, diff --git a/metrics/prometheus/testdata/prometheus.want b/metrics/prometheus/testdata/prometheus.want new file mode 100644 index 0000000000..861c5f5cf0 --- /dev/null +++ b/metrics/prometheus/testdata/prometheus.want @@ -0,0 +1,70 @@ +# TYPE system_cpu_schedlatency_count counter +system_cpu_schedlatency_count 5645 + +# TYPE system_cpu_schedlatency summary +system_cpu_schedlatency {quantile="0.5"} 0 +system_cpu_schedlatency {quantile="0.75"} 7168 +system_cpu_schedlatency {quantile="0.95"} 1.6777216e+07 +system_cpu_schedlatency {quantile="0.99"} 2.9360128e+07 +system_cpu_schedlatency {quantile="0.999"} 3.3554432e+07 +system_cpu_schedlatency {quantile="0.9999"} 3.3554432e+07 + +# TYPE system_memory_pauses_count counter +system_memory_pauses_count 14 + +# TYPE system_memory_pauses summary +system_memory_pauses {quantile="0.5"} 32768 +system_memory_pauses {quantile="0.75"} 57344 +system_memory_pauses {quantile="0.95"} 196608 +system_memory_pauses {quantile="0.99"} 196608 +system_memory_pauses {quantile="0.999"} 196608 +system_memory_pauses {quantile="0.9999"} 196608 + +# TYPE test_counter gauge +test_counter 12345 + +# TYPE test_counter_float64 gauge +test_counter_float64 54321.98 + +# TYPE test_gauge gauge +test_gauge 23456 + +# TYPE test_gauge_float64 gauge +test_gauge_float64 34567.89 + +# TYPE test_gauge_info gauge +test_gauge_info {arch="amd64", commit="7caa2d8163ae3132c1c2d6978c76610caee2d949", os="linux", protocol_versions="64 65 66", version="1.10.18-unstable"} 1 + +# TYPE test_histogram_count counter +test_histogram_count 3 + +# TYPE test_histogram summary +test_histogram {quantile="0.5"} 2 +test_histogram {quantile="0.75"} 3 +test_histogram {quantile="0.95"} 3 +test_histogram {quantile="0.99"} 3 +test_histogram {quantile="0.999"} 3 +test_histogram {quantile="0.9999"} 3 + +# TYPE test_meter gauge +test_meter 0 + +# TYPE test_resetting_timer_count counter +test_resetting_timer_count 6 + +# TYPE test_resetting_timer summary +test_resetting_timer {quantile="0.50"} 1.25e+07 +test_resetting_timer {quantile="0.95"} 1.2e+08 +test_resetting_timer {quantile="0.99"} 1.2e+08 + +# TYPE test_timer_count counter +test_timer_count 6 + +# TYPE test_timer summary +test_timer {quantile="0.5"} 2.25e+07 +test_timer {quantile="0.75"} 4.8e+07 +test_timer {quantile="0.95"} 1.2e+08 +test_timer {quantile="0.99"} 1.2e+08 +test_timer {quantile="0.999"} 1.2e+08 +test_timer {quantile="0.9999"} 1.2e+08 + diff --git a/metrics/registry.go b/metrics/registry.go index ec6e37c54f..8bfbc08042 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "reflect" + "sort" "strings" "sync" ) @@ -47,17 +48,39 @@ type Registry interface { Unregister(string) } +type orderedRegistry struct { + StandardRegistry +} + +// Call the given function for each registered metric. +func (r *orderedRegistry) Each(f func(string, interface{})) { + var names []string + reg := r.registered() + for name := range reg { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + f(name, reg[name]) + } +} + +// NewRegistry creates a new registry. +func NewRegistry() Registry { + return new(StandardRegistry) +} + +// NewOrderedRegistry creates a new ordered registry (for testing). +func NewOrderedRegistry() Registry { + return new(orderedRegistry) +} + // The standard implementation of a Registry uses sync.map // of names to metrics. type StandardRegistry struct { metrics sync.Map } -// Create a new registry. -func NewRegistry() Registry { - return &StandardRegistry{} -} - // Call the given function for each registered metric. func (r *StandardRegistry) Each(f func(string, interface{})) { for name, i := range r.registered() { @@ -127,13 +150,13 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values := make(map[string]interface{}) switch metric := i.(type) { case Counter: - values["count"] = metric.Count() + values["count"] = metric.Snapshot().Count() case CounterFloat64: - values["count"] = metric.Count() + values["count"] = metric.Snapshot().Count() case Gauge: - values["value"] = metric.Value() + values["value"] = metric.Snapshot().Value() case GaugeFloat64: - values["value"] = metric.Value() + values["value"] = metric.Snapshot().Value() case Healthcheck: values["error"] = nil metric.Check() @@ -191,7 +214,7 @@ func (r *StandardRegistry) Unregister(name string) { func (r *StandardRegistry) loadOrRegister(name string, i interface{}) (interface{}, bool, bool) { switch i.(type) { - case Counter, CounterFloat64, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer, ResettingTimer: + case Counter, CounterFloat64, Gauge, GaugeFloat64, GaugeInfo, Healthcheck, Histogram, Meter, Timer, ResettingTimer: default: return nil, false, false } diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 7cc5cf14fe..75012dd4ac 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -85,11 +85,11 @@ func TestRegistryDuplicate(t *testing.T) { func TestRegistryGet(t *testing.T) { r := NewRegistry() r.Register("foo", NewCounter()) - if count := r.Get("foo").(Counter).Count(); count != 0 { + if count := r.Get("foo").(Counter).Snapshot().Count(); count != 0 { t.Fatal(count) } r.Get("foo").(Counter).Inc(1) - if count := r.Get("foo").(Counter).Count(); count != 1 { + if count := r.Get("foo").(Counter).Snapshot().Count(); count != 1 { t.Fatal(count) } } diff --git a/metrics/resetting_sample.go b/metrics/resetting_sample.go index 43c1129cd0..c38ffcd3ec 100644 --- a/metrics/resetting_sample.go +++ b/metrics/resetting_sample.go @@ -17,7 +17,7 @@ type resettingSample struct { } // Snapshot returns a read-only copy of the sample with the original reset. -func (rs *resettingSample) Snapshot() Sample { +func (rs *resettingSample) Snapshot() SampleSnapshot { s := rs.Sample.Snapshot() rs.Sample.Clear() return s diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 8e23c8eeea..6802e3fcea 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -1,22 +1,24 @@ package metrics import ( - "math" "sync" "time" - - "golang.org/x/exp/slices" ) // Initial slice capacity for the values stored in a ResettingTimer const InitialResettingTimerSliceCap = 10 +type ResettingTimerSnapshot interface { + Count() int + Mean() float64 + Max() int64 + Min() int64 + Percentiles([]float64) []float64 +} + // ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval. type ResettingTimer interface { - Values() []int64 - Snapshot() ResettingTimer - Percentiles([]float64) []int64 - Mean() float64 + Snapshot() ResettingTimerSnapshot Time(func()) Update(time.Duration) UpdateSince(time.Time) @@ -52,70 +54,40 @@ func NewResettingTimer() ResettingTimer { } // NilResettingTimer is a no-op ResettingTimer. -type NilResettingTimer struct { -} - -// Values is a no-op. -func (NilResettingTimer) Values() []int64 { return nil } - -// Snapshot is a no-op. -func (NilResettingTimer) Snapshot() ResettingTimer { - return &ResettingTimerSnapshot{ - values: []int64{}, - } -} - -// Time is a no-op. -func (NilResettingTimer) Time(f func()) { f() } - -// Update is a no-op. -func (NilResettingTimer) Update(time.Duration) {} - -// Percentiles panics. -func (NilResettingTimer) Percentiles([]float64) []int64 { - panic("Percentiles called on a NilResettingTimer") -} - -// Mean panics. -func (NilResettingTimer) Mean() float64 { - panic("Mean called on a NilResettingTimer") -} - -// UpdateSince is a no-op. -func (NilResettingTimer) UpdateSince(time.Time) {} +type NilResettingTimer struct{} + +func (NilResettingTimer) Values() []int64 { return nil } +func (n NilResettingTimer) Snapshot() ResettingTimerSnapshot { return n } +func (NilResettingTimer) Time(f func()) { f() } +func (NilResettingTimer) Update(time.Duration) {} +func (NilResettingTimer) Percentiles([]float64) []float64 { return nil } +func (NilResettingTimer) Mean() float64 { return 0.0 } +func (NilResettingTimer) Max() int64 { return 0 } +func (NilResettingTimer) Min() int64 { return 0 } +func (NilResettingTimer) UpdateSince(time.Time) {} +func (NilResettingTimer) Count() int { return 0 } // StandardResettingTimer is the standard implementation of a ResettingTimer. // and Meter. type StandardResettingTimer struct { values []int64 - mutex sync.Mutex -} + sum int64 // sum is a running count of the total sum, used later to calculate mean -// Values returns a slice with all measurements. -func (t *StandardResettingTimer) Values() []int64 { - return t.values + mutex sync.Mutex } // Snapshot resets the timer and returns a read-only copy of its contents. -func (t *StandardResettingTimer) Snapshot() ResettingTimer { +func (t *StandardResettingTimer) Snapshot() ResettingTimerSnapshot { t.mutex.Lock() defer t.mutex.Unlock() - currentValues := t.values - t.values = make([]int64, 0, InitialResettingTimerSliceCap) - - return &ResettingTimerSnapshot{ - values: currentValues, + snapshot := &resettingTimerSnapshot{} + if len(t.values) > 0 { + snapshot.mean = float64(t.sum) / float64(len(t.values)) + snapshot.values = t.values + t.values = make([]int64, 0, InitialResettingTimerSliceCap) } -} - -// Percentiles panics. -func (t *StandardResettingTimer) Percentiles([]float64) []int64 { - panic("Percentiles called on a StandardResettingTimer") -} - -// Mean panics. -func (t *StandardResettingTimer) Mean() float64 { - panic("Mean called on a StandardResettingTimer") + t.sum = 0 + return snapshot } // Record the duration of the execution of the given function. @@ -130,106 +102,70 @@ func (t *StandardResettingTimer) Update(d time.Duration) { t.mutex.Lock() defer t.mutex.Unlock() t.values = append(t.values, int64(d)) + t.sum += int64(d) } // Record the duration of an event that started at a time and ends now. func (t *StandardResettingTimer) UpdateSince(ts time.Time) { - t.mutex.Lock() - defer t.mutex.Unlock() - t.values = append(t.values, int64(time.Since(ts))) + t.Update(time.Since(ts)) } -// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer. -type ResettingTimerSnapshot struct { +// resettingTimerSnapshot is a point-in-time copy of another ResettingTimer. +type resettingTimerSnapshot struct { values []int64 mean float64 - thresholdBoundaries []int64 + max int64 + min int64 + thresholdBoundaries []float64 calculated bool } -// Snapshot returns the snapshot. -func (t *ResettingTimerSnapshot) Snapshot() ResettingTimer { return t } - -// Time panics. -func (*ResettingTimerSnapshot) Time(func()) { - panic("Time called on a ResettingTimerSnapshot") -} - -// Update panics. -func (*ResettingTimerSnapshot) Update(time.Duration) { - panic("Update called on a ResettingTimerSnapshot") -} - -// UpdateSince panics. -func (*ResettingTimerSnapshot) UpdateSince(time.Time) { - panic("UpdateSince called on a ResettingTimerSnapshot") -} - -// Values returns all values from snapshot. -func (t *ResettingTimerSnapshot) Values() []int64 { - return t.values +// Count return the length of the values from snapshot. +func (t *resettingTimerSnapshot) Count() int { + return len(t.values) } // Percentiles returns the boundaries for the input percentiles. -func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []int64 { +// note: this method is not thread safe +func (t *resettingTimerSnapshot) Percentiles(percentiles []float64) []float64 { t.calc(percentiles) - return t.thresholdBoundaries } // Mean returns the mean of the snapshotted values -func (t *ResettingTimerSnapshot) Mean() float64 { +// note: this method is not thread safe +func (t *resettingTimerSnapshot) Mean() float64 { if !t.calculated { - t.calc([]float64{}) + t.calc(nil) } return t.mean } -func (t *ResettingTimerSnapshot) calc(percentiles []float64) { - slices.Sort(t.values) - - count := len(t.values) - if count > 0 { - min := t.values[0] - max := t.values[count-1] - - cumulativeValues := make([]int64, count) - cumulativeValues[0] = min - for i := 1; i < count; i++ { - cumulativeValues[i] = t.values[i] + cumulativeValues[i-1] - } - - t.thresholdBoundaries = make([]int64, len(percentiles)) - - thresholdBoundary := max - - for i, pct := range percentiles { - if count > 1 { - var abs float64 - if pct >= 0 { - abs = pct - } else { - abs = 100 + pct - } - // poor man's math.Round(x): - // math.Floor(x + 0.5) - indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5)) - if pct >= 0 && indexOfPerc > 0 { - indexOfPerc -= 1 // index offset=0 - } - thresholdBoundary = t.values[indexOfPerc] - } - - t.thresholdBoundaries[i] = thresholdBoundary - } - - sum := cumulativeValues[count-1] - t.mean = float64(sum) / float64(count) - } else { - t.thresholdBoundaries = make([]int64, len(percentiles)) - t.mean = 0 +// Max returns the max of the snapshotted values +// note: this method is not thread safe +func (t *resettingTimerSnapshot) Max() int64 { + if !t.calculated { + t.calc(nil) + } + return t.max +} + +// Min returns the min of the snapshotted values +// note: this method is not thread safe +func (t *resettingTimerSnapshot) Min() int64 { + if !t.calculated { + t.calc(nil) } + return t.min +} - t.calculated = true +func (t *resettingTimerSnapshot) calc(percentiles []float64) { + scores := CalculatePercentiles(t.values, percentiles) + t.thresholdBoundaries = scores + if len(t.values) == 0 { + return + } + t.min = t.values[0] + t.max = t.values[len(t.values)-1] } diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go index 77c49dc386..4571fc8eb0 100644 --- a/metrics/resetting_timer_test.go +++ b/metrics/resetting_timer_test.go @@ -10,9 +10,9 @@ func TestResettingTimer(t *testing.T) { values []int64 start int end int - wantP50 int64 - wantP95 int64 - wantP99 int64 + wantP50 float64 + wantP95 float64 + wantP99 float64 wantMean float64 wantMin int64 wantMax int64 @@ -21,14 +21,14 @@ func TestResettingTimer(t *testing.T) { values: []int64{}, start: 1, end: 11, - wantP50: 5, wantP95: 10, wantP99: 10, + wantP50: 5.5, wantP95: 10, wantP99: 10, wantMin: 1, wantMax: 10, wantMean: 5.5, }, { values: []int64{}, start: 1, end: 101, - wantP50: 50, wantP95: 95, wantP99: 99, + wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, wantMin: 1, wantMax: 100, wantMean: 50.5, }, { @@ -56,11 +56,11 @@ func TestResettingTimer(t *testing.T) { values: []int64{1, 10}, start: 0, end: 0, - wantP50: 1, wantP95: 10, wantP99: 10, + wantP50: 5.5, wantP95: 10, wantP99: 10, wantMin: 1, wantMax: 10, wantMean: 5.5, }, } - for ind, tt := range tests { + for i, tt := range tests { timer := NewResettingTimer() for i := tt.start; i < tt.end; i++ { @@ -70,37 +70,27 @@ func TestResettingTimer(t *testing.T) { for _, v := range tt.values { timer.Update(time.Duration(v)) } - snap := timer.Snapshot() - ps := snap.Percentiles([]float64{50, 95, 99}) - - val := snap.Values() + ps := snap.Percentiles([]float64{0.50, 0.95, 0.99}) - if len(val) > 0 { - if tt.wantMin != val[0] { - t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) - } - - if tt.wantMax != val[len(val)-1] { - t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) - } + if have, want := snap.Min(), tt.wantMin; have != want { + t.Fatalf("%d: min: have %d, want %d", i, have, want) } - - if tt.wantMean != snap.Mean() { - t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + if have, want := snap.Max(), tt.wantMax; have != want { + t.Fatalf("%d: max: have %d, want %d", i, have, want) } - - if tt.wantP50 != ps[0] { - t.Fatalf("%d: p50: got %d, want %d", ind, ps[0], tt.wantP50) + if have, want := snap.Mean(), tt.wantMean; have != want { + t.Fatalf("%d: mean: have %v, want %v", i, have, want) } - - if tt.wantP95 != ps[1] { - t.Fatalf("%d: p95: got %d, want %d", ind, ps[1], tt.wantP95) + if have, want := ps[0], tt.wantP50; have != want { + t.Errorf("%d: p50: have %v, want %v", i, have, want) } - - if tt.wantP99 != ps[2] { - t.Fatalf("%d: p99: got %d, want %d", ind, ps[2], tt.wantP99) + if have, want := ps[1], tt.wantP95; have != want { + t.Errorf("%d: p95: have %v, want %v", i, have, want) + } + if have, want := ps[2], tt.wantP99; have != want { + t.Errorf("%d: p99: have %v, want %v", i, have, want) } } } @@ -110,11 +100,11 @@ func TestResettingTimerWithFivePercentiles(t *testing.T) { values []int64 start int end int - wantP05 int64 - wantP20 int64 - wantP50 int64 - wantP95 int64 - wantP99 int64 + wantP05 float64 + wantP20 float64 + wantP50 float64 + wantP95 float64 + wantP99 float64 wantMean float64 wantMin int64 wantMax int64 @@ -123,14 +113,14 @@ func TestResettingTimerWithFivePercentiles(t *testing.T) { values: []int64{}, start: 1, end: 11, - wantP05: 1, wantP20: 2, wantP50: 5, wantP95: 10, wantP99: 10, + wantP05: 1, wantP20: 2.2, wantP50: 5.5, wantP95: 10, wantP99: 10, wantMin: 1, wantMax: 10, wantMean: 5.5, }, { values: []int64{}, start: 1, end: 101, - wantP05: 5, wantP20: 20, wantP50: 50, wantP95: 95, wantP99: 99, + wantP05: 5.050000000000001, wantP20: 20.200000000000003, wantP50: 50.5, wantP95: 95.94999999999999, wantP99: 99.99, wantMin: 1, wantMax: 100, wantMean: 50.5, }, { @@ -158,7 +148,7 @@ func TestResettingTimerWithFivePercentiles(t *testing.T) { values: []int64{1, 10}, start: 0, end: 0, - wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 10, wantP99: 10, + wantP05: 1, wantP20: 1, wantP50: 5.5, wantP95: 10, wantP99: 10, wantMin: 1, wantMax: 10, wantMean: 5.5, }, } @@ -175,42 +165,33 @@ func TestResettingTimerWithFivePercentiles(t *testing.T) { snap := timer.Snapshot() - ps := snap.Percentiles([]float64{5, 20, 50, 95, 99}) - - val := snap.Values() + ps := snap.Percentiles([]float64{0.05, 0.20, 0.50, 0.95, 0.99}) - if len(val) > 0 { - if tt.wantMin != val[0] { - t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) - } + if tt.wantMin != snap.Min() { + t.Errorf("%d: min: got %d, want %d", ind, snap.Min(), tt.wantMin) + } - if tt.wantMax != val[len(val)-1] { - t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) - } + if tt.wantMax != snap.Max() { + t.Errorf("%d: max: got %d, want %d", ind, snap.Max(), tt.wantMax) } if tt.wantMean != snap.Mean() { - t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + t.Errorf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) } - if tt.wantP05 != ps[0] { - t.Fatalf("%d: p05: got %d, want %d", ind, ps[0], tt.wantP05) + t.Errorf("%d: p05: got %v, want %v", ind, ps[0], tt.wantP05) } - if tt.wantP20 != ps[1] { - t.Fatalf("%d: p20: got %d, want %d", ind, ps[1], tt.wantP20) + t.Errorf("%d: p20: got %v, want %v", ind, ps[1], tt.wantP20) } - if tt.wantP50 != ps[2] { - t.Fatalf("%d: p50: got %d, want %d", ind, ps[2], tt.wantP50) + t.Errorf("%d: p50: got %v, want %v", ind, ps[2], tt.wantP50) } - if tt.wantP95 != ps[3] { - t.Fatalf("%d: p95: got %d, want %d", ind, ps[3], tt.wantP95) + t.Errorf("%d: p95: got %v, want %v", ind, ps[3], tt.wantP95) } - if tt.wantP99 != ps[4] { - t.Fatalf("%d: p99: got %d, want %d", ind, ps[4], tt.wantP99) + t.Errorf("%d: p99: got %v, want %v", ind, ps[4], tt.wantP99) } } } diff --git a/metrics/sample.go b/metrics/sample.go index 252a878f58..5398dd42d5 100644 --- a/metrics/sample.go +++ b/metrics/sample.go @@ -11,10 +11,7 @@ import ( const rescaleThreshold = time.Hour -// Samples maintain a statistically-significant selection of values from -// a stream. -type Sample interface { - Clear() +type SampleSnapshot interface { Count() int64 Max() int64 Mean() float64 @@ -22,14 +19,19 @@ type Sample interface { Percentile(float64) float64 Percentiles([]float64) []float64 Size() int - Snapshot() Sample StdDev() float64 Sum() int64 - Update(int64) - Values() []int64 Variance() float64 } +// Samples maintain a statistically-significant selection of values from +// a stream. +type Sample interface { + Snapshot() SampleSnapshot + Clear() + Update(int64) +} + // ExpDecaySample is an exponentially-decaying sample using a forward-decaying // priority reservoir. See Cormode et al's "Forward Decay: A Practical Time // Decay Model for Streaming Systems". @@ -77,72 +79,29 @@ func (s *ExpDecaySample) Clear() { s.values.Clear() } -// Count returns the number of samples recorded, which may exceed the -// reservoir size. -func (s *ExpDecaySample) Count() int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return s.count -} - -// Max returns the maximum value in the sample, which may not be the maximum -// value ever to be part of the sample. -func (s *ExpDecaySample) Max() int64 { - return SampleMax(s.Values()) -} - -// Mean returns the mean of the values in the sample. -func (s *ExpDecaySample) Mean() float64 { - return SampleMean(s.Values()) -} - -// Min returns the minimum value in the sample, which may not be the minimum -// value ever to be part of the sample. -func (s *ExpDecaySample) Min() int64 { - return SampleMin(s.Values()) -} - -// Percentile returns an arbitrary percentile of values in the sample. -func (s *ExpDecaySample) Percentile(p float64) float64 { - return SamplePercentile(s.Values(), p) -} - -// Percentiles returns a slice of arbitrary percentiles of values in the -// sample. -func (s *ExpDecaySample) Percentiles(ps []float64) []float64 { - return SamplePercentiles(s.Values(), ps) -} - -// Size returns the size of the sample, which is at most the reservoir size. -func (s *ExpDecaySample) Size() int { - s.mutex.Lock() - defer s.mutex.Unlock() - return s.values.Size() -} - // Snapshot returns a read-only copy of the sample. -func (s *ExpDecaySample) Snapshot() Sample { +func (s *ExpDecaySample) Snapshot() SampleSnapshot { s.mutex.Lock() defer s.mutex.Unlock() - vals := s.values.Values() - values := make([]int64, len(vals)) - for i, v := range vals { - values[i] = v.v - } - return &SampleSnapshot{ - count: s.count, - values: values, + var ( + samples = s.values.Values() + values = make([]int64, len(samples)) + max int64 = math.MinInt64 + min int64 = math.MaxInt64 + sum int64 + ) + for i, item := range samples { + v := item.v + values[i] = v + sum += v + if v > max { + max = v + } + if v < min { + min = v + } } -} - -// StdDev returns the standard deviation of the values in the sample. -func (s *ExpDecaySample) StdDev() float64 { - return SampleStdDev(s.Values()) -} - -// Sum returns the sum of the values in the sample. -func (s *ExpDecaySample) Sum() int64 { - return SampleSum(s.Values()) + return newSampleSnapshotPrecalculated(s.count, values, min, max, sum) } // Update samples a new value. @@ -150,23 +109,6 @@ func (s *ExpDecaySample) Update(v int64) { s.update(time.Now(), v) } -// Values returns a copy of the values in the sample. -func (s *ExpDecaySample) Values() []int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - vals := s.values.Values() - values := make([]int64, len(vals)) - for i, v := range vals { - values[i] = v.v - } - return values -} - -// Variance returns the variance of the values in the sample. -func (s *ExpDecaySample) Variance() float64 { - return SampleVariance(s.Values()) -} - // update samples a new value at a particular timestamp. This is a method all // its own to facilitate testing. func (s *ExpDecaySample) update(t time.Time, v int64) { @@ -202,207 +144,160 @@ func (s *ExpDecaySample) update(t time.Time, v int64) { // NilSample is a no-op Sample. type NilSample struct{} -// Clear is a no-op. -func (NilSample) Clear() {} - -// Count is a no-op. -func (NilSample) Count() int64 { return 0 } - -// Max is a no-op. -func (NilSample) Max() int64 { return 0 } - -// Mean is a no-op. -func (NilSample) Mean() float64 { return 0.0 } - -// Min is a no-op. -func (NilSample) Min() int64 { return 0 } - -// Percentile is a no-op. -func (NilSample) Percentile(p float64) float64 { return 0.0 } - -// Percentiles is a no-op. -func (NilSample) Percentiles(ps []float64) []float64 { - return make([]float64, len(ps)) -} - -// Size is a no-op. -func (NilSample) Size() int { return 0 } - -// Sample is a no-op. -func (NilSample) Snapshot() Sample { return NilSample{} } - -// StdDev is a no-op. -func (NilSample) StdDev() float64 { return 0.0 } - -// Sum is a no-op. -func (NilSample) Sum() int64 { return 0 } - -// Update is a no-op. -func (NilSample) Update(v int64) {} - -// Values is a no-op. -func (NilSample) Values() []int64 { return []int64{} } - -// Variance is a no-op. -func (NilSample) Variance() float64 { return 0.0 } - -// SampleMax returns the maximum value of the slice of int64. -func SampleMax(values []int64) int64 { - if len(values) == 0 { - return 0 - } - var max int64 = math.MinInt64 - for _, v := range values { - if max < v { - max = v - } - } - return max -} - -// SampleMean returns the mean value of the slice of int64. -func SampleMean(values []int64) float64 { - if len(values) == 0 { - return 0.0 - } - return float64(SampleSum(values)) / float64(len(values)) -} - -// SampleMin returns the minimum value of the slice of int64. -func SampleMin(values []int64) int64 { - if len(values) == 0 { - return 0 - } - var min int64 = math.MaxInt64 - for _, v := range values { - if min > v { - min = v - } - } - return min -} +func (NilSample) Clear() {} +func (NilSample) Snapshot() SampleSnapshot { return (*emptySnapshot)(nil) } +func (NilSample) Update(v int64) {} // SamplePercentiles returns an arbitrary percentile of the slice of int64. func SamplePercentile(values []int64, p float64) float64 { - return SamplePercentiles(values, []float64{p})[0] + return CalculatePercentiles(values, []float64{p})[0] } -// SamplePercentiles returns a slice of arbitrary percentiles of the slice of -// int64. -func SamplePercentiles(values []int64, ps []float64) []float64 { +// CalculatePercentiles returns a slice of arbitrary percentiles of the slice of +// int64. This method returns interpolated results, so e.g if there are only two +// values, [0, 10], a 50% percentile will land between them. +// +// Note: As a side-effect, this method will also sort the slice of values. +// Note2: The input format for percentiles is NOT percent! To express 50%, use 0.5, not 50. +func CalculatePercentiles(values []int64, ps []float64) []float64 { scores := make([]float64, len(ps)) size := len(values) - if size > 0 { - slices.Sort(values) - for i, p := range ps { - pos := p * float64(size+1) - if pos < 1.0 { - scores[i] = float64(values[0]) - } else if pos >= float64(size) { - scores[i] = float64(values[size-1]) - } else { - lower := float64(values[int(pos)-1]) - upper := float64(values[int(pos)]) - scores[i] = lower + (pos-math.Floor(pos))*(upper-lower) - } + if size == 0 { + return scores + } + slices.Sort(values) + for i, p := range ps { + pos := p * float64(size+1) + + if pos < 1.0 { + scores[i] = float64(values[0]) + } else if pos >= float64(size) { + scores[i] = float64(values[size-1]) + } else { + lower := float64(values[int(pos)-1]) + upper := float64(values[int(pos)]) + scores[i] = lower + (pos-math.Floor(pos))*(upper-lower) } } return scores } -// SampleSnapshot is a read-only copy of another Sample. -type SampleSnapshot struct { +// sampleSnapshot is a read-only copy of another Sample. +type sampleSnapshot struct { count int64 values []int64 + + max int64 + min int64 + mean float64 + sum int64 + variance float64 } -func NewSampleSnapshot(count int64, values []int64) *SampleSnapshot { - return &SampleSnapshot{ +// newSampleSnapshotPrecalculated creates a read-only sampleSnapShot, using +// precalculated sums to avoid iterating the values +func newSampleSnapshotPrecalculated(count int64, values []int64, min, max, sum int64) *sampleSnapshot { + if len(values) == 0 { + return &sampleSnapshot{ + count: count, + values: values, + } + } + return &sampleSnapshot{ count: count, values: values, + max: max, + min: min, + mean: float64(sum) / float64(len(values)), + sum: sum, } } -// Clear panics. -func (*SampleSnapshot) Clear() { - panic("Clear called on a SampleSnapshot") +// newSampleSnapshot creates a read-only sampleSnapShot, and calculates some +// numbers. +func newSampleSnapshot(count int64, values []int64) *sampleSnapshot { + var ( + max int64 = math.MinInt64 + min int64 = math.MaxInt64 + sum int64 + ) + for _, v := range values { + sum += v + if v > max { + max = v + } + if v < min { + min = v + } + } + return newSampleSnapshotPrecalculated(count, values, min, max, sum) } // Count returns the count of inputs at the time the snapshot was taken. -func (s *SampleSnapshot) Count() int64 { return s.count } +func (s *sampleSnapshot) Count() int64 { return s.count } // Max returns the maximal value at the time the snapshot was taken. -func (s *SampleSnapshot) Max() int64 { return SampleMax(s.values) } +func (s *sampleSnapshot) Max() int64 { return s.max } // Mean returns the mean value at the time the snapshot was taken. -func (s *SampleSnapshot) Mean() float64 { return SampleMean(s.values) } +func (s *sampleSnapshot) Mean() float64 { return s.mean } // Min returns the minimal value at the time the snapshot was taken. -func (s *SampleSnapshot) Min() int64 { return SampleMin(s.values) } +func (s *sampleSnapshot) Min() int64 { return s.min } // Percentile returns an arbitrary percentile of values at the time the // snapshot was taken. -func (s *SampleSnapshot) Percentile(p float64) float64 { +func (s *sampleSnapshot) Percentile(p float64) float64 { return SamplePercentile(s.values, p) } // Percentiles returns a slice of arbitrary percentiles of values at the time // the snapshot was taken. -func (s *SampleSnapshot) Percentiles(ps []float64) []float64 { - return SamplePercentiles(s.values, ps) +func (s *sampleSnapshot) Percentiles(ps []float64) []float64 { + return CalculatePercentiles(s.values, ps) } // Size returns the size of the sample at the time the snapshot was taken. -func (s *SampleSnapshot) Size() int { return len(s.values) } +func (s *sampleSnapshot) Size() int { return len(s.values) } // Snapshot returns the snapshot. -func (s *SampleSnapshot) Snapshot() Sample { return s } +func (s *sampleSnapshot) Snapshot() SampleSnapshot { return s } // StdDev returns the standard deviation of values at the time the snapshot was // taken. -func (s *SampleSnapshot) StdDev() float64 { return SampleStdDev(s.values) } +func (s *sampleSnapshot) StdDev() float64 { + if s.variance == 0.0 { + s.variance = SampleVariance(s.mean, s.values) + } + return math.Sqrt(s.variance) +} // Sum returns the sum of values at the time the snapshot was taken. -func (s *SampleSnapshot) Sum() int64 { return SampleSum(s.values) } - -// Update panics. -func (*SampleSnapshot) Update(int64) { - panic("Update called on a SampleSnapshot") -} +func (s *sampleSnapshot) Sum() int64 { return s.sum } // Values returns a copy of the values in the sample. -func (s *SampleSnapshot) Values() []int64 { +func (s *sampleSnapshot) Values() []int64 { values := make([]int64, len(s.values)) copy(values, s.values) return values } // Variance returns the variance of values at the time the snapshot was taken. -func (s *SampleSnapshot) Variance() float64 { return SampleVariance(s.values) } - -// SampleStdDev returns the standard deviation of the slice of int64. -func SampleStdDev(values []int64) float64 { - return math.Sqrt(SampleVariance(values)) -} - -// SampleSum returns the sum of the slice of int64. -func SampleSum(values []int64) int64 { - var sum int64 - for _, v := range values { - sum += v +func (s *sampleSnapshot) Variance() float64 { + if s.variance == 0.0 { + s.variance = SampleVariance(s.mean, s.values) } - return sum + return s.variance } // SampleVariance returns the variance of the slice of int64. -func SampleVariance(values []int64) float64 { +func SampleVariance(mean float64, values []int64) float64 { if len(values) == 0 { return 0.0 } - m := SampleMean(values) var sum float64 for _, v := range values { - d := float64(v) - m + d := float64(v) - mean sum += d * d } return sum / float64(len(values)) @@ -445,83 +340,14 @@ func (s *UniformSample) Clear() { s.values = make([]int64, 0, s.reservoirSize) } -// Count returns the number of samples recorded, which may exceed the -// reservoir size. -func (s *UniformSample) Count() int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return s.count -} - -// Max returns the maximum value in the sample, which may not be the maximum -// value ever to be part of the sample. -func (s *UniformSample) Max() int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleMax(s.values) -} - -// Mean returns the mean of the values in the sample. -func (s *UniformSample) Mean() float64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleMean(s.values) -} - -// Min returns the minimum value in the sample, which may not be the minimum -// value ever to be part of the sample. -func (s *UniformSample) Min() int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleMin(s.values) -} - -// Percentile returns an arbitrary percentile of values in the sample. -func (s *UniformSample) Percentile(p float64) float64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SamplePercentile(s.values, p) -} - -// Percentiles returns a slice of arbitrary percentiles of values in the -// sample. -func (s *UniformSample) Percentiles(ps []float64) []float64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SamplePercentiles(s.values, ps) -} - -// Size returns the size of the sample, which is at most the reservoir size. -func (s *UniformSample) Size() int { - s.mutex.Lock() - defer s.mutex.Unlock() - return len(s.values) -} - // Snapshot returns a read-only copy of the sample. -func (s *UniformSample) Snapshot() Sample { +func (s *UniformSample) Snapshot() SampleSnapshot { s.mutex.Lock() - defer s.mutex.Unlock() values := make([]int64, len(s.values)) copy(values, s.values) - return &SampleSnapshot{ - count: s.count, - values: values, - } -} - -// StdDev returns the standard deviation of the values in the sample. -func (s *UniformSample) StdDev() float64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleStdDev(s.values) -} - -// Sum returns the sum of the values in the sample. -func (s *UniformSample) Sum() int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleSum(s.values) + count := s.count + s.mutex.Unlock() + return newSampleSnapshot(count, values) } // Update samples a new value. @@ -544,22 +370,6 @@ func (s *UniformSample) Update(v int64) { } } -// Values returns a copy of the values in the sample. -func (s *UniformSample) Values() []int64 { - s.mutex.Lock() - defer s.mutex.Unlock() - values := make([]int64, len(s.values)) - copy(values, s.values) - return values -} - -// Variance returns the variance of the values in the sample. -func (s *UniformSample) Variance() float64 { - s.mutex.Lock() - defer s.mutex.Unlock() - return SampleVariance(s.values) -} - // expDecaySample represents an individual sample in a heap. type expDecaySample struct { k float64 diff --git a/metrics/sample_test.go b/metrics/sample_test.go index 3ae128d56f..7967357055 100644 --- a/metrics/sample_test.go +++ b/metrics/sample_test.go @@ -8,28 +8,36 @@ import ( "time" ) +const epsilonPercentile = .00000000001 + // Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively // expensive computations like Variance, the cost of copying the Sample, as // approximated by a make and copy, is much greater than the cost of the // computation for small samples and only slightly less for large samples. func BenchmarkCompute1000(b *testing.B) { s := make([]int64, 1000) + var sum int64 for i := 0; i < len(s); i++ { s[i] = int64(i) + sum += int64(i) } + mean := float64(sum) / float64(len(s)) b.ResetTimer() for i := 0; i < b.N; i++ { - SampleVariance(s) + SampleVariance(mean, s) } } func BenchmarkCompute1000000(b *testing.B) { s := make([]int64, 1000000) + var sum int64 for i := 0; i < len(s); i++ { s[i] = int64(i) + sum += int64(i) } + mean := float64(sum) / float64(len(s)) b.ResetTimer() for i := 0; i < b.N; i++ { - SampleVariance(s) + SampleVariance(mean, s) } } func BenchmarkCopy1000(b *testing.B) { @@ -79,65 +87,42 @@ func BenchmarkUniformSample1028(b *testing.B) { benchmarkSample(b, NewUniformSample(1028)) } -func TestExpDecaySample10(t *testing.T) { - s := NewExpDecaySample(100, 0.99) - for i := 0; i < 10; i++ { - s.Update(int64(i)) - } - if size := s.Count(); size != 10 { - t.Errorf("s.Count(): 10 != %v\n", size) - } - if size := s.Size(); size != 10 { - t.Errorf("s.Size(): 10 != %v\n", size) - } - if l := len(s.Values()); l != 10 { - t.Errorf("len(s.Values()): 10 != %v\n", l) - } - for _, v := range s.Values() { - if v > 10 || v < 0 { - t.Errorf("out of range [0, 10): %v\n", v) - } +func min(a, b int) int { + if a < b { + return a } + return b } -func TestExpDecaySample100(t *testing.T) { - s := NewExpDecaySample(1000, 0.01) - for i := 0; i < 100; i++ { - s.Update(int64(i)) - } - if size := s.Count(); size != 100 { - t.Errorf("s.Count(): 100 != %v\n", size) - } - if size := s.Size(); size != 100 { - t.Errorf("s.Size(): 100 != %v\n", size) - } - if l := len(s.Values()); l != 100 { - t.Errorf("len(s.Values()): 100 != %v\n", l) - } - for _, v := range s.Values() { - if v > 100 || v < 0 { - t.Errorf("out of range [0, 100): %v\n", v) +func TestExpDecaySample(t *testing.T) { + for _, tc := range []struct { + reservoirSize int + alpha float64 + updates int + }{ + {100, 0.99, 10}, + {1000, 0.01, 100}, + {100, 0.99, 1000}, + } { + sample := NewExpDecaySample(tc.reservoirSize, tc.alpha) + for i := 0; i < tc.updates; i++ { + sample.Update(int64(i)) } - } -} - -func TestExpDecaySample1000(t *testing.T) { - s := NewExpDecaySample(100, 0.99) - for i := 0; i < 1000; i++ { - s.Update(int64(i)) - } - if size := s.Count(); size != 1000 { - t.Errorf("s.Count(): 1000 != %v\n", size) - } - if size := s.Size(); size != 100 { - t.Errorf("s.Size(): 100 != %v\n", size) - } - if l := len(s.Values()); l != 100 { - t.Errorf("len(s.Values()): 100 != %v\n", l) - } - for _, v := range s.Values() { - if v > 1000 || v < 0 { - t.Errorf("out of range [0, 1000): %v\n", v) + snap := sample.Snapshot() + if have, want := int(snap.Count()), tc.updates; have != want { + t.Errorf("have %d want %d", have, want) + } + if have, want := snap.Size(), min(tc.updates, tc.reservoirSize); have != want { + t.Errorf("have %d want %d", have, want) + } + values := snap.(*sampleSnapshot).values + if have, want := len(values), min(tc.updates, tc.reservoirSize); have != want { + t.Errorf("have %d want %d", have, want) + } + for _, v := range values { + if v > int64(tc.updates) || v < 0 { + t.Errorf("out of range [0, %d): %v", tc.updates, v) + } } } } @@ -147,15 +132,16 @@ func TestExpDecaySample1000(t *testing.T) { // The priority becomes +Inf quickly after starting if this is done, // effectively freezing the set of samples until a rescale step happens. func TestExpDecaySampleNanosecondRegression(t *testing.T) { - s := NewExpDecaySample(100, 0.99) + sw := NewExpDecaySample(100, 0.99) for i := 0; i < 100; i++ { - s.Update(10) + sw.Update(10) } time.Sleep(1 * time.Millisecond) for i := 0; i < 100; i++ { - s.Update(20) + sw.Update(20) } - v := s.Values() + s := sw.Snapshot() + v := s.(*sampleSnapshot).values avg := float64(0) for i := 0; i < len(v); i++ { avg += float64(v[i]) @@ -194,24 +180,27 @@ func TestExpDecaySampleStatistics(t *testing.T) { for i := 1; i <= 10000; i++ { s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) } - testExpDecaySampleStatistics(t, s) + testExpDecaySampleStatistics(t, s.Snapshot()) } func TestUniformSample(t *testing.T) { - s := NewUniformSample(100) + sw := NewUniformSample(100) for i := 0; i < 1000; i++ { - s.Update(int64(i)) + sw.Update(int64(i)) } + s := sw.Snapshot() if size := s.Count(); size != 1000 { t.Errorf("s.Count(): 1000 != %v\n", size) } if size := s.Size(); size != 100 { t.Errorf("s.Size(): 100 != %v\n", size) } - if l := len(s.Values()); l != 100 { + values := s.(*sampleSnapshot).values + + if l := len(values); l != 100 { t.Errorf("len(s.Values()): 100 != %v\n", l) } - for _, v := range s.Values() { + for _, v := range values { if v > 1000 || v < 0 { t.Errorf("out of range [0, 100): %v\n", v) } @@ -219,12 +208,13 @@ func TestUniformSample(t *testing.T) { } func TestUniformSampleIncludesTail(t *testing.T) { - s := NewUniformSample(100) + sw := NewUniformSample(100) max := 100 for i := 0; i < max; i++ { - s.Update(int64(i)) + sw.Update(int64(i)) } - v := s.Values() + s := sw.Snapshot() + v := s.(*sampleSnapshot).values sum := 0 exp := (max - 1) * max / 2 for i := 0; i < len(v); i++ { @@ -250,7 +240,7 @@ func TestUniformSampleStatistics(t *testing.T) { for i := 1; i <= 10000; i++ { s.Update(int64(i)) } - testUniformSampleStatistics(t, s) + testUniformSampleStatistics(t, s.Snapshot()) } func benchmarkSample(b *testing.B, s Sample) { @@ -267,7 +257,7 @@ func benchmarkSample(b *testing.B, s Sample) { b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N) } -func testExpDecaySampleStatistics(t *testing.T, s Sample) { +func testExpDecaySampleStatistics(t *testing.T, s SampleSnapshot) { if count := s.Count(); count != 10000 { t.Errorf("s.Count(): 10000 != %v\n", count) } @@ -295,7 +285,7 @@ func testExpDecaySampleStatistics(t *testing.T, s Sample) { } } -func testUniformSampleStatistics(t *testing.T, s Sample) { +func testUniformSampleStatistics(t *testing.T, s SampleSnapshot) { if count := s.Count(); count != 10000 { t.Errorf("s.Count(): 10000 != %v\n", count) } @@ -349,8 +339,22 @@ func TestUniformSampleConcurrentUpdateCount(t *testing.T) { } }() for i := 0; i < 1000; i++ { - s.Count() + s.Snapshot().Count() time.Sleep(5 * time.Millisecond) } quit <- struct{}{} } + +func BenchmarkCalculatePercentiles(b *testing.B) { + pss := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999} + var vals []int64 + for i := 0; i < 1000; i++ { + vals = append(vals, int64(rand.Int31())) + } + v := make([]int64, len(vals)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + copy(v, vals) + _ = CalculatePercentiles(v, pss) + } +} diff --git a/metrics/syslog.go b/metrics/syslog.go index f23b07e199..fd856d6973 100644 --- a/metrics/syslog.go +++ b/metrics/syslog.go @@ -16,13 +16,15 @@ func Syslog(r Registry, d time.Duration, w *syslog.Writer) { r.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: - w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Count())) + w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Snapshot().Count())) case CounterFloat64: - w.Info(fmt.Sprintf("counter %s: count: %f", name, metric.Count())) + w.Info(fmt.Sprintf("counter %s: count: %f", name, metric.Snapshot().Count())) case Gauge: - w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Value())) + w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Snapshot().Value())) case GaugeFloat64: - w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Value())) + w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Snapshot().Value())) + case GaugeInfo: + w.Info(fmt.Sprintf("gauge %s: value: %s", name, metric.Snapshot().Value())) case Healthcheck: metric.Check() w.Info(fmt.Sprintf("healthcheck %s: error: %v", name, metric.Error())) diff --git a/metrics/testdata/opentsb.want b/metrics/testdata/opentsb.want new file mode 100644 index 0000000000..43fe1b2ac2 --- /dev/null +++ b/metrics/testdata/opentsb.want @@ -0,0 +1,23 @@ +put pre.elite.count 978307200 1337 host=hal9000 +put pre.elite.one-minute 978307200 0.00 host=hal9000 +put pre.elite.five-minute 978307200 0.00 host=hal9000 +put pre.elite.fifteen-minute 978307200 0.00 host=hal9000 +put pre.elite.mean 978307200 0.00 host=hal9000 +put pre.foo.value 978307200 {"chain_id":"5"} host=hal9000 +put pre.months.count 978307200 12 host=hal9000 +put pre.pi.value 978307200 3.140000 host=hal9000 +put pre.second.count 978307200 1 host=hal9000 +put pre.second.min 978307200 1000 host=hal9000 +put pre.second.max 978307200 1000 host=hal9000 +put pre.second.mean 978307200 1000.00 host=hal9000 +put pre.second.std-dev 978307200 0.00 host=hal9000 +put pre.second.50-percentile 978307200 1000.00 host=hal9000 +put pre.second.75-percentile 978307200 1000.00 host=hal9000 +put pre.second.95-percentile 978307200 1000.00 host=hal9000 +put pre.second.99-percentile 978307200 1000.00 host=hal9000 +put pre.second.999-percentile 978307200 1000.00 host=hal9000 +put pre.second.one-minute 978307200 0.00 host=hal9000 +put pre.second.five-minute 978307200 0.00 host=hal9000 +put pre.second.fifteen-minute 978307200 0.00 host=hal9000 +put pre.second.mean-rate 978307200 0.00 host=hal9000 +put pre.tau.count 978307200 1.570000 host=hal9000 diff --git a/metrics/timer.go b/metrics/timer.go index 2e1a9be472..576ad8aa3e 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -5,26 +5,18 @@ import ( "time" ) +type TimerSnapshot interface { + HistogramSnapshot + MeterSnapshot +} + // Timers capture the duration and rate of events. type Timer interface { - Count() int64 - Max() int64 - Mean() float64 - Min() int64 - Percentile(float64) float64 - Percentiles([]float64) []float64 - Rate1() float64 - Rate5() float64 - Rate15() float64 - RateMean() float64 - Snapshot() Timer - StdDev() float64 + Snapshot() TimerSnapshot Stop() - Sum() int64 Time(func()) - Update(time.Duration) UpdateSince(time.Time) - Variance() float64 + Update(time.Duration) } // GetOrRegisterTimer returns an existing Timer or constructs and registers a @@ -78,61 +70,11 @@ func NewTimer() Timer { // NilTimer is a no-op Timer. type NilTimer struct{} -// Count is a no-op. -func (NilTimer) Count() int64 { return 0 } - -// Max is a no-op. -func (NilTimer) Max() int64 { return 0 } - -// Mean is a no-op. -func (NilTimer) Mean() float64 { return 0.0 } - -// Min is a no-op. -func (NilTimer) Min() int64 { return 0 } - -// Percentile is a no-op. -func (NilTimer) Percentile(p float64) float64 { return 0.0 } - -// Percentiles is a no-op. -func (NilTimer) Percentiles(ps []float64) []float64 { - return make([]float64, len(ps)) -} - -// Rate1 is a no-op. -func (NilTimer) Rate1() float64 { return 0.0 } - -// Rate5 is a no-op. -func (NilTimer) Rate5() float64 { return 0.0 } - -// Rate15 is a no-op. -func (NilTimer) Rate15() float64 { return 0.0 } - -// RateMean is a no-op. -func (NilTimer) RateMean() float64 { return 0.0 } - -// Snapshot is a no-op. -func (NilTimer) Snapshot() Timer { return NilTimer{} } - -// StdDev is a no-op. -func (NilTimer) StdDev() float64 { return 0.0 } - -// Stop is a no-op. -func (NilTimer) Stop() {} - -// Sum is a no-op. -func (NilTimer) Sum() int64 { return 0 } - -// Time is a no-op. -func (NilTimer) Time(f func()) { f() } - -// Update is a no-op. -func (NilTimer) Update(time.Duration) {} - -// UpdateSince is a no-op. -func (NilTimer) UpdateSince(time.Time) {} - -// Variance is a no-op. -func (NilTimer) Variance() float64 { return 0.0 } +func (NilTimer) Snapshot() TimerSnapshot { return (*emptySnapshot)(nil) } +func (NilTimer) Stop() {} +func (NilTimer) Time(f func()) { f() } +func (NilTimer) Update(time.Duration) {} +func (NilTimer) UpdateSince(time.Time) {} // StandardTimer is the standard implementation of a Timer and uses a Histogram // and Meter. @@ -142,82 +84,21 @@ type StandardTimer struct { mutex sync.Mutex } -// Count returns the number of events recorded. -func (t *StandardTimer) Count() int64 { - return t.histogram.Count() -} - -// Max returns the maximum value in the sample. -func (t *StandardTimer) Max() int64 { - return t.histogram.Max() -} - -// Mean returns the mean of the values in the sample. -func (t *StandardTimer) Mean() float64 { - return t.histogram.Mean() -} - -// Min returns the minimum value in the sample. -func (t *StandardTimer) Min() int64 { - return t.histogram.Min() -} - -// Percentile returns an arbitrary percentile of the values in the sample. -func (t *StandardTimer) Percentile(p float64) float64 { - return t.histogram.Percentile(p) -} - -// Percentiles returns a slice of arbitrary percentiles of the values in the -// sample. -func (t *StandardTimer) Percentiles(ps []float64) []float64 { - return t.histogram.Percentiles(ps) -} - -// Rate1 returns the one-minute moving average rate of events per second. -func (t *StandardTimer) Rate1() float64 { - return t.meter.Rate1() -} - -// Rate5 returns the five-minute moving average rate of events per second. -func (t *StandardTimer) Rate5() float64 { - return t.meter.Rate5() -} - -// Rate15 returns the fifteen-minute moving average rate of events per second. -func (t *StandardTimer) Rate15() float64 { - return t.meter.Rate15() -} - -// RateMean returns the meter's mean rate of events per second. -func (t *StandardTimer) RateMean() float64 { - return t.meter.RateMean() -} - // Snapshot returns a read-only copy of the timer. -func (t *StandardTimer) Snapshot() Timer { +func (t *StandardTimer) Snapshot() TimerSnapshot { t.mutex.Lock() defer t.mutex.Unlock() - return &TimerSnapshot{ - histogram: t.histogram.Snapshot().(*HistogramSnapshot), - meter: t.meter.Snapshot().(*MeterSnapshot), + return &timerSnapshot{ + histogram: t.histogram.Snapshot(), + meter: t.meter.Snapshot(), } } -// StdDev returns the standard deviation of the values in the sample. -func (t *StandardTimer) StdDev() float64 { - return t.histogram.StdDev() -} - // Stop stops the meter. func (t *StandardTimer) Stop() { t.meter.Stop() } -// Sum returns the sum in the sample. -func (t *StandardTimer) Sum() int64 { - return t.histogram.Sum() -} - // Record the duration of the execution of the given function. func (t *StandardTimer) Time(f func()) { ts := time.Now() @@ -241,86 +122,63 @@ func (t *StandardTimer) UpdateSince(ts time.Time) { t.meter.Mark(1) } -// Variance returns the variance of the values in the sample. -func (t *StandardTimer) Variance() float64 { - return t.histogram.Variance() -} - -// TimerSnapshot is a read-only copy of another Timer. -type TimerSnapshot struct { - histogram *HistogramSnapshot - meter *MeterSnapshot +// timerSnapshot is a read-only copy of another Timer. +type timerSnapshot struct { + histogram HistogramSnapshot + meter MeterSnapshot } // Count returns the number of events recorded at the time the snapshot was // taken. -func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() } +func (t *timerSnapshot) Count() int64 { return t.histogram.Count() } // Max returns the maximum value at the time the snapshot was taken. -func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() } +func (t *timerSnapshot) Max() int64 { return t.histogram.Max() } + +// Size returns the size of the sample at the time the snapshot was taken. +func (t *timerSnapshot) Size() int { return t.histogram.Size() } // Mean returns the mean value at the time the snapshot was taken. -func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() } +func (t *timerSnapshot) Mean() float64 { return t.histogram.Mean() } // Min returns the minimum value at the time the snapshot was taken. -func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() } +func (t *timerSnapshot) Min() int64 { return t.histogram.Min() } // Percentile returns an arbitrary percentile of sampled values at the time the // snapshot was taken. -func (t *TimerSnapshot) Percentile(p float64) float64 { +func (t *timerSnapshot) Percentile(p float64) float64 { return t.histogram.Percentile(p) } // Percentiles returns a slice of arbitrary percentiles of sampled values at // the time the snapshot was taken. -func (t *TimerSnapshot) Percentiles(ps []float64) []float64 { +func (t *timerSnapshot) Percentiles(ps []float64) []float64 { return t.histogram.Percentiles(ps) } // Rate1 returns the one-minute moving average rate of events per second at the // time the snapshot was taken. -func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() } +func (t *timerSnapshot) Rate1() float64 { return t.meter.Rate1() } // Rate5 returns the five-minute moving average rate of events per second at // the time the snapshot was taken. -func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() } +func (t *timerSnapshot) Rate5() float64 { return t.meter.Rate5() } // Rate15 returns the fifteen-minute moving average rate of events per second // at the time the snapshot was taken. -func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() } +func (t *timerSnapshot) Rate15() float64 { return t.meter.Rate15() } // RateMean returns the meter's mean rate of events per second at the time the // snapshot was taken. -func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() } - -// Snapshot returns the snapshot. -func (t *TimerSnapshot) Snapshot() Timer { return t } +func (t *timerSnapshot) RateMean() float64 { return t.meter.RateMean() } // StdDev returns the standard deviation of the values at the time the snapshot // was taken. -func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } - -// Stop is a no-op. -func (t *TimerSnapshot) Stop() {} +func (t *timerSnapshot) StdDev() float64 { return t.histogram.StdDev() } // Sum returns the sum at the time the snapshot was taken. -func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } - -// Time panics. -func (*TimerSnapshot) Time(func()) { - panic("Time called on a TimerSnapshot") -} - -// Update panics. -func (*TimerSnapshot) Update(time.Duration) { - panic("Update called on a TimerSnapshot") -} - -// UpdateSince panics. -func (*TimerSnapshot) UpdateSince(time.Time) { - panic("UpdateSince called on a TimerSnapshot") -} +func (t *timerSnapshot) Sum() int64 { return t.histogram.Sum() } // Variance returns the variance of the values at the time the snapshot was // taken. -func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() } +func (t *timerSnapshot) Variance() float64 { return t.histogram.Variance() } diff --git a/metrics/timer_test.go b/metrics/timer_test.go index 903e8e8d49..f10de16c9c 100644 --- a/metrics/timer_test.go +++ b/metrics/timer_test.go @@ -18,7 +18,7 @@ func BenchmarkTimer(b *testing.B) { func TestGetOrRegisterTimer(t *testing.T) { r := NewRegistry() NewRegisteredTimer("foo", r).Update(47) - if tm := GetOrRegisterTimer("foo", r); tm.Count() != 1 { + if tm := GetOrRegisterTimer("foo", r).Snapshot(); tm.Count() != 1 { t.Fatal(tm) } } @@ -27,7 +27,7 @@ func TestTimerExtremes(t *testing.T) { tm := NewTimer() tm.Update(math.MaxInt64) tm.Update(0) - if stdDev := tm.StdDev(); stdDev != 4.611686018427388e+18 { + if stdDev := tm.Snapshot().StdDev(); stdDev != 4.611686018427388e+18 { t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev) } } @@ -56,7 +56,7 @@ func TestTimerFunc(t *testing.T) { }) var ( drift = time.Millisecond * 2 - measured = time.Duration(tm.Max()) + measured = time.Duration(tm.Snapshot().Max()) ceil = actualTime + drift floor = actualTime - drift ) @@ -66,7 +66,7 @@ func TestTimerFunc(t *testing.T) { } func TestTimerZero(t *testing.T) { - tm := NewTimer() + tm := NewTimer().Snapshot() if count := tm.Count(); count != 0 { t.Errorf("tm.Count(): 0 != %v\n", count) } @@ -110,5 +110,5 @@ func ExampleGetOrRegisterTimer() { m := "account.create.latency" t := GetOrRegisterTimer(m, nil) t.Update(47) - fmt.Println(t.Max()) // Output: 47 + fmt.Println(t.Snapshot().Max()) // Output: 47 } diff --git a/metrics/writer.go b/metrics/writer.go index 82434e9d1d..098da45c27 100644 --- a/metrics/writer.go +++ b/metrics/writer.go @@ -29,16 +29,19 @@ func WriteOnce(r Registry, w io.Writer) { switch metric := namedMetric.m.(type) { case Counter: fmt.Fprintf(w, "counter %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %9d\n", metric.Count()) + fmt.Fprintf(w, " count: %9d\n", metric.Snapshot().Count()) case CounterFloat64: fmt.Fprintf(w, "counter %s\n", namedMetric.name) - fmt.Fprintf(w, " count: %f\n", metric.Count()) + fmt.Fprintf(w, " count: %f\n", metric.Snapshot().Count()) case Gauge: fmt.Fprintf(w, "gauge %s\n", namedMetric.name) - fmt.Fprintf(w, " value: %9d\n", metric.Value()) + fmt.Fprintf(w, " value: %9d\n", metric.Snapshot().Value()) case GaugeFloat64: fmt.Fprintf(w, "gauge %s\n", namedMetric.name) - fmt.Fprintf(w, " value: %f\n", metric.Value()) + fmt.Fprintf(w, " value: %f\n", metric.Snapshot().Value()) + case GaugeInfo: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %s\n", metric.Snapshot().Value().String()) case Healthcheck: metric.Check() fmt.Fprintf(w, "healthcheck %s\n", namedMetric.name) diff --git a/miner/ordering_test.go b/miner/ordering_test.go index ea0f62f913..a0e7b7a0b4 100644 --- a/miner/ordering_test.go +++ b/miner/ordering_test.go @@ -98,7 +98,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { } groups[addr] = append(groups[addr], &txpool.LazyTransaction{ Hash: tx.Hash(), - Tx: &txpool.Transaction{Tx: tx}, + Tx: tx, Time: tx.Time(), GasFeeCap: tx.GasFeeCap(), GasTipCap: tx.GasTipCap(), @@ -111,7 +111,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { txs := types.Transactions{} for tx := txset.Peek(); tx != nil; tx = txset.Peek() { - txs = append(txs, tx.Tx.Tx) + txs = append(txs, tx.Tx) txset.Shift() } if len(txs) != expectedCount { @@ -163,7 +163,7 @@ func TestTransactionTimeSort(t *testing.T) { groups[addr] = append(groups[addr], &txpool.LazyTransaction{ Hash: tx.Hash(), - Tx: &txpool.Transaction{Tx: tx}, + Tx: tx, Time: tx.Time(), GasFeeCap: tx.GasFeeCap(), GasTipCap: tx.GasTipCap(), @@ -174,7 +174,7 @@ func TestTransactionTimeSort(t *testing.T) { txs := types.Transactions{} for tx := txset.Peek(); tx != nil; tx = txset.Peek() { - txs = append(txs, tx.Tx.Tx) + txs = append(txs, tx.Tx) txset.Shift() } if len(txs) != len(keys) { diff --git a/miner/worker.go b/miner/worker.go index 8a6124c1a8..267f372f81 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -40,6 +40,7 @@ import ( "github.com/ava-labs/avalanchego/utils/units" "github.com/ava-labs/subnet-evm/consensus" "github.com/ava-labs/subnet-evm/consensus/dummy" + "github.com/ava-labs/subnet-evm/consensus/misc/eip4844" "github.com/ava-labs/subnet-evm/core" "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/txpool" @@ -68,6 +69,8 @@ type environment struct { header *types.Header txs []*types.Transaction receipts []*types.Receipt + sidecars []*types.BlobTxSidecar + blobs int size uint64 rules params.Rules @@ -95,10 +98,11 @@ type worker struct { pendingLogsFeed event.Feed // Subscriptions - mux *event.TypeMux // TODO replace - mu sync.RWMutex // The lock used to protect the coinbase and extra fields - coinbase common.Address - clock *mockable.Clock // Allows us mock the clock for testing + mux *event.TypeMux // TODO replace + mu sync.RWMutex // The lock used to protect the coinbase and extra fields + coinbase common.Address + clock *mockable.Clock // Allows us mock the clock for testing + beaconRoot *common.Hash // TODO: not set anywhere, retained for upstream compatibility and future use } func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, clock *mockable.Clock) *worker { @@ -168,6 +172,19 @@ func (w *worker) commitNewWork(predicateContext *precompileconfig.PredicateConte return nil, fmt.Errorf("failed to calculate new base fee: %w", err) } } + // Apply EIP-4844, EIP-4788. + if w.chainConfig.IsCancun(header.Number, header.Time) { + var excessBlobGas uint64 + if w.chainConfig.IsCancun(parent.Number, parent.Time) { + excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) + } else { + // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 + excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) + } + header.BlobGasUsed = new(uint64) + header.ExcessBlobGas = &excessBlobGas + header.ParentBeaconRoot = w.beaconRoot + } if w.coinbase == (common.Address{}) { return nil, errors.New("cannot mine without etherbase") @@ -195,6 +212,11 @@ func (w *worker) commitNewWork(predicateContext *precompileconfig.PredicateConte if err != nil { return nil, fmt.Errorf("failed to create new current environment: %w", err) } + if header.ParentBeaconRoot != nil { + context := core.NewEVMBlockContext(header, w.chain, nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, w.chainConfig, vm.Config{}) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) + } // Ensure we always stop prefetcher after block building is complete. defer func() { if env.state == nil { @@ -209,10 +231,9 @@ func (w *worker) commitNewWork(predicateContext *precompileconfig.PredicateConte return nil, err } - // Fill the block with all available pending transactions. pending := w.eth.TxPool().PendingWithBaseFee(true, header.BaseFee) - // Split the pending transactions into locals and remotes + // Split the pending transactions into locals and remotes. localTxs := make(map[common.Address][]*txpool.LazyTransaction) remoteTxs := pending for _, account := range w.eth.TxPool().Locals() { @@ -221,6 +242,8 @@ func (w *worker) commitNewWork(predicateContext *precompileconfig.PredicateConte localTxs[account] = txs } } + + // Fill the block with all available pending transactions. if len(localTxs) > 0 { txs := newTransactionsByPriceAndNonce(env.signer, localTxs, header.BaseFee) w.commitTransactions(env, txs, header.Coinbase) @@ -253,7 +276,47 @@ func (w *worker) createCurrentEnvironment(predicateContext *precompileconfig.Pre }, nil } -func (w *worker) commitTransaction(env *environment, tx *txpool.Transaction, coinbase common.Address) ([]*types.Log, error) { +func (w *worker) commitTransaction(env *environment, tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { + if tx.Type() == types.BlobTxType { + return w.commitBlobTransaction(env, tx, coinbase) + } + + receipt, err := w.applyTransaction(env, tx, coinbase) + if err != nil { + return nil, err + } + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) + return receipt.Logs, nil +} + +func (w *worker) commitBlobTransaction(env *environment, tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { + sc := tx.BlobTxSidecar() + if sc == nil { + panic("blob transaction without blobs in miner") + } + // Checking against blob gas limit: It's kind of ugly to perform this check here, but there + // isn't really a better place right now. The blob gas limit is checked at block validation time + // and not during execution. This means core.ApplyTransaction will not return an error if the + // tx has too many blobs. So we have to explicitly check it here. + if (env.blobs+len(sc.Blobs))*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { + return nil, errors.New("max data blobs reached") + } + + receipt, err := w.applyTransaction(env, tx, coinbase) + if err != nil { + return nil, err + } + env.txs = append(env.txs, tx.WithoutBlobTxSidecar()) + env.receipts = append(env.receipts, receipt) + env.sidecars = append(env.sidecars, sc) + env.blobs += len(sc.Blobs) + *env.header.BlobGasUsed += receipt.BlobGasUsed + return receipt.Logs, nil +} + +// applyTransaction runs the transaction. If execution fails, state and gas pool are reverted. +func (w *worker) applyTransaction(env *environment, tx *types.Transaction, coinbase common.Address) (*types.Receipt, error) { var ( snap = env.state.Snapshot() gp = env.gasPool.Gas() @@ -261,30 +324,25 @@ func (w *worker) commitTransaction(env *environment, tx *txpool.Transaction, coi ) if env.rules.IsDurango { - results, err := core.CheckPredicates(env.rules, env.predicateContext, tx.Tx) + results, err := core.CheckPredicates(env.rules, env.predicateContext, tx) if err != nil { - log.Debug("Transaction predicate failed verification in miner", "tx", tx.Tx.Hash(), "err", err) + log.Debug("Transaction predicate failed verification in miner", "tx", tx.Hash(), "err", err) return nil, err } - env.predicateResults.SetTxResults(tx.Tx.Hash(), results) + env.predicateResults.SetTxResults(tx.Hash(), results) blockContext = core.NewEVMBlockContextWithPredicateResults(env.header, w.chain, &coinbase, env.predicateResults) } else { blockContext = core.NewEVMBlockContext(env.header, w.chain, &coinbase) } - receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, blockContext, env.gasPool, env.state, env.header, tx.Tx, &env.header.GasUsed, *w.chain.GetVMConfig()) + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, blockContext, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) - env.predicateResults.DeleteTxResults(tx.Tx.Hash()) - return nil, err + env.predicateResults.DeleteTxResults(tx.Hash()) } - env.txs = append(env.txs, tx.Tx) - env.receipts = append(env.receipts, receipt) - env.size += tx.Tx.Size() - - return receipt.Logs, nil + return receipt, err } func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAndNonce, coinbase common.Address) { @@ -302,38 +360,37 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn tx := ltx.Resolve() if tx == nil { log.Warn("Ignoring evicted transaction") - txs.Pop() continue } // Abort transaction if it won't fit in the block and continue to search for a smaller // transction that will fit. - if totalTxsSize := env.size + tx.Tx.Size(); totalTxsSize > targetTxsSize { - log.Trace("Skipping transaction that would exceed target size", "hash", tx.Tx.Hash(), "totalTxsSize", totalTxsSize, "txSize", tx.Tx.Size()) - + if totalTxsSize := env.size + tx.Size(); totalTxsSize > targetTxsSize { + log.Trace("Skipping transaction that would exceed target size", "hash", tx.Hash(), "totalTxsSize", totalTxsSize, "txSize", tx.Size()) txs.Pop() continue } + // Error may be ignored here. The error has already been checked // during transaction acceptance is the transaction pool. - from, _ := types.Sender(env.signer, tx.Tx) + from, _ := types.Sender(env.signer, tx) // Check whether the tx is replay protected. If we're not in the EIP155 hf // phase, start ignoring the sender until we do. - if tx.Tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { - log.Trace("Ignoring reply protected transaction", "hash", tx.Tx.Hash(), "eip155", w.chainConfig.EIP155Block) - + if tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { + log.Trace("Ignoring replay protected transaction", "hash", tx.Hash(), "eip155", w.chainConfig.EIP155Block) txs.Pop() continue } + // Start executing the transaction - env.state.SetTxContext(tx.Tx.Hash(), env.tcount) + env.state.SetTxContext(tx.Hash(), env.tcount) _, err := w.commitTransaction(env, tx, coinbase) switch { case errors.Is(err, core.ErrNonceTooLow): // New head notification data race between the transaction pool and miner, shift - log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Tx.Nonce()) + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) txs.Shift() case errors.Is(err, nil): @@ -343,7 +400,7 @@ func (w *worker) commitTransactions(env *environment, txs *transactionsByPriceAn default: // Transaction is regarded as invalid, drop all consecutive transactions from // the same sender because of `nonce-too-high` clause. - log.Debug("Transaction failed, account skipped", "hash", tx.Tx.Hash(), "err", err) + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) txs.Pop() } } diff --git a/params/protocol_params.go b/params/protocol_params.go index 5668eaa8b2..fe55d86402 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -26,7 +26,11 @@ package params -import "math/big" +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) const ( GasLimitBoundDivisor uint64 = 1024 // The bound divisor of the gas limit, used in update calculations. @@ -169,12 +173,13 @@ const ( BlobTxBytesPerFieldElement = 32 // Size in bytes of a field element BlobTxFieldElementsPerBlob = 4096 // Number of field elements stored in a single data blob BlobTxHashVersion = 0x01 // Version byte of the commitment hash - BlobTxMaxBlobGasPerBlock = 1 << 19 // Maximum consumable blob gas for data blobs per block - BlobTxTargetBlobGasPerBlock = 1 << 18 // Target consumable blob gas for data blobs per block (for 1559-like pricing) BlobTxBlobGasPerBlob = 1 << 17 // Gas consumption of a single data blob (== blob byte size) BlobTxMinBlobGasprice = 1 // Minimum gas price for data blobs - BlobTxBlobGaspriceUpdateFraction = 2225652 // Controls the maximum rate of change for blob gas price + BlobTxBlobGaspriceUpdateFraction = 3338477 // Controls the maximum rate of change for blob gas price BlobTxPointEvaluationPrecompileGas = 50000 // Gas price for the point evaluation precompile. + + BlobTxTargetBlobGasPerBlock = 3 * BlobTxBlobGasPerBlob // Target consumable blob gas for data blobs per block (for 1559-like pricing) + MaxBlobGasPerBlock = 6 * BlobTxBlobGasPerBlob // Maximum consumable blob gas for data blobs per block ) // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations @@ -185,4 +190,9 @@ var ( GenesisDifficulty = big.NewInt(131072) // Difficulty of the Genesis block. MinimumDifficulty = big.NewInt(131072) // The minimum that the difficulty may ever be. DurationLimit = big.NewInt(13) // The decision boundary on the blocktime duration used to determine whether difficulty should go up or not. + + // BeaconRootsStorageAddress is the address where historical beacon roots are stored as per EIP-4788 + BeaconRootsStorageAddress = common.HexToAddress("0xbEac00dDB15f3B6d645C48263dC93862413A222D") + // SystemAddress is where the system-transaction is sent from as per EIP-4788 + SystemAddress common.Address = common.HexToAddress("0xfffffffffffffffffffffffffffffffffffffffe") ) diff --git a/params/version.go b/params/version.go index f7ce5902fe..c0af17abb8 100644 --- a/params/version.go +++ b/params/version.go @@ -32,7 +32,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 12 // Minor version component of the current release + VersionMinor = 13 // Minor version component of the current release VersionPatch = 2 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) diff --git a/plugin/evm/config.go b/plugin/evm/config.go index f0dde1c266..3f3d06e023 100644 --- a/plugin/evm/config.go +++ b/plugin/evm/config.go @@ -206,10 +206,12 @@ type Config struct { // on RPC nodes. AcceptedCacheSize int `json:"accepted-cache-size"` - // TxLookupLimit is the maximum number of blocks from head whose tx indices + // TransactionHistory is the maximum number of blocks from head whose tx indices // are reserved: // * 0: means no limit // * N: means N block limit [HEAD-N+1, HEAD] and delete extra indexes + TransactionHistory uint64 `json:"transaction-history"` + // Deprecated, use 'TransactionHistory' instead. TxLookupLimit uint64 `json:"tx-lookup-limit"` // SkipTxIndexing skips indexing transactions. @@ -325,3 +327,14 @@ func (c *Config) Validate() error { } return nil } + +func (c *Config) Deprecate() string { + msg := "" + // Deprecate the old config options and set the new ones. + if c.TxLookupLimit != 0 { + msg += "tx-lookup-limit is deprecated, use transaction-history instead. " + c.TransactionHistory = c.TxLookupLimit + } + + return msg +} diff --git a/plugin/evm/gossip.go b/plugin/evm/gossip.go index 65f68fd4d2..8e1498a4b2 100644 --- a/plugin/evm/gossip.go +++ b/plugin/evm/gossip.go @@ -133,8 +133,8 @@ func (g *GossipEthTxPool) Subscribe(ctx context.Context) { if reset { log.Debug("resetting bloom filter", "reason", "reached max filled ratio") - g.mempool.IteratePending(func(tx *txpool.Transaction) bool { - g.bloom.Add(&GossipEthTx{Tx: tx.Tx}) + g.mempool.IteratePending(func(tx *types.Transaction) bool { + g.bloom.Add(&GossipEthTx{Tx: tx}) return true }) } @@ -147,7 +147,7 @@ func (g *GossipEthTxPool) Subscribe(ctx context.Context) { // Add enqueues the transaction to the mempool. Subscribe should be called // to receive an event if tx is actually added to the mempool or not. func (g *GossipEthTxPool) Add(tx *GossipEthTx) error { - return g.mempool.Add([]*txpool.Transaction{{Tx: tx.Tx}}, false, false)[0] + return g.mempool.Add([]*types.Transaction{tx.Tx}, false, false)[0] } // Has should just return whether or not the [txID] is still in the mempool, @@ -157,8 +157,8 @@ func (g *GossipEthTxPool) Has(txID ids.ID) bool { } func (g *GossipEthTxPool) Iterate(f func(tx *GossipEthTx) bool) { - g.mempool.IteratePending(func(tx *txpool.Transaction) bool { - return f(&GossipEthTx{Tx: tx.Tx}) + g.mempool.IteratePending(func(tx *types.Transaction) bool { + return f(&GossipEthTx{Tx: tx}) }) } diff --git a/plugin/evm/handler.go b/plugin/evm/handler.go index 2c955b5e83..f01db79b04 100644 --- a/plugin/evm/handler.go +++ b/plugin/evm/handler.go @@ -55,11 +55,7 @@ func (h *GossipHandler) HandleEthTxs(nodeID ids.NodeID, msg message.EthTxsGossip return nil } h.stats.IncEthTxsGossipReceived() - wrapped := make([]*txpool.Transaction, len(txs)) - for i, tx := range txs { - wrapped[i] = &txpool.Transaction{Tx: tx} - } - errs := h.txPool.Add(wrapped, false, false) + errs := h.txPool.Add(txs, false, false) for i, err := range errs { if err != nil { log.Trace( diff --git a/plugin/evm/log.go b/plugin/evm/log.go index fcc70d6525..1c4166600b 100644 --- a/plugin/evm/log.go +++ b/plugin/evm/log.go @@ -4,18 +4,11 @@ package evm import ( - "encoding/json" - "fmt" "io" - "reflect" - "time" "github.com/ethereum/go-ethereum/log" -) -const ( - errorKey = "LOG15_ERROR" - timeFormat = "2006-01-02T15:04:05-0700" + subnetevmlog "github.com/ava-labs/subnet-evm/log" ) type SubnetEVMLogger struct { @@ -25,9 +18,9 @@ type SubnetEVMLogger struct { // InitLogger initializes logger with alias and sets the log level and format with the original [os.StdErr] interface // along with the context logger. func InitLogger(alias string, level string, jsonFormat bool, writer io.Writer) (SubnetEVMLogger, error) { - logFormat := SubnetEVMTermFormat(alias) + logFormat := subnetevmlog.SubnetEVMTermFormat(alias) if jsonFormat { - logFormat = SubnetEVMJSONFormat(alias) + logFormat = subnetevmlog.SubnetEVMJSONFormat(alias) } // Create handler @@ -37,85 +30,17 @@ func InitLogger(alias string, level string, jsonFormat bool, writer io.Writer) ( if err := c.SetLogLevel(level); err != nil { return SubnetEVMLogger{}, err } + log.PrintOrigins(true) return c, nil } // SetLogLevel sets the log level of initialized log handler. -func (c *SubnetEVMLogger) SetLogLevel(level string) error { +func (s *SubnetEVMLogger) SetLogLevel(level string) error { // Set log level logLevel, err := log.LvlFromString(level) if err != nil { return err } - log.Root().SetHandler(log.LvlFilterHandler(logLevel, c)) + log.Root().SetHandler(log.LvlFilterHandler(logLevel, s)) return nil } - -func SubnetEVMTermFormat(alias string) log.Format { - prefix := fmt.Sprintf("<%s Chain>", alias) - return log.FormatFunc(func(r *log.Record) []byte { - location := fmt.Sprintf("%+v", r.Call) - newMsg := fmt.Sprintf("%s %s: %s", prefix, location, r.Msg) - r.Msg = newMsg - return log.TerminalFormat(false).Format(r) - }) -} - -func SubnetEVMJSONFormat(alias string) log.Format { - prefix := fmt.Sprintf("%s Chain", alias) - return log.FormatFunc(func(r *log.Record) []byte { - props := make(map[string]interface{}, 5+len(r.Ctx)/2) - props["timestamp"] = r.Time - props["level"] = r.Lvl.String() - props[r.KeyNames.Msg] = r.Msg - props["logger"] = prefix - props["caller"] = fmt.Sprintf("%+v", r.Call) - for i := 0; i < len(r.Ctx); i += 2 { - k, ok := r.Ctx[i].(string) - if !ok { - props[errorKey] = fmt.Sprintf("%+v is not a string key", r.Ctx[i]) - } else { - // The number of arguments is normalized from the geth logger - // to ensure that this will not cause an index out of bounds error - props[k] = formatJSONValue(r.Ctx[i+1]) - } - } - - b, err := json.Marshal(props) - if err != nil { - b, _ = json.Marshal(map[string]string{ - errorKey: err.Error(), - }) - return b - } - - b = append(b, '\n') - return b - }) -} - -func formatJSONValue(value interface{}) (result interface{}) { - defer func() { - if err := recover(); err != nil { - if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() { - result = "nil" - } else { - panic(err) - } - } - }() - - switch v := value.(type) { - case time.Time: - return v.Format(timeFormat) - - case error: - return v.Error() - - case fmt.Stringer: - return v.String() - - default: - return v - } -} diff --git a/plugin/evm/syncervm_test.go b/plugin/evm/syncervm_test.go index 2da98bcb0d..0d5b3d860b 100644 --- a/plugin/evm/syncervm_test.go +++ b/plugin/evm/syncervm_test.go @@ -283,7 +283,7 @@ func createSyncServerAndClientVMs(t *testing.T, test syncTest) *syncVMSetup { }) // make some accounts - trieDB := trie.NewDatabase(serverVM.chaindb) + trieDB := trie.NewDatabase(serverVM.chaindb, nil) root, accounts := statesync.FillAccountsWithOverlappingStorage(t, trieDB, types.EmptyRootHash, 1000, 16) // patch serverVM's lastAcceptedBlock to have the new root diff --git a/plugin/evm/tx_gossip_test.go b/plugin/evm/tx_gossip_test.go index fb6d425321..10c14e5587 100644 --- a/plugin/evm/tx_gossip_test.go +++ b/plugin/evm/tx_gossip_test.go @@ -27,7 +27,6 @@ import ( "google.golang.org/protobuf/proto" - "github.com/ava-labs/subnet-evm/core/txpool" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/utils" ) @@ -121,7 +120,7 @@ func TestEthTxGossip(t *testing.T) { signedTx, err := types.SignTx(tx, types.NewEIP155Signer(vm.chainConfig.ChainID), key) require.NoError(err) - errs := vm.txPool.Add([]*txpool.Transaction{{Tx: signedTx}}, true, true) + errs := vm.txPool.Add([]*types.Transaction{signedTx}, true, true) require.Len(errs, 1) require.Nil(errs[0]) @@ -194,7 +193,7 @@ func TestEthTxPushGossipOutbound(t *testing.T) { require.NoError(err) // issue a tx - require.NoError(vm.txPool.Add([]*txpool.Transaction{{Tx: signedTx}}, true, true)[0]) + require.NoError(vm.txPool.Add([]*types.Transaction{signedTx}, true, true)[0]) vm.ethTxPushGossiper.Get().Add(&GossipEthTx{signedTx}) sent := <-sender.SentAppGossip diff --git a/plugin/evm/vm.go b/plugin/evm/vm.go index dd46e42251..b07adf5816 100644 --- a/plugin/evm/vm.go +++ b/plugin/evm/vm.go @@ -36,6 +36,7 @@ import ( "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/peer" "github.com/ava-labs/subnet-evm/plugin/evm/message" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" "github.com/ava-labs/subnet-evm/rpc" statesyncclient "github.com/ava-labs/subnet-evm/sync/client" @@ -270,6 +271,11 @@ func (vm *VM) Initialize( if err := vm.config.Validate(); err != nil { return err } + // We should deprecate config flags as the first thing, before we do anything else + // because this can set old flags to new flags. log the message after we have + // initialized the logger. + deprecateMsg := vm.config.Deprecate() + vm.ctx = chainCtx // Create logger @@ -287,6 +293,10 @@ func (vm *VM) Initialize( log.Info("Initializing Subnet EVM VM", "Version", Version, "Config", vm.config) + if deprecateMsg != "" { + log.Warn("Deprecation Warning", "msg", deprecateMsg) + } + if len(fxs) > 0 { return errUnsupportedFXs } @@ -778,10 +788,12 @@ func (vm *VM) setAppRequestHandlers() { // Create separate EVM TrieDB (read only) for serving leafs requests. // We create a separate TrieDB here, so that it has a separate cache from the one // used by the node when processing blocks. - evmTrieDB := trie.NewDatabaseWithConfig( + evmTrieDB := trie.NewDatabase( vm.chaindb, &trie.Config{ - Cache: vm.config.StateSyncServerTrieCache, + HashDB: &hashdb.Config{ + CleanCacheSize: vm.config.StateSyncServerTrieCache * units.MiB, + }, }, ) diff --git a/rpc/websocket.go b/rpc/websocket.go index b43b1b5a1a..652a175753 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -292,24 +292,21 @@ type websocketCodec struct { conn *websocket.Conn info PeerInfo - wg sync.WaitGroup - pingReset chan struct{} + wg sync.WaitGroup + pingReset chan struct{} + pongReceived chan struct{} } func newWebsocketCodec(conn *websocket.Conn, host string, req http.Header) ServerCodec { conn.SetReadLimit(wsMessageSizeLimit) - conn.SetPongHandler(func(appData string) error { - conn.SetReadDeadline(time.Time{}) - return nil - }) - encode := func(v interface{}, isErrorResponse bool) error { return conn.WriteJSON(v) } wc := &websocketCodec{ - jsonCodec: NewFuncCodec(conn, encode, conn.ReadJSON).(*jsonCodec), - conn: conn, - pingReset: make(chan struct{}, 1), + jsonCodec: NewFuncCodec(conn, encode, conn.ReadJSON).(*jsonCodec), + conn: conn, + pingReset: make(chan struct{}, 1), + pongReceived: make(chan struct{}), info: PeerInfo{ Transport: "ws", RemoteAddr: conn.RemoteAddr().String(), @@ -320,6 +317,13 @@ func newWebsocketCodec(conn *websocket.Conn, host string, req http.Header) Serve wc.info.HTTP.Origin = req.Get("Origin") wc.info.HTTP.UserAgent = req.Get("User-Agent") // Start pinger. + conn.SetPongHandler(func(appData string) error { + select { + case wc.pongReceived <- struct{}{}: + case <-wc.closed(): + } + return nil + }) wc.wg.Add(1) go wc.pingLoop() return wc @@ -352,26 +356,31 @@ func (wc *websocketCodec) writeJSONSkipDeadline(ctx context.Context, v interface // pingLoop sends periodic ping frames when the connection is idle. func (wc *websocketCodec) pingLoop() { - var timer = time.NewTimer(wsPingInterval) + var pingTimer = time.NewTimer(wsPingInterval) defer wc.wg.Done() - defer timer.Stop() + defer pingTimer.Stop() for { select { case <-wc.closed(): return + case <-wc.pingReset: - if !timer.Stop() { - <-timer.C + if !pingTimer.Stop() { + <-pingTimer.C } - timer.Reset(wsPingInterval) - case <-timer.C: + pingTimer.Reset(wsPingInterval) + + case <-pingTimer.C: wc.jsonCodec.encMu.Lock() wc.conn.SetWriteDeadline(time.Now().Add(wsPingWriteTimeout)) wc.conn.WriteMessage(websocket.PingMessage, nil) wc.conn.SetReadDeadline(time.Now().Add(wsPongTimeout)) wc.jsonCodec.encMu.Unlock() - timer.Reset(wsPingInterval) + pingTimer.Reset(wsPingInterval) + + case <-wc.pongReceived: + wc.conn.SetReadDeadline(time.Time{}) } } } diff --git a/sync/client/client_test.go b/sync/client/client_test.go index 0e78bf8fa4..0ec9cafb77 100644 --- a/sync/client/client_test.go +++ b/sync/client/client_test.go @@ -143,7 +143,8 @@ func TestGetBlocks(t *testing.T) { Config: params.TestChainConfig, } memdb := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(memdb) + tdb := trie.NewDatabase(memdb, nil) + genesis := gspec.MustCommit(memdb, tdb) engine := dummy.NewETHFaker() numBlocks := 110 blocks, _, err := core.GenerateChain(params.TestChainConfig, genesis, engine, memdb, numBlocks, 0, func(i int, b *core.BlockGen) {}) @@ -410,7 +411,7 @@ func TestGetLeafs(t *testing.T) { const leafsLimit = 1024 - trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase()) + trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) largeTrieRoot, largeTrieKeys, _ := syncutils.GenerateTrie(t, trieDB, 100_000, common.HashLength) smallTrieRoot, _, _ := syncutils.GenerateTrie(t, trieDB, leafsLimit, common.HashLength) @@ -781,7 +782,7 @@ func TestGetLeafs(t *testing.T) { func TestGetLeafsRetries(t *testing.T) { rand.Seed(1) - trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase()) + trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase(), nil) root, _, _ := syncutils.GenerateTrie(t, trieDB, 100_000, common.HashLength) handler := handlers.NewLeafsRequestHandler(trieDB, nil, message.Codec, handlerstats.NewNoopHandlerStats()) diff --git a/sync/handlers/block_request_test.go b/sync/handlers/block_request_test.go index 537881c85b..968f71a309 100644 --- a/sync/handlers/block_request_test.go +++ b/sync/handlers/block_request_test.go @@ -17,6 +17,7 @@ import ( "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" + "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" @@ -105,7 +106,8 @@ func TestBlockRequestHandler(t *testing.T) { Config: params.TestChainConfig, } memdb := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(memdb) + tdb := trie.NewDatabase(memdb, nil) + genesis := gspec.MustCommit(memdb, tdb) engine := dummy.NewETHFaker() blocks, _, err := core.GenerateChain(params.TestChainConfig, genesis, engine, memdb, 96, 0, func(i int, b *core.BlockGen) {}) if err != nil { @@ -162,7 +164,8 @@ func TestBlockRequestHandlerLargeBlocks(t *testing.T) { signer = types.LatestSigner(gspec.Config) ) memdb := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(memdb) + tdb := trie.NewDatabase(memdb, nil) + genesis := gspec.MustCommit(memdb, tdb) engine := dummy.NewETHFaker() blocks, _, err := core.GenerateChain(gspec.Config, genesis, engine, memdb, 96, 0, func(i int, b *core.BlockGen) { var data []byte @@ -215,7 +218,8 @@ func TestBlockRequestHandlerCtxExpires(t *testing.T) { Config: params.TestChainConfig, } memdb := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(memdb) + tdb := trie.NewDatabase(memdb, nil) + genesis := gspec.MustCommit(memdb, tdb) engine := dummy.NewETHFaker() blocks, _, err := core.GenerateChain(params.TestChainConfig, genesis, engine, memdb, 11, 0, func(i int, b *core.BlockGen) {}) if err != nil { diff --git a/sync/handlers/leafs_request_test.go b/sync/handlers/leafs_request_test.go index cd1758a561..9b0bf82570 100644 --- a/sync/handlers/leafs_request_test.go +++ b/sync/handlers/leafs_request_test.go @@ -27,7 +27,7 @@ func TestLeafsRequestHandler_OnLeafsRequest(t *testing.T) { rand.Seed(1) mockHandlerStats := &stats.MockHandlerStats{} memdb := rawdb.NewMemoryDatabase() - trieDB := trie.NewDatabase(memdb) + trieDB := trie.NewDatabase(memdb, nil) corruptedTrieRoot, _, _ := syncutils.GenerateTrie(t, trieDB, 100, common.HashLength) tr, err := trie.New(trie.TrieID(corruptedTrieRoot), trieDB) diff --git a/sync/statesync/state_syncer.go b/sync/statesync/state_syncer.go index 9c646148d3..df250b30f2 100644 --- a/sync/statesync/state_syncer.go +++ b/sync/statesync/state_syncer.go @@ -67,7 +67,7 @@ func NewStateSyncer(config *StateSyncerConfig) (*stateSync, error) { db: config.DB, client: config.Client, root: config.Root, - trieDB: trie.NewDatabase(config.DB), + trieDB: trie.NewDatabase(config.DB, nil), snapshot: snapshot.NewDiskLayer(config.DB), stats: newTrieSyncStats(), triesInProgress: make(map[common.Hash]*trieToSync), diff --git a/sync/statesync/sync_test.go b/sync/statesync/sync_test.go index 6f8e81f0be..5d098d0856 100644 --- a/sync/statesync/sync_test.go +++ b/sync/statesync/sync_test.go @@ -74,7 +74,7 @@ func testSync(t *testing.T, test syncTest) { return } - assertDBConsistency(t, root, clientDB, serverTrieDB, trie.NewDatabase(clientDB)) + assertDBConsistency(t, root, clientDB, serverTrieDB, trie.NewDatabase(clientDB, nil)) } // testSyncResumes tests a series of syncTests work as expected, invoking a callback function after each @@ -120,7 +120,7 @@ func TestSimpleSyncCases(t *testing.T) { "accounts": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, nil) return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, @@ -128,7 +128,7 @@ func TestSimpleSyncCases(t *testing.T) { "accounts with code": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount { if index%3 == 0 { codeBytes := make([]byte, 256) @@ -149,7 +149,7 @@ func TestSimpleSyncCases(t *testing.T) { "accounts with code and storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, numAccounts) return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, @@ -157,7 +157,7 @@ func TestSimpleSyncCases(t *testing.T) { "accounts with storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount { if i%5 == 0 { account.Root, _, _ = syncutils.GenerateTrie(t, serverTrieDB, 16, common.HashLength) @@ -171,7 +171,7 @@ func TestSimpleSyncCases(t *testing.T) { "accounts with overlapping storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, numAccounts, 3) return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, @@ -179,7 +179,7 @@ func TestSimpleSyncCases(t *testing.T) { "failed to fetch leafs": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccountsSmall, nil) return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, @@ -191,7 +191,7 @@ func TestSimpleSyncCases(t *testing.T) { "failed to fetch code": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, numAccountsSmall) return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, @@ -211,7 +211,7 @@ func TestSimpleSyncCases(t *testing.T) { func TestCancelSync(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) // Create trie with 2000 accounts (more than one leaf request) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, 2000) ctx, cancel := context.WithCancel(context.Background()) @@ -252,7 +252,7 @@ func (i *interruptLeafsIntercept) getLeafsIntercept(request message.LeafsRequest func TestResumeSyncAccountsTrieInterrupted(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, 2000, 3) clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ @@ -278,7 +278,7 @@ func TestResumeSyncAccountsTrieInterrupted(t *testing.T) { func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 2000, func(t *testing.T, index int, account types.StateAccount) types.StateAccount { @@ -310,7 +310,7 @@ func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) { func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) largeStorageRoot1, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) largeStorageRoot2, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) @@ -351,7 +351,7 @@ func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) { func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount { @@ -383,7 +383,7 @@ func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testi func TestResumeSyncLargeStorageTrieWithSpreadOutDuplicatesInterrupted(t *testing.T) { serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount { @@ -436,7 +436,7 @@ func TestResyncNewRootAfterDeletes(t *testing.T) { }, "delete intermediate storage nodes": { deleteBetweenSyncs: func(t *testing.T, root common.Hash, clientDB ethdb.Database) { - clientTrieDB := trie.NewDatabase(clientDB) + clientTrieDB := trie.NewDatabase(clientDB, nil) tr, err := trie.New(trie.TrieID(root), clientTrieDB) if err != nil { t.Fatal(err) @@ -482,7 +482,7 @@ func TestResyncNewRootAfterDeletes(t *testing.T) { }, "delete intermediate account trie nodes": { deleteBetweenSyncs: func(t *testing.T, root common.Hash, clientDB ethdb.Database) { - clientTrieDB := trie.NewDatabase(clientDB) + clientTrieDB := trie.NewDatabase(clientDB, nil) tr, err := trie.New(trie.TrieID(root), clientTrieDB) if err != nil { t.Fatal(err) @@ -501,7 +501,7 @@ func testSyncerSyncsToNewRoot(t *testing.T, deleteBetweenSyncs func(*testing.T, rand.Seed(1) clientDB := rawdb.NewMemoryDatabase() serverDB := rawdb.NewMemoryDatabase() - serverTrieDB := trie.NewDatabase(serverDB) + serverTrieDB := trie.NewDatabase(serverDB, nil) root1, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, 1000, 3) root2, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, root1, 1000, 3) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 848038397c..b4d78104f4 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -43,6 +43,8 @@ import ( "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" + "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -111,6 +113,7 @@ type stTransaction struct { GasLimit []uint64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey []byte `json:"secretKey"` + Sender *common.Address `json:"sender"` BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` BlobGasFeeCap *big.Int `json:"maxFeePerBlobGas,omitempty"` } @@ -200,43 +203,50 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { - snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) - if checkedErr := t.checkError(subtest, err); checkedErr != nil { - return snaps, statedb, checkedErr +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string, postCheck func(err error, snaps *snapshot.Tree, state *state.StateDB)) (result error) { + triedb, snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter, scheme) + + // Invoke the callback at the end of function for further analysis. + defer func() { + postCheck(result, snaps, statedb) + + if triedb != nil { + triedb.Close() + } + }() + checkedErr := t.checkError(subtest, err) + if checkedErr != nil { + return checkedErr } // The error has been checked; if it was unexpected, it's already returned. if err != nil { // Here, an error exists but it was expected. // We do not check the post state or logs. - return snaps, statedb, nil + return nil } post := t.json.Post[subtest.Fork][subtest.Index] // N.B: We need to do this in a two-step process, because the first Commit takes care // of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed. if root != common.Hash(post.Root) { - return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) + return fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { - return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) + return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - // Re-init the post-state instance for further operation - statedb, err = state.New(root, statedb.Database(), snaps) - if err != nil { - return nil, nil, err - } - return snaps, statedb, nil + statedb, _ = state.New(root, statedb.Database(), snaps) + return nil } // RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, common.Hash, error) { +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB, common.Hash, error) { config, eips, err := GetChainConfig(subtest.Fork) if err != nil { - return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} + return nil, nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips + block := t.genesis(config).ToBlock() - snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) + triedb, snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter, scheme) var baseFee *big.Int if config.IsSubnetEVM(0) { @@ -250,7 +260,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post, baseFee) if err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } // Try to recover tx with current signer @@ -258,11 +269,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh var ttx types.Transaction err := ttx.UnmarshalBinary(post.TxBytes) if err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } if _, err := types.Sender(types.LatestSigner(config), &ttx); err != nil { - return nil, nil, common.Hash{}, err + triedb.Close() + return nil, nil, nil, common.Hash{}, err } } @@ -275,6 +288,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.Difficulty = big.NewInt(0) } evm := vm.NewEVM(context, txContext, statedb, config, vmconfig) + // Execute the message. snapshot := statedb.Snapshot() gaspool := new(core.GasPool) @@ -291,11 +305,18 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh statedb.AddBalance(block.Coinbase(), new(big.Int)) // Commit block root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number()), false) - return snaps, statedb, root, err + return triedb, snaps, statedb, root, err } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) (*snapshot.Tree, *state.StateDB) { - sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) +func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool, scheme string) (*trie.Database, *snapshot.Tree, *state.StateDB) { + tconf := &trie.Config{Preimages: true} + if scheme == rawdb.HashScheme { + tconf.HashDB = hashdb.Defaults + } else { + tconf.PathDB = pathdb.Defaults + } + triedb := trie.NewDatabase(db, tconf) + sdb := state.NewDatabaseWithNodeDB(db, triedb) statedb, _ := state.New(types.EmptyRootHash, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) @@ -316,10 +337,10 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo AsyncBuild: false, SkipVerify: true, } - snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), common.Hash{}, root) + snaps, _ = snapshot.New(snapconfig, db, triedb, common.Hash{}, root) } statedb, _ = state.New(root, sdb, snaps) - return snaps, statedb + return triedb, snaps, statedb } func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { @@ -341,9 +362,12 @@ func (t *StateTest) genesis(config *params.ChainConfig) *core.Genesis { } func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Message, error) { - // Derive sender from private key if present. var from common.Address - if len(tx.PrivateKey) > 0 { + // If 'sender' field is present, use that + if tx.Sender != nil { + from = *tx.Sender + } else if len(tx.PrivateKey) > 0 { + // Derive sender from private key if needed. key, err := crypto.ToECDSA(tx.PrivateKey) if err != nil { return nil, fmt.Errorf("invalid private key: %v", err) diff --git a/trie/database.go b/trie/database.go index 4be40dc49f..9aa537503c 100644 --- a/trie/database.go +++ b/trie/database.go @@ -23,24 +23,23 @@ import ( "github.com/ava-labs/subnet-evm/trie/triedb/pathdb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ava-labs/subnet-evm/trie/triestate" - "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" -) - -const ( - cacheStatsUpdateFrequency = 1000 // update trie cache stats once per 1000 ops + "github.com/ethereum/go-ethereum/log" ) // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Preimages bool // Flag whether the preimage of trie key is recorded - StatsPrefix string // Prefix for cache stats (disabled if empty) - PathDB *pathdb.Config // Configs for experimental path-based scheme, not used yet. + Preimages bool // Flag whether the preimage of node key is recorded + HashDB *hashdb.Config // Configs for hash-based scheme + PathDB *pathdb.Config // Configs for experimental path-based scheme +} - // Testing hooks - OnCommit func(states *triestate.Set) // Hook invoked when commit is performed +// HashDefaults represents a config for using hash-based scheme with +// default settings. +var HashDefaults = &Config{ + Preimages: false, + HashDB: hashdb.Defaults, } // backend defines the methods needed to access/update trie nodes in different @@ -53,9 +52,12 @@ type backend interface { // according to the state scheme. Initialized(genesisRoot common.Hash) bool - // Size returns the current storage size of the memory cache in front of the - // persistent database layer. - Size() common.StorageSize + // Size returns the current storage size of the diff layers on top of the + // disk layer and the storage size of the nodes cached in the disk layer. + // + // For hash scheme, there is no differentiation between diff layer nodes + // and dirty disk layer nodes, so both are merged into the second return. + Size() (common.StorageSize, common.StorageSize) // Update performs a state transition by committing dirty nodes contained // in the given set in order to update state from the specified parent to @@ -73,13 +75,6 @@ type backend interface { Close() error } -type cache interface { - HasGet([]byte, []byte) ([]byte, bool) - Del([]byte) - Set([]byte, []byte) - SaveToFileConcurrent(dir string, threads int) error -} - // Database is the wrapper of the underlying backend which is shared by different // types of node backend as an entrypoint. It's responsible for all interactions // relevant with trie nodes and node preimages. @@ -90,36 +85,30 @@ type Database struct { backend backend // The backend for managing trie nodes } -// prepare initializes the database with provided configs, but the -// database backend is still left as nil. -func prepare(diskdb ethdb.Database, config *Config) *Database { +// NewDatabase initializes the trie database with default settings, note +// the legacy hash-based scheme is used by default. +func NewDatabase(diskdb ethdb.Database, config *Config) *Database { + // Sanitize the config and use the default one if it's not specified. + if config == nil { + config = HashDefaults + } var preimages *preimageStore - if config != nil && config.Preimages { + if config.Preimages { preimages = newPreimageStore(diskdb) } - return &Database{ + db := &Database{ config: config, diskdb: diskdb, preimages: preimages, } -} - -// NewDatabase initializes the trie database with default settings, namely -// the legacy hash-based scheme is used by default. -func NewDatabase(diskdb ethdb.Database) *Database { - return NewDatabaseWithConfig(diskdb, nil) -} - -// NewDatabaseWithConfig initializes the trie database with provided configs. -// The path-based scheme is not activated yet, always initialized with legacy -// hash-based scheme by default. -func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { - var cleans cache - if config != nil && config.Cache != 0 { - cleans = utils.NewMeteredCache(config.Cache*1024*1024, config.StatsPrefix, cacheStatsUpdateFrequency) + if config.HashDB != nil && config.PathDB != nil { + log.Crit("Both 'hash' and 'path' mode are configured") + } + if config.PathDB != nil { + db.backend = pathdb.New(diskdb, config.PathDB) + } else { + db.backend = hashdb.New(diskdb, config.HashDB, mptResolver{}) } - db := prepare(diskdb, config) - db.backend = hashdb.New(diskdb, cleans, mptResolver{}) return db } @@ -143,9 +132,6 @@ func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { // The passed in maps(nodes, states) will be retained to avoid copying everything. // Therefore, these maps must not be changed afterwards. func (db *Database) Update(root common.Hash, parent common.Hash, block uint64, nodes *trienode.MergedNodeSet, states *triestate.Set) error { - if db.config != nil && db.config.OnCommit != nil { - db.config.OnCommit(states) - } if db.preimages != nil { db.preimages.commit(false) } @@ -173,18 +159,19 @@ func (db *Database) Commit(root common.Hash, report bool) error { return db.backend.Commit(root, report) } -// Size returns the storage size of dirty trie nodes in front of the persistent -// database and the size of cached preimages. -func (db *Database) Size() (common.StorageSize, common.StorageSize) { +// Size returns the storage size of diff layer nodes above the persistent disk +// layer, the dirty nodes buffered within the disk layer, and the size of cached +// preimages. +func (db *Database) Size() (common.StorageSize, common.StorageSize, common.StorageSize) { var ( - storages common.StorageSize - preimages common.StorageSize + diffs, nodes common.StorageSize + preimages common.StorageSize ) - storages = db.backend.Size() + diffs, nodes = db.backend.Size() if db.preimages != nil { preimages = db.preimages.size() } - return storages, preimages + return diffs, nodes, preimages } // Initialized returns an indicator if the state data is already initialized @@ -213,6 +200,15 @@ func (db *Database) WritePreimages() { } } +// Preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (db *Database) Preimage(hash common.Hash) []byte { + if db.preimages == nil { + return nil + } + return db.preimages.preimage(hash) +} + // Cap iteratively flushes old but still referenced trie nodes until the total // memory usage goes below the given threshold. The held pre-images accumulated // up to this point will be flushed in case the size exceeds the threshold. @@ -264,3 +260,72 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { } return hdb.Node(hash) } + +// Recover rollbacks the database to a specified historical point. The state is +// supported as the rollback destination only if it's canonical state and the +// corresponding trie histories are existent. It's only supported by path-based +// database and will return an error for others. +func (db *Database) Recover(target common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Recover(target, &trieLoader{db: db}) +} + +// Recoverable returns the indicator if the specified state is enabled to be +// recovered. It's only supported by path-based database and will return an +// error for others. +func (db *Database) Recoverable(root common.Hash) (bool, error) { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return false, errors.New("not supported") + } + return pdb.Recoverable(root), nil +} + +// Disable deactivates the database and invalidates all available state layers +// as stale to prevent access to the persistent state, which is in the syncing +// stage. +// +// It's only supported by path-based database and will return an error for others. +func (db *Database) Disable() error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Disable() +} + +// Enable activates database and resets the state tree with the provided persistent +// state root once the state sync is finished. +func (db *Database) Enable(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Enable(root) +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the snapshot without +// flattening everything down (bad for reorgs). It's only supported by path-based +// database and will return an error for others. +func (db *Database) Journal(root common.Hash) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Journal(root) +} + +// SetBufferSize sets the node buffer size to the provided value(in bytes). +// It's only supported by path-based database and will return an error for +// others. +func (db *Database) SetBufferSize(size int) error { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.SetBufferSize(size) +} diff --git a/trie/database_test.go b/trie/database_test.go index a3621392f6..fc5598df6a 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -35,11 +35,16 @@ import ( // newTestDatabase initializes the trie database with specified scheme. func newTestDatabase(diskdb ethdb.Database, scheme string) *Database { - db := prepare(diskdb, nil) + config := &Config{Preimages: false} if scheme == rawdb.HashScheme { - db.backend = hashdb.New(diskdb, nil, mptResolver{}) + config.HashDB = &hashdb.Config{ + CleanCacheSize: 0, + } // disable clean cache } else { - db.backend = pathdb.New(diskdb, &pathdb.Config{}) // disable clean/dirty cache + config.PathDB = &pathdb.Config{ + CleanCacheSize: 0, + DirtyCacheSize: 0, + } // disable clean/dirty cache } - return db + return NewDatabase(diskdb, config) } diff --git a/trie/encoding.go b/trie/encoding.go index 5c2da51871..aaa131ef1f 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -61,9 +61,8 @@ func hexToCompact(hex []byte) []byte { return buf } -// hexToCompactInPlace places the compact key in input buffer, returning the length -// needed for the representation -func hexToCompactInPlace(hex []byte) int { +// hexToCompactInPlace places the compact key in input buffer, returning the compacted key. +func hexToCompactInPlace(hex []byte) []byte { var ( hexLen = len(hex) // length of the hex input firstByte = byte(0) @@ -87,7 +86,7 @@ func hexToCompactInPlace(hex []byte) int { hex[bi] = hex[ni]<<4 | hex[ni+1] } hex[0] = firstByte - return binLen + return hex[:binLen] } func compactToHex(compact []byte) []byte { diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 65cc5333c0..e25e4ae600 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -96,8 +96,7 @@ func TestHexToCompactInPlace(t *testing.T) { } { hexBytes, _ := hex.DecodeString(key) exp := hexToCompact(hexBytes) - sz := hexToCompactInPlace(hexBytes) - got := hexBytes[:sz] + got := hexToCompactInPlace(hexBytes) if !bytes.Equal(exp, got) { t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, key, got, exp) } @@ -112,8 +111,7 @@ func TestHexToCompactInPlaceRandom(t *testing.T) { hexBytes := keybytesToHex(key) hexOrig := []byte(string(hexBytes)) exp := hexToCompact(hexBytes) - sz := hexToCompactInPlace(hexBytes) - got := hexBytes[:sz] + got := hexToCompactInPlace(hexBytes) if !bytes.Equal(exp, got) { t.Fatalf("encoding err \ncpt %x\nhex %x\ngot %x\nexp %x\n", @@ -129,6 +127,13 @@ func BenchmarkHexToCompact(b *testing.B) { } } +func BenchmarkHexToCompactInPlace(b *testing.B) { + testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/} + for i := 0; i < b.N; i++ { + hexToCompactInPlace(testBytes) + } +} + func BenchmarkCompactToHex(b *testing.B) { testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/} for i := 0; i < b.N; i++ { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 6e14e9b29c..e34fd23206 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -28,7 +28,6 @@ package trie import ( "bytes" - "encoding/binary" "fmt" "math/rand" "testing" @@ -38,12 +37,10 @@ import ( "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func TestEmptyIterator(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) @@ -56,7 +53,7 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -99,7 +96,7 @@ func (k *kv) cmp(other *kv) int { } func TestIteratorLargeData(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -218,7 +215,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -259,7 +256,7 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) @@ -268,7 +265,7 @@ func TestDifferenceIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) @@ -301,7 +298,7 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - dba := NewDatabase(rawdb.NewMemoryDatabase()) + dba := NewDatabase(rawdb.NewMemoryDatabase(), nil) triea := NewEmpty(dba) for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) @@ -310,7 +307,7 @@ func TestUnionIterator(t *testing.T) { dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) - dbb := NewDatabase(rawdb.NewMemoryDatabase()) + dbb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) @@ -354,7 +351,7 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -547,96 +544,6 @@ func TestIteratorNodeBlob(t *testing.T) { testIteratorNodeBlob(t, rawdb.PathScheme) } -type loggingDb struct { - getCount uint64 - backend ethdb.KeyValueStore -} - -func (l *loggingDb) Has(key []byte) (bool, error) { - return l.backend.Has(key) -} - -func (l *loggingDb) Get(key []byte) ([]byte, error) { - l.getCount++ - return l.backend.Get(key) -} - -func (l *loggingDb) Put(key []byte, value []byte) error { - return l.backend.Put(key, value) -} - -func (l *loggingDb) Delete(key []byte) error { - return l.backend.Delete(key) -} - -func (l *loggingDb) NewBatch() ethdb.Batch { - return l.backend.NewBatch() -} - -func (l *loggingDb) NewBatchWithSize(size int) ethdb.Batch { - return l.backend.NewBatchWithSize(size) -} - -func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - return l.backend.NewIterator(prefix, start) -} - -func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) { - return l.backend.NewSnapshot() -} - -func (l *loggingDb) Stat(property string) (string, error) { - return l.backend.Stat(property) -} - -func (l *loggingDb) Compact(start []byte, limit []byte) error { - return l.backend.Compact(start, limit) -} - -func (l *loggingDb) Close() error { - return l.backend.Close() -} - -// makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { - // Create an empty trie - logDb := &loggingDb{0, memorydb.New()} - triedb := NewDatabase(rawdb.NewDatabase(logDb)) - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) - - // Fill it with some arbitrary data - for i := 0; i < 10000; i++ { - key := make([]byte, 32) - val := make([]byte, 32) - binary.BigEndian.PutUint64(key, uint64(i)) - binary.BigEndian.PutUint64(val, uint64(i)) - key = crypto.Keccak256(key) - val = crypto.Keccak256(val) - trie.MustUpdate(key, val) - } - root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) - triedb.Commit(root, false) - - // Return the generated trie - trie, _ = NewStateTrie(TrieID(root), triedb) - return triedb, trie, logDb -} - -// Tests that the node iterator indeed walks over the entire database contents. -func TestNodeIteratorLargeTrie(t *testing.T) { - // Create some arbitrary test trie to iterate - db, trie, logDb := makeLargeTestTrie() - db.Cap(0) // flush everything - // Do a seek operation - trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885")) - // master: 24 get operations - // this pr: 6 get operations - if have, want := logDb.getCount, uint64(6); have != want { - t.Fatalf("Too many lookups during seek, have %d want %d", have, want) - } -} - func testIteratorNodeBlob(t *testing.T, scheme string) { var ( db = rawdb.NewMemoryDatabase() @@ -710,7 +617,7 @@ func isTrieNode(scheme string, key, val []byte) (bool, []byte, common.Hash) { } hash = common.BytesToHash(key) } else { - ok, remain := rawdb.IsAccountTrieNode(key) + ok, remain := rawdb.ResolveAccountTrieNodeKey(key) if !ok { return false, nil, common.Hash{} } diff --git a/trie/proof_test.go b/trie/proof_test.go index 42d6fda662..33bbb345e0 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -104,7 +104,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -155,7 +155,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -405,7 +405,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.MustUpdate(entry.k, entry.v) @@ -477,7 +477,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -512,7 +512,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -619,7 +619,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -693,7 +693,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -1046,7 +1046,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1065,7 +1065,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -1090,7 +1090,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i, key := range keys { trie.MustUpdate(key, vals[i]) } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 99935378e0..fab78d19bd 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -41,14 +41,14 @@ import ( ) func newEmptySecure() *StateTrie { - trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase())) + trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), NewDatabase(rawdb.NewMemoryDatabase(), nil)) return trie } // makeTestStateTrie creates a large enough secure trie for testing. func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie, _ := NewStateTrie(TrieID(types.EmptyRootHash), triedb) // Fill it with some arbitrary data diff --git a/trie/stacktrie.go b/trie/stacktrie.go index b65f00a2ef..e9a89741e9 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -454,7 +454,7 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) { case extNode: st.children[0].hashRec(hasher, append(path, st.key...)) - n := shortNode{Key: hexToCompact(st.key)} + n := shortNode{Key: hexToCompactInPlace(st.key)} if len(st.children[0].val) < 32 { n.Val = rawNode(st.children[0].val) } else { @@ -470,7 +470,7 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) { case leafNode: st.key = append(st.key, byte(16)) - n := shortNode{Key: hexToCompact(st.key), Val: valueNode(st.val)} + n := shortNode{Key: hexToCompactInPlace(st.key), Val: valueNode(st.val)} n.encode(hasher.encbuf) encodedNode = hasher.encodedBytes() diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index ab4e8c3602..68396af944 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -175,6 +175,38 @@ func TestStackTrieInsertAndHash(t *testing.T) { {"123e", "x___________________________2", "0d230561e398c579e09a9f7b69ceaf7d3970f5a436fdb28b68b7a37c5bdd6b80"}, {"13aa", "x___________________________3", "ff0dc70ce2e5db90ee42a4c2ad12139596b890e90eb4e16526ab38fa465b35cf"}, }, + { // branch node with short values + {"01", "a", "b48605025f5f4b129d40a420e721aa7d504487f015fce85b96e52126365ef7dc"}, + {"80", "b", "2dc6b680daf74db067cb7aeaad73265ded93d96fce190fcbf64f498d475672ab"}, + {"ee", "c", "017dc705a54ac5328dd263fa1bae68d655310fb3e3f7b7bc57e9a43ddf99c4bf"}, + {"ff", "d", "bd5a3584d271d459bd4eb95247b2fc88656b3671b60c1125ffe7bc0b689470d0"}, + }, + { // ext node with short branch node, then becoming long + {"a0", "a", "a83e028cb1e4365935661a9fd36a5c65c30b9ab416eaa877424146ca2a69d088"}, + {"a1", "b", "f586a4639b07b01798ca65e05c253b75d51135ebfbf6f8d6e87c0435089e65f0"}, + {"a2", "c", "63e297c295c008e09a8d531e18d57f270b6bc403e23179b915429db948cd62e3"}, + {"a3", "d", "94a7b721535578e9381f1f4e4b6ec29f8bdc5f0458a30320684c562f5d47b4b5"}, + {"a4", "e", "4b7e66d1c81965cdbe8fab8295ef56bc57fefdc5733d4782d2f8baf630f083c6"}, + {"a5", "f", "2997e7b502198ce1783b5277faacf52b25844fb55a99b63e88bdbbafac573106"}, + {"a6", "g", "bee629dd27a40772b2e1a67ec6db270d26acdf8d3b674dfae27866ad6ae1f48b"}, + }, + { // branch node with short values, then long ones + {"a001", "v1", "b9cc982d995392b51e6787f1915f0b88efd4ad8b30f138da0a3e2242f2323e35"}, + {"b002", "v2", "a7b474bc77ef5097096fa0ee6298fdae8928c0bc3724e7311cd0fa9ed1942fc7"}, + {"c003", "v___________________________3", "dceb5bb7c92b0e348df988a8d9fc36b101397e38ebd405df55ba6ee5f14a264a"}, + {"d004", "v___________________________4", "36e60ecb86b9626165e1c6543c42ecbe4d83bca58e8e1124746961511fce362a"}, + }, + { // ext node to branch node with short values, then long ones + {"8002", "v1", "3258fcb3e9e7d7234ecd3b8d4743999e4ab3a21592565e0a5ca64c141e8620d9"}, + {"8004", "v2", "b6cb95b7024a83c17624a3c9bed09b4b5e8ed426f49f54b8ad13c39028b1e75a"}, + {"8008", "v___________________________3", "c769d82963abe6f0900bf69754738eeb2f84559777cfa87a44f54e1aab417871"}, + {"800d", "v___________________________4", "1cad1fdaab1a6fa95d7b780fd680030e423eb76669971368ba04797a8d9cdfc9"}, + }, + { // ext node with a child of size 31 (Y) and branch node with a child of size 31 (X) + {"000001", "ZZZZZZZZZ", "cef154b87c03c563408520ff9b26923c360cbc3ddb590c079bedeeb25a8c9c77"}, + {"000002", "Y", "2130735e600f612f6e657a32bd7be64ddcaec6512c5694844b19de713922895d"}, + {"000003", "XXXXXXXXXXXXXXXXXXXXXXXXXXXX", "962c0fffdeef7612a4f7bff1950d67e3e81c878e48b9ae45b3b374253b050bd8"}, + }, } st := NewStackTrie(nil) for i, test := range tests { @@ -198,7 +230,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -213,7 +245,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -239,7 +271,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -264,7 +296,7 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string V string @@ -292,7 +324,7 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) kvs := []struct { K string V string @@ -361,7 +393,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/tracer_test.go b/trie/tracer_test.go index 06e48578d7..f3406a0250 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -61,7 +61,7 @@ func TestTrieTracer(t *testing.T) { // Tests if the trie diffs are tracked correctly. Tracer should capture // all non-leaf dirty nodes, no matter the node is embedded or not. func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) // Determine all new nodes are tracked @@ -104,7 +104,7 @@ func TestTrieTracerNoop(t *testing.T) { } func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for _, val := range vals { trie.MustUpdate([]byte(val.k), []byte(val.v)) } @@ -128,7 +128,7 @@ func TestAccessList(t *testing.T) { func testAccessList(t *testing.T, vals []struct{ k, v string }) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) orig = trie.Copy() ) @@ -211,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { // Tests origin values won't be tracked in Iterator or Prover func TestAccessListLeak(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) // Create trie from scratch @@ -262,7 +262,7 @@ func TestAccessListLeak(t *testing.T) { // in its parent due to the smaller size of the original tree node. func TestTinyTree(t *testing.T) { var ( - db = NewDatabase(rawdb.NewMemoryDatabase()) + db = NewDatabase(rawdb.NewMemoryDatabase(), nil) trie = NewEmpty(db) ) for _, val := range tiny { diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 64ba0f14b1..b43f8cec29 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -28,6 +28,7 @@ package trie import ( "github.com/ava-labs/subnet-evm/core/types" + "github.com/ava-labs/subnet-evm/trie/triestate" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" ) @@ -93,3 +94,18 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { } return blob, nil } + +// trieLoader implements triestate.TrieLoader for constructing tries. +type trieLoader struct { + db *Database +} + +// OpenTrie opens the main account trie. +func (l *trieLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { + return New(TrieID(root), l.db) +} + +// OpenStorageTrie opens the storage trie of an account. +func (l *trieLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { + return New(StorageTrieID(stateRoot, addrHash, root), l.db) +} diff --git a/trie/trie_test.go b/trie/trie_test.go index 82db275e3d..6a46746ada 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -56,7 +56,7 @@ func init() { } func TestEmptyTrie(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) res := trie.Hash() exp := types.EmptyRootHash if res != exp { @@ -65,7 +65,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) key := make([]byte, 32) value := []byte("test") trie.MustUpdate(key, value) @@ -75,8 +75,13 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { + testMissingRoot(t, rawdb.HashScheme) + testMissingRoot(t, rawdb.PathScheme) +} + +func testMissingRoot(t *testing.T, scheme string) { root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") - trie, err := New(TrieID(root), NewDatabase(rawdb.NewMemoryDatabase())) + trie, err := New(TrieID(root), newTestDatabase(rawdb.NewMemoryDatabase(), scheme)) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -172,7 +177,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { } func TestInsert(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -184,7 +189,7 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") @@ -195,7 +200,7 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -220,7 +225,7 @@ func TestGet(t *testing.T) { } func TestDelete(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -247,7 +252,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -271,7 +276,7 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) + db := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -332,7 +337,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) trie.MustUpdate([]byte("key1"), []byte{99, 99, 99, 99}) trie.MustUpdate([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -614,7 +619,7 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B) { - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { @@ -631,7 +636,7 @@ func benchGet(b *testing.B) { } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -661,7 +666,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) i := 0 for ; i < len(addresses)/2; i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -699,7 +704,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -713,7 +718,7 @@ func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -726,7 +731,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) @@ -739,7 +744,7 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -805,11 +810,17 @@ func (s *spongeDb) Stat(property string) (string, error) { panic("implement func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Put(key []byte, value []byte) error { - valbrief := value + var ( + keybrief = key + valbrief = value + ) + if len(keybrief) > 8 { + keybrief = keybrief[:8] + } if len(valbrief) > 8 { valbrief = valbrief[:8] } - s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, key[:8], len(value), valbrief)) + s.journal = append(s.journal, fmt.Sprintf("%v: PUT([%x...], [%d bytes] %x...)\n", s.id, keybrief, len(value), valbrief)) s.sponge.Write(key) s.sponge.Write(value) return nil @@ -847,7 +858,7 @@ func TestCommitSequence(t *testing.T) { addresses, accounts := makeAccounts(tc.count) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -878,7 +889,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { prng := rand.New(rand.NewSource(int64(i))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Fill the trie with elements for i := 0; i < tc.count; i++ { @@ -910,7 +921,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { prng := rand.New(rand.NewSource(int64(count))) // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -969,7 +980,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do. func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} - db := NewDatabase(rawdb.NewDatabase(s)) + db := NewDatabase(rawdb.NewDatabase(s), nil) trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} @@ -1046,7 +1057,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1097,7 +1108,7 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1149,7 +1160,7 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - triedb := NewDatabase(rawdb.NewMemoryDatabase()) + triedb := NewDatabase(rawdb.NewMemoryDatabase(), nil) trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index e25e9c1ea4..5ace4aede4 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -38,12 +38,17 @@ import ( "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ava-labs/subnet-evm/trie/triestate" + "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) +const ( + cacheStatsUpdateFrequency = 1000 // update trie cache stats once per 1000 ops +) + var ( memcacheCleanHitMeter = metrics.NewRegisteredMeter("hashdb/memcache/clean/hit", nil) memcacheCleanMissMeter = metrics.NewRegisteredMeter("hashdb/memcache/clean/miss", nil) @@ -86,9 +91,25 @@ type cache interface { HasGet([]byte, []byte) ([]byte, bool) Del([]byte) Set([]byte, []byte) + Reset() SaveToFileConcurrent(string, int) error } +// Config contains the settings for database. +type Config struct { + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + StatsPrefix string // Prefix for cache stats (disabled if empty) +} + +// Defaults is the default setting for database if it's not specified. +// Notably, clean cache is disabled explicitly, +var Defaults = &Config{ + // Explicitly set clean cache size to 0 to avoid creating fastcache, + // otherwise database must be closed when it's no longer needed to + // prevent memory leak. + CleanCacheSize: 0, +} + // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. @@ -144,7 +165,14 @@ func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash commo } // New initializes the hash-based node database. -func New(diskdb ethdb.Database, cleans cache, resolver ChildResolver) *Database { +func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database { + if config == nil { + config = Defaults + } + var cleans cache + if config.CleanCacheSize > 0 { + cleans = utils.NewMeteredCache(config.CleanCacheSize, config.StatsPrefix, cacheStatsUpdateFrequency) + } return &Database{ diskdb: diskdb, resolver: resolver, @@ -690,7 +718,10 @@ func (db *Database) update(root common.Hash, parent common.Hash, nodes *trienode // Size returns the current storage size of the memory cache in front of the // persistent database layer. -func (db *Database) Size() common.StorageSize { +// +// The first return will always be 0, representing the memory stored in unbounded +// diff layers above the dirty cache. This is only available in pathdb. +func (db *Database) Size() (common.StorageSize, common.StorageSize) { db.lock.RLock() defer db.lock.RUnlock() @@ -698,11 +729,16 @@ func (db *Database) Size() common.StorageSize { // the total memory consumption, the maintenance metadata is also needed to be // counted. var metadataSize = common.StorageSize(len(db.dirties) * cachedNodeSize) - return db.dirtiesSize + db.childrenSize + metadataSize + return 0, db.dirtiesSize + db.childrenSize + metadataSize } // Close closes the trie database and releases all held resources. -func (db *Database) Close() error { return nil } +func (db *Database) Close() error { + if db.cleans != nil { + db.cleans.Reset() + } + return nil +} // Scheme returns the node scheme used in the database. func (db *Database) Scheme() string { diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 17f27e3a25..b8d7d46eff 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -42,8 +42,26 @@ import ( "github.com/ethereum/go-ethereum/log" ) -// maxDiffLayers is the maximum diff layers allowed in the layer tree. -const maxDiffLayers = 128 +const ( + // maxDiffLayers is the maximum diff layers allowed in the layer tree. + maxDiffLayers = 128 + + // defaultCleanSize is the default memory allowance of clean cache. + defaultCleanSize = 16 * 1024 * 1024 + + // maxBufferSize is the maximum memory allowance of node buffer. + // Too large nodebuffer will cause the system to pause for a long + // time when write happens. Also, the largest batch that pebble can + // support is 4GB, node will panic if batch size exceeds this limit. + maxBufferSize = 256 * 1024 * 1024 + + // DefaultBufferSize is the default memory allowance of node buffer + // that aggregates the writes from above until it's flushed into the + // disk. It's meant to be used once the initial sync is finished. + // Do not increase the buffer size arbitrarily, otherwise the system + // pause time will increase when the database writes happen. + DefaultBufferSize = 64 * 1024 * 1024 +) // layer is the interface implemented by all state layers which includes some // public methods and some additional methods for internal usage. @@ -77,30 +95,33 @@ type layer interface { // Config contains the settings for database. type Config struct { - StateLimit uint64 // Number of recent blocks to maintain state history for - CleanSize int // Maximum memory allowance (in bytes) for caching clean nodes - DirtySize int // Maximum memory allowance (in bytes) for caching dirty nodes - ReadOnly bool // Flag whether the database is opened in read only mode. + StateHistory uint64 // Number of recent blocks to maintain state history for + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes + ReadOnly bool // Flag whether the database is opened in read only mode. } -var ( - // defaultCleanSize is the default memory allowance of clean cache. - defaultCleanSize = 16 * 1024 * 1024 - - // defaultBufferSize is the default memory allowance of node buffer - // that aggregates the writes from above until it's flushed into the - // disk. Do not increase the buffer size arbitrarily, otherwise the - // system pause time will increase when the database writes happen. - defaultBufferSize = 128 * 1024 * 1024 -) +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (c *Config) sanitize() *Config { + conf := *c + if conf.DirtyCacheSize > maxBufferSize { + log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize)) + conf.DirtyCacheSize = maxBufferSize + } + return &conf +} // Defaults contains default settings for Ethereum mainnet. var Defaults = &Config{ - StateLimit: params.FullImmutabilityThreshold, - CleanSize: defaultCleanSize, - DirtySize: defaultBufferSize, + StateHistory: params.FullImmutabilityThreshold, + CleanCacheSize: defaultCleanSize, + DirtyCacheSize: DefaultBufferSize, } +// ReadOnly is the config in order to open database in read only mode. +var ReadOnly = &Config{ReadOnly: true} + // Database is a multiple-layered structure for maintaining in-memory trie nodes. // It consists of one persistent base layer backed by a key-value store, on top // of which arbitrarily many in-memory diff layers are stacked. The memory diffs @@ -116,7 +137,8 @@ type Database struct { // readOnly is the flag whether the mutation is allowed to be applied. // It will be set automatically when the database is journaled during // the shutdown to reject all following unexpected mutations. - readOnly bool // Indicator if database is opened in read only mode + readOnly bool // Flag if database is opened in read only mode + waitSync bool // Flag if database is deactivated due to initial state sync bufferSize int // Memory allowance (in bytes) for caching dirty nodes config *Config // Configuration for database diskdb ethdb.Database // Persistent storage for matured trie nodes @@ -134,9 +156,11 @@ func New(diskdb ethdb.Database, config *Config) *Database { if config == nil { config = Defaults } + config = config.sanitize() + db := &Database{ readOnly: config.ReadOnly, - bufferSize: config.DirtySize, + bufferSize: config.DirtyCacheSize, config: config, diskdb: diskdb, } @@ -151,23 +175,29 @@ func New(diskdb ethdb.Database, config *Config) *Database { // Because the freezer can only be opened once at the same time, this // mechanism also ensures that at most one **non-readOnly** database // is opened at the same time to prevent accidental mutation. - //if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly { - // freezer, err := rawdb.NewStateHistoryFreezer(ancient, false) - // if err != nil { - // log.Crit("Failed to open state history freezer", "err", err) - // } - // db.freezer = freezer - - // // Truncate the extra state histories above in freezer in case - // // it's not aligned with the disk layer. - // pruned, err := truncateFromHead(db.diskdb, freezer, db.tree.bottom().stateID()) - // if err != nil { - // log.Crit("Failed to truncate extra state histories", "err", err) - // } - // if pruned != 0 { - // log.Warn("Truncated extra state histories", "number", pruned) - // } - //} + // if ancient, err := diskdb.AncientDatadir(); err == nil && ancient != "" && !db.readOnly { + // freezer, err := rawdb.NewStateFreezer(ancient, false) + // if err != nil { + // log.Crit("Failed to open state history freezer", "err", err) + // } + // db.freezer = freezer + + // // Truncate the extra state histories above in freezer in case + // // it's not aligned with the disk layer. + // pruned, err := truncateFromHead(db.diskdb, freezer, db.tree.bottom().stateID()) + // if err != nil { + // log.Crit("Failed to truncate extra state histories", "err", err) + // } + // if pruned != 0 { + // log.Warn("Truncated extra state histories", "number", pruned) + // } + // } + // // Disable database in case node is still in the initial state sync stage. + // if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly { + // if err := db.Disable(); err != nil { + // log.Crit("Failed to disable database", "err", err) // impossible to happen + // } + // } log.Warn("Path-based state scheme is an experimental feature") return db } @@ -193,9 +223,9 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6 db.lock.Lock() defer db.lock.Unlock() - // Short circuit if the database is in read only mode. - if db.readOnly { - return errSnapshotReadOnly + // Short circuit if the mutation is not allowed. + if err := db.modifyAllowed(); err != nil { + return err } if err := db.tree.add(root, parentRoot, block, nodes, states); err != nil { return err @@ -216,45 +246,59 @@ func (db *Database) Commit(root common.Hash, report bool) error { db.lock.Lock() defer db.lock.Unlock() - // Short circuit if the database is in read only mode. - if db.readOnly { - return errSnapshotReadOnly + // Short circuit if the mutation is not allowed. + if err := db.modifyAllowed(); err != nil { + return err } return db.tree.cap(root, 0) } -// Reset rebuilds the database with the specified state as the base. -// -// - if target state is empty, clear the stored state and all layers on top -// - if target state is non-empty, ensure the stored state matches with it -// and clear all other layers on top. -func (db *Database) Reset(root common.Hash) error { +// Disable deactivates the database and invalidates all available state layers +// as stale to prevent access to the persistent state, which is in the syncing +// stage. +func (db *Database) Disable() error { db.lock.Lock() defer db.lock.Unlock() // Short circuit if the database is in read only mode. if db.readOnly { - return errSnapshotReadOnly + return errDatabaseReadOnly } - batch := db.diskdb.NewBatch() - root = types.TrieRootHash(root) - if root == types.EmptyRootHash { - // Empty state is requested as the target, nuke out - // the root node and leave all others as dangling. - rawdb.DeleteAccountTrieNode(batch, nil) - } else { - // Ensure the requested state is existent before any - // action is applied. - _, hash := rawdb.ReadAccountTrieNode(db.diskdb, nil) - if hash != root { - return fmt.Errorf("state is mismatched, local: %x, target: %x", hash, root) - } + // Prevent duplicated disable operation. + if db.waitSync { + log.Error("Reject duplicated disable operation") + return nil } - // Mark the disk layer as stale before applying any mutation. + db.waitSync = true + + // Mark the disk layer as stale to prevent access to persistent state. db.tree.bottom().markStale() + // Write the initial sync flag to persist it across restarts. + // rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncRunning) + log.Info("Disabled trie database due to state sync") + return nil +} + +// Enable activates database and resets the state tree with the provided persistent +// state root once the state sync is finished. +func (db *Database) Enable(root common.Hash) error { + db.lock.Lock() + defer db.lock.Unlock() + + // Short circuit if the database is in read only mode. + if db.readOnly { + return errDatabaseReadOnly + } + // Ensure the provided state root matches the stored one. + root = types.TrieRootHash(root) + _, stored := rawdb.ReadAccountTrieNode(db.diskdb, nil) + if stored != root { + return fmt.Errorf("state root mismatch: stored %x, synced %x", stored, root) + } // Drop the stale state journal in persistent database and // reset the persistent state id back to zero. + batch := db.diskdb.NewBatch() rawdb.DeleteTrieJournal(batch) rawdb.WritePersistentStateID(batch, 0) if err := batch.Write(); err != nil { @@ -272,8 +316,11 @@ func (db *Database) Reset(root common.Hash) error { // } // Re-construct a new disk layer backed by persistent state // with **empty clean cache and node buffer**. - dl := newDiskLayer(root, 0, db, nil, newNodeBuffer(db.bufferSize, nil, 0)) - db.tree.reset(dl) + db.tree.reset(newDiskLayer(root, 0, db, nil, newNodeBuffer(db.bufferSize, nil, 0))) + + // Re-enable the database as the final step. + db.waitSync = false + // rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished) log.Info("Rebuilt trie database", "root", root) return nil } @@ -323,27 +370,34 @@ func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() + // Set the database to read-only mode to prevent all + // following mutations. db.readOnly = true - return nil + + // Release the memory held by clean cache. + db.tree.bottom().resetCache() + // NOTE(freezer): This is disabled since we do not have a freezer. - // if db.freezer == nil { - // return nil - // } - // return db.freezer.Close() + // Close the attached state history freezer. + //if db.freezer == nil { + // return nil + //} + //return db.freezer.Close() + return nil } // Size returns the current storage size of the memory cache in front of the // persistent database layer. -func (db *Database) Size() (size common.StorageSize) { +func (db *Database) Size() (diffs common.StorageSize, nodes common.StorageSize) { db.tree.forEach(func(layer layer) { if diff, ok := layer.(*diffLayer); ok { - size += common.StorageSize(diff.memory) + diffs += common.StorageSize(diff.memory) } if disk, ok := layer.(*diskLayer); ok { - size += disk.size() + nodes += disk.size() } }) - return size + return diffs, nodes } // Initialized returns an indicator if the state data is already @@ -363,6 +417,10 @@ func (db *Database) SetBufferSize(size int) error { db.lock.Lock() defer db.lock.Unlock() + if size > maxBufferSize { + log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize)) + size = maxBufferSize + } db.bufferSize = size return db.tree.bottom().setBufferSize(db.bufferSize) } @@ -371,3 +429,15 @@ func (db *Database) SetBufferSize(size int) error { func (db *Database) Scheme() string { return rawdb.PathScheme } + +// modifyAllowed returns the indicator if mutation is allowed. This function +// assumes the db.lock is already held. +func (db *Database) modifyAllowed() error { + if db.readOnly { + return errDatabaseReadOnly + } + if db.waitSync { + return errDatabaseWaitSync + } + return nil +} diff --git a/trie/triedb/pathdb/database_test.go b/trie/triedb/pathdb/database_test.go index 2948e22140..d1b7a0b26d 100644 --- a/trie/triedb/pathdb/database_test.go +++ b/trie/triedb/pathdb/database_test.go @@ -57,7 +57,8 @@ func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[comm h.Update(key.Bytes(), val) } } - return h.Commit(false) + root, nodes, _ := h.Commit(false) + return root, nodes } func generateAccount(storageRoot common.Hash) types.StateAccount { @@ -111,7 +112,7 @@ func newTester(t *testing.T) *tester { // NOTE(freezer): This is disabled since we do not have a freezer. // disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) disk = rawdb.NewMemoryDatabase() - db = New(disk, &Config{CleanSize: 256 * 1024, DirtySize: 256 * 1024}) + db = New(disk, &Config{CleanCacheSize: 256 * 1024, DirtyCacheSize: 256 * 1024}) obj = &tester{ db: db, preimages: make(map[common.Hash]common.Address), @@ -426,39 +427,40 @@ func TestDatabaseRecoverable(t *testing.T) { } } -func TestReset(t *testing.T) { - var ( - tester = newTester(t) - // index = tester.bottomIndex() - ) +func TestDisable(t *testing.T) { + tester := newTester(t) defer tester.release() - // Reset database to unknown target, should reject it - if err := tester.db.Reset(testutil.RandomHash()); err == nil { - t.Fatal("Failed to reject invalid reset") + _, stored := rawdb.ReadAccountTrieNode(tester.db.diskdb, nil) + if err := tester.db.Disable(); err != nil { + t.Fatal("Failed to deactivate database") } - // Reset database to state persisted in the disk - if err := tester.db.Reset(types.EmptyRootHash); err != nil { - t.Fatalf("Failed to reset database %v", err) + if err := tester.db.Enable(types.EmptyRootHash); err == nil { + t.Fatalf("Invalid activation should be rejected") } + if err := tester.db.Enable(stored); err != nil { + t.Fatal("Failed to activate database") + } + // Ensure journal is deleted from disk if blob := rawdb.ReadTrieJournal(tester.db.diskdb); len(blob) != 0 { t.Fatal("Failed to clean journal") } // NOTE(freezer): This is disabled since we do not have a freezer. // Ensure all trie histories are removed - // for i := 0; i <= index; i++ { - // _, err := readHistory(tester.db.freezer, uint64(i+1)) - // if err == nil { - // t.Fatalf("Failed to clean state history, index %d", i+1) - // } + // n, err := tester.db.freezer.Ancients() + // if err != nil { + // t.Fatal("Failed to clean state history") + // } + // if n != 0 { + // t.Fatal("Failed to clean state history") // } // Verify layer tree structure, single disk layer is expected if tester.db.tree.len() != 1 { t.Fatalf("Extra layer kept %d", tester.db.tree.len()) } - if tester.db.tree.bottom().rootHash() != types.EmptyRootHash { - t.Fatalf("Root hash is not matched exp %x got %x", types.EmptyRootHash, tester.db.tree.bottom().rootHash()) + if tester.db.tree.bottom().rootHash() != stored { + t.Fatalf("Root hash is not matched exp %x got %x", stored, tester.db.tree.bottom().rootHash()) } } diff --git a/trie/triedb/pathdb/difflayer.go b/trie/triedb/pathdb/difflayer.go index 52f08bf46a..997b3b636b 100644 --- a/trie/triedb/pathdb/difflayer.go +++ b/trie/triedb/pathdb/difflayer.go @@ -124,7 +124,7 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept if n.Hash != hash { dirtyFalseMeter.Mark(1) log.Error("Unexpected trie node in diff layer", "owner", owner, "path", path, "expect", hash, "got", n.Hash) - return nil, newUnexpectedNodeError("diff", hash, n.Hash, owner, path) + return nil, newUnexpectedNodeError("diff", hash, n.Hash, owner, path, n.Blob) } dirtyHitMeter.Mark(1) dirtyNodeHitDepthHist.Update(int64(depth)) diff --git a/trie/triedb/pathdb/difflayer_test.go b/trie/triedb/pathdb/difflayer_test.go index 5520a5779e..7b7570f6b1 100644 --- a/trie/triedb/pathdb/difflayer_test.go +++ b/trie/triedb/pathdb/difflayer_test.go @@ -39,7 +39,7 @@ import ( func emptyLayer() *diskLayer { return &diskLayer{ db: New(rawdb.NewMemoryDatabase(), nil), - buffer: newNodeBuffer(defaultBufferSize, nil, 0), + buffer: newNodeBuffer(DefaultBufferSize, nil, 0), } } diff --git a/trie/triedb/pathdb/disklayer.go b/trie/triedb/pathdb/disklayer.go index aa3287ccfd..937f4e6474 100644 --- a/trie/triedb/pathdb/disklayer.go +++ b/trie/triedb/pathdb/disklayer.go @@ -57,8 +57,8 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C // Initialize a clean cache if the memory allowance is not zero // or reuse the provided cache if it is not nil (inherited from // the original disk layer). - if cleans == nil && db.config.CleanSize != 0 { - cleans = fastcache.New(db.config.CleanSize) + if cleans == nil && db.config.CleanCacheSize != 0 { + cleans = fastcache.New(db.config.CleanCacheSize) } return &diskLayer{ root: root, @@ -160,7 +160,7 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b if nHash != hash { diskFalseMeter.Mark(1) log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash) - return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path) + return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob) } if dl.cleans != nil && len(nBlob) > 0 { dl.cleans.Set(key, nBlob) @@ -188,7 +188,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // corresponding states(journal), the stored state history will // be truncated in the next restart. // if dl.db.freezer != nil { - // err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateLimit) + // err := writeHistory(dl.db.diskdb, dl.db.freezer, bottom, dl.db.config.StateHistory) // if err != nil { // return nil, err // } @@ -288,6 +288,20 @@ func (dl *diskLayer) size() common.StorageSize { return common.StorageSize(dl.buffer.size) } +// resetCache releases the memory held by clean cache to prevent memory leak. +func (dl *diskLayer) resetCache() { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // Stale disk layer loses the ownership of clean cache. + if dl.stale { + return + } + if dl.cleans != nil { + dl.cleans.Reset() + } +} + // hasher is used to compute the sha256 hash of the provided data. type hasher struct{ sha crypto.KeccakState } diff --git a/trie/triedb/pathdb/errors.go b/trie/triedb/pathdb/errors.go index be6bf6c36e..af6e3464d2 100644 --- a/trie/triedb/pathdb/errors.go +++ b/trie/triedb/pathdb/errors.go @@ -31,12 +31,17 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" ) var ( - // errSnapshotReadOnly is returned if the database is opened in read only mode - // and mutation is requested. - errSnapshotReadOnly = errors.New("read only") + // errDatabaseReadOnly is returned if the database is opened in read only mode + // to prevent any mutation. + errDatabaseReadOnly = errors.New("read only") + + // errDatabaseWaitSync is returned if the initial state sync is not completed + // yet and database is disabled to prevent accessing state. + errDatabaseWaitSync = errors.New("waiting for sync") // errSnapshotStale is returned from data accessors if the underlying layer // layer had been invalidated due to the chain progressing forward far enough @@ -58,6 +63,10 @@ var ( errUnexpectedNode = errors.New("unexpected node") ) -func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte) error { - return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x", errUnexpectedNode, loc, owner, path, expHash, gotHash) +func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte, blob []byte) error { + blobHex := "nil" + if len(blob) > 0 { + blobHex = hexutil.Encode(blob) + } + return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x, blob: %s", errUnexpectedNode, loc, owner, path, expHash, gotHash, blobHex) } diff --git a/trie/triedb/pathdb/journal.go b/trie/triedb/pathdb/journal.go index d35f00bab6..6a9edccaee 100644 --- a/trie/triedb/pathdb/journal.go +++ b/trie/triedb/pathdb/journal.go @@ -31,6 +31,7 @@ import ( "errors" "fmt" "io" + "time" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" @@ -351,13 +352,21 @@ func (db *Database) Journal(root common.Hash) error { if l == nil { return fmt.Errorf("triedb layer [%#x] missing", root) } + disk := db.tree.bottom() + if l, ok := l.(*diffLayer); ok { + log.Info("Persisting dirty state to disk", "head", l.block, "root", root, "layers", l.id-disk.id+disk.buffer.layers) + } else { // disk layer only on noop runs (likely) or deep reorgs (unlikely) + log.Info("Persisting dirty state to disk", "root", root, "layers", disk.buffer.layers) + } + start := time.Now() + // Run the journaling db.lock.Lock() defer db.lock.Unlock() // Short circuit if the database is in read only mode. if db.readOnly { - return errSnapshotReadOnly + return errDatabaseReadOnly } // Firstly write out the metadata of journal journal := new(bytes.Buffer) @@ -383,6 +392,6 @@ func (db *Database) Journal(root common.Hash) error { // Set the db in read only mode to reject all following mutations db.readOnly = true - log.Info("Stored journal in triedb", "disk", diskroot, "size", common.StorageSize(journal.Len())) + log.Info("Persisted dirty state to disk", "size", common.StorageSize(journal.Len()), "elapsed", common.PrettyDuration(time.Since(start))) return nil } diff --git a/trie/triedb/pathdb/layertree.go b/trie/triedb/pathdb/layertree.go index 58b112c6bb..bc052da711 100644 --- a/trie/triedb/pathdb/layertree.go +++ b/trie/triedb/pathdb/layertree.go @@ -142,7 +142,35 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { return err } // Replace the entire layer tree with the flat base - tree.layers = map[common.Hash]layer{base.rootHash(): base} + // tree.layers = map[common.Hash]layer{base.rootHash(): base} + // + // Note: The original code above is replaced with the code below + // since we need to keep the children of the base layer, as these + // layers may be accessed by blocks in processing. + children := make(map[common.Hash][]common.Hash) + for root, layer := range tree.layers { + if dl, ok := layer.(*diffLayer); ok { + parent := dl.parentLayer().rootHash() + children[parent] = append(children[parent], root) + if parent == base.rootHash() { + dl.lock.Lock() + dl.parent = base + dl.lock.Unlock() + } + } + } + + newLayers := map[common.Hash]layer{base.rootHash(): base} + var keepChildren func(root common.Hash) + keepChildren = func(root common.Hash) { + for _, child := range children[root] { + childLayer := tree.layers[child] + newLayers[child] = childLayer + keepChildren(child) + } + } + keepChildren(base.rootHash()) + tree.layers = newLayers return nil } // Dive until we run out of layers or reach the persistent database diff --git a/trie/triedb/pathdb/nodebuffer.go b/trie/triedb/pathdb/nodebuffer.go index 9a0ce7fb02..ad0c45a4ef 100644 --- a/trie/triedb/pathdb/nodebuffer.go +++ b/trie/triedb/pathdb/nodebuffer.go @@ -81,7 +81,7 @@ func (b *nodebuffer) node(owner common.Hash, path []byte, hash common.Hash) (*tr if n.Hash != hash { dirtyFalseMeter.Mark(1) log.Error("Unexpected trie node in node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash) - return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path) + return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob) } return n, nil } diff --git a/trie/triedb/pathdb/testutils.go b/trie/triedb/pathdb/testutils.go index 71b845d2ad..cf09e18609 100644 --- a/trie/triedb/pathdb/testutils.go +++ b/trie/triedb/pathdb/testutils.go @@ -90,7 +90,7 @@ func (h *testHasher) Delete(key []byte) error { // Commit computes the new hash of the states and returns the set with all // state changes. -func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { var ( nodes = make(map[common.Hash][]byte) set = trienode.NewNodeSet(h.owner) @@ -118,7 +118,7 @@ func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { if root == types.EmptyRootHash && h.root != types.EmptyRootHash { set.AddNode(nil, trienode.NewDeleted()) } - return root, set + return root, set, nil } // hash performs the hash computation upon the provided states. diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 6504ac6518..aa282a8e30 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -53,7 +53,7 @@ type Trie interface { // Commit the trie and returns a set of dirty nodes generated along with // the new root hash. - Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) + Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) } // TrieLoader wraps functions to load tries. @@ -139,7 +139,10 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre return nil, fmt.Errorf("failed to revert state, err: %w", err) } } - root, result := tr.Commit(false) + root, result, err := tr.Commit(false) + if err != nil { + return nil, err + } if root != prevRoot { return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) } @@ -191,7 +194,10 @@ func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != prev.Root { return errors.New("failed to reset storage trie") } @@ -242,7 +248,10 @@ func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { return err } } - root, result := st.Commit(false) + root, result, err := st.Commit(false) + if err != nil { + return err + } if root != types.EmptyRootHash { return errors.New("failed to clear storage trie") } diff --git a/warp/handlers/signature_request_test.go b/warp/handlers/signature_request_test.go index cffd2dbd81..f10c144bb4 100644 --- a/warp/handlers/signature_request_test.go +++ b/warp/handlers/signature_request_test.go @@ -62,12 +62,12 @@ func TestMessageSignatureHandler(t *testing.T) { }, signature[:] }, verifyStats: func(t *testing.T, stats *handlerStats) { - require.EqualValues(t, 1, stats.messageSignatureRequest.Count()) - require.EqualValues(t, 1, stats.messageSignatureHit.Count()) - require.EqualValues(t, 0, stats.messageSignatureMiss.Count()) - require.EqualValues(t, 0, stats.blockSignatureRequest.Count()) - require.EqualValues(t, 0, stats.blockSignatureHit.Count()) - require.EqualValues(t, 0, stats.blockSignatureMiss.Count()) + require.EqualValues(t, 1, stats.messageSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 1, stats.messageSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureMiss.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureMiss.Snapshot().Count()) }, }, "offchain message": { @@ -77,12 +77,12 @@ func TestMessageSignatureHandler(t *testing.T) { }, offchainSignature[:] }, verifyStats: func(t *testing.T, stats *handlerStats) { - require.EqualValues(t, 1, stats.messageSignatureRequest.Count()) - require.EqualValues(t, 1, stats.messageSignatureHit.Count()) - require.EqualValues(t, 0, stats.messageSignatureMiss.Count()) - require.EqualValues(t, 0, stats.blockSignatureRequest.Count()) - require.EqualValues(t, 0, stats.blockSignatureHit.Count()) - require.EqualValues(t, 0, stats.blockSignatureMiss.Count()) + require.EqualValues(t, 1, stats.messageSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 1, stats.messageSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureMiss.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureMiss.Snapshot().Count()) }, }, "unknown message": { @@ -92,12 +92,12 @@ func TestMessageSignatureHandler(t *testing.T) { }, emptySignature[:] }, verifyStats: func(t *testing.T, stats *handlerStats) { - require.EqualValues(t, 1, stats.messageSignatureRequest.Count()) - require.EqualValues(t, 0, stats.messageSignatureHit.Count()) - require.EqualValues(t, 1, stats.messageSignatureMiss.Count()) - require.EqualValues(t, 0, stats.blockSignatureRequest.Count()) - require.EqualValues(t, 0, stats.blockSignatureHit.Count()) - require.EqualValues(t, 0, stats.blockSignatureMiss.Count()) + require.EqualValues(t, 1, stats.messageSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureHit.Snapshot().Count()) + require.EqualValues(t, 1, stats.messageSignatureMiss.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureMiss.Snapshot().Count()) }, }, } @@ -177,12 +177,12 @@ func TestBlockSignatureHandler(t *testing.T) { }, signature[:] }, verifyStats: func(t *testing.T, stats *handlerStats) { - require.EqualValues(t, 0, stats.messageSignatureRequest.Count()) - require.EqualValues(t, 0, stats.messageSignatureHit.Count()) - require.EqualValues(t, 0, stats.messageSignatureMiss.Count()) - require.EqualValues(t, 1, stats.blockSignatureRequest.Count()) - require.EqualValues(t, 1, stats.blockSignatureHit.Count()) - require.EqualValues(t, 0, stats.blockSignatureMiss.Count()) + require.EqualValues(t, 0, stats.messageSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureMiss.Snapshot().Count()) + require.EqualValues(t, 1, stats.blockSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 1, stats.blockSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureMiss.Snapshot().Count()) }, }, "unknown block": { @@ -192,12 +192,12 @@ func TestBlockSignatureHandler(t *testing.T) { }, emptySignature[:] }, verifyStats: func(t *testing.T, stats *handlerStats) { - require.EqualValues(t, 0, stats.messageSignatureRequest.Count()) - require.EqualValues(t, 0, stats.messageSignatureHit.Count()) - require.EqualValues(t, 0, stats.messageSignatureMiss.Count()) - require.EqualValues(t, 1, stats.blockSignatureRequest.Count()) - require.EqualValues(t, 0, stats.blockSignatureHit.Count()) - require.EqualValues(t, 1, stats.blockSignatureMiss.Count()) + require.EqualValues(t, 0, stats.messageSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureHit.Snapshot().Count()) + require.EqualValues(t, 0, stats.messageSignatureMiss.Snapshot().Count()) + require.EqualValues(t, 1, stats.blockSignatureRequest.Snapshot().Count()) + require.EqualValues(t, 0, stats.blockSignatureHit.Snapshot().Count()) + require.EqualValues(t, 1, stats.blockSignatureMiss.Snapshot().Count()) }, }, }