diff --git a/commitment/bin_patricia_hashed.go b/commitment/bin_patricia_hashed.go index deb3fd7a3..877d3218f 100644 --- a/commitment/bin_patricia_hashed.go +++ b/commitment/bin_patricia_hashed.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Erigon contributors + Copyright 2022 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import ( "github.com/ledgerwatch/log/v3" "golang.org/x/crypto/sha3" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/rlp" ) @@ -53,7 +54,7 @@ func hexToBin(hex []byte) bitstring { // encodes bitstring to its compact representation func binToCompact(bin []byte) []byte { - compact := make([]byte, 2+(len(bin)+7)/8) + compact := make([]byte, 2+common.BitLenToByteLen(len(bin))) binary.BigEndian.PutUint16(compact, uint16(len(bin))) for i := 0; i < len(bin); i++ { if bin[i] != 0 { @@ -432,7 +433,7 @@ func (cell *BinaryCell) accountForHashing(buffer []byte, storageRootHash [length if cell.Nonce < 128 && cell.Nonce != 0 { nonceBytes = 0 } else { - nonceBytes = (bits.Len64(cell.Nonce) + 7) / 8 + nonceBytes = common.BitLenToByteLen(bits.Len64(cell.Nonce)) } var structLength = uint(balanceBytes + nonceBytes + 2) @@ -443,7 +444,7 @@ func (cell *BinaryCell) accountForHashing(buffer []byte, storageRootHash [length buffer[0] = byte(192 + structLength) pos = 1 } else { - lengthBytes := (bits.Len(structLength) + 7) / 8 + lengthBytes := common.BitLenToByteLen(bits.Len(structLength)) buffer[0] = byte(247 + lengthBytes) for i := lengthBytes; i > 0; i-- { diff --git a/commitment/hex_patricia_hashed.go b/commitment/hex_patricia_hashed.go index 0fc7d63f8..8732d2ae7 100644 --- a/commitment/hex_patricia_hashed.go +++ b/commitment/hex_patricia_hashed.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Erigon contributors + Copyright 2022 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -27,10 +27,10 @@ import ( "strings" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" - "github.com/ledgerwatch/log/v3" + "golang.org/x/crypto/sha3" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/rlp" ) @@ -409,7 +409,7 @@ func (cell *Cell) accountForHashing(buffer []byte, storageRootHash [length.Hash] if cell.Nonce < 128 && cell.Nonce != 0 { nonceBytes = 0 } else { - nonceBytes = (bits.Len64(cell.Nonce) + 7) / 8 + nonceBytes = common.BitLenToByteLen(bits.Len64(cell.Nonce)) } var structLength = uint(balanceBytes + nonceBytes + 2) @@ -420,7 +420,7 @@ func (cell *Cell) accountForHashing(buffer []byte, storageRootHash [length.Hash] buffer[0] = byte(192 + structLength) pos = 1 } else { - lengthBytes := (bits.Len(structLength) + 7) / 8 + lengthBytes := common.BitLenToByteLen(bits.Len(structLength)) buffer[0] = byte(247 + lengthBytes) for i := lengthBytes; i > 0; i-- { diff --git a/common/bytes.go b/common/bytes.go index 8b8695e70..59929736c 100644 --- a/common/bytes.go +++ b/common/bytes.go @@ -1,5 +1,5 @@ /* - Copyright 2021 Erigon contributors + Copyright 2021 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -61,3 +61,7 @@ func EnsureEnoughSize(in []byte, size int) []byte { } return in[:size] // Reuse the space if it has enough capacity } + +func BitLenToByteLen(bitLen int) (byteLen int) { + return (bitLen + 7) / 8 +} diff --git a/go.mod b/go.mod index 9d9557d2a..a36cc3384 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/go-stack/stack v1.8.1 github.com/google/btree v1.1.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 - github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/holiman/uint256 v1.2.2 github.com/matryer/moq v0.3.1 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 @@ -37,12 +37,12 @@ require ( github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/btree v1.6.0 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.10.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.8.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.9.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.55.0 + google.golang.org/grpc v1.56.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.30.0 ) @@ -109,10 +109,10 @@ require ( go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.7.0 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index f58730ccd..716463ffd 100644 --- a/go.sum +++ b/go.sum @@ -202,8 +202,8 @@ github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= +github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/holiman/uint256 v1.2.2 h1:TXKcSGc2WaxPD2+bmzAsVthL4+pEN0YwXcL5qED83vk= github.com/holiman/uint256 v1.2.2/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= @@ -419,8 +419,8 @@ golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= @@ -462,8 +462,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -474,8 +474,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -509,8 +509,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -524,8 +524,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -555,8 +555,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -564,8 +564,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE= +google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/kv/bitmapdb/bitmapdb.go b/kv/bitmapdb/bitmapdb.go index b6945a374..0311eb607 100644 --- a/kv/bitmapdb/bitmapdb.go +++ b/kv/bitmapdb/bitmapdb.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Erigon contributors + Copyright 2022 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +26,8 @@ import ( "github.com/RoaringBitmap/roaring" "github.com/RoaringBitmap/roaring/roaring64" "github.com/c2h5oh/datasize" - libcommon "github.com/ledgerwatch/erigon-lib/common" + + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv" ) @@ -171,7 +172,7 @@ func TruncateRange(db kv.RwTx, bucket string, key []byte, to uint32) error { if _, err := chunk.WriteTo(buf); err != nil { return err } - return db.Put(bucket, chunkKey, libcommon.Copy(buf.Bytes())) + return db.Put(bucket, chunkKey, common.Copy(buf.Bytes())) }) } @@ -326,7 +327,7 @@ func TruncateRange64(db kv.RwTx, bucket string, key []byte, to uint64) error { if _, err := chunk.WriteTo(buf); err != nil { return err } - return db.Put(bucket, chunkKey, libcommon.Copy(buf.Bytes())) + return db.Put(bucket, chunkKey, common.Copy(buf.Bytes())) }) } @@ -408,7 +409,7 @@ func Walk(c kv.Cursor, startkey []byte, fixedbits int, walker func(k, v []byte) } func Bytesmask(fixedbits int) (fixedbytes int, mask byte) { - fixedbytes = (fixedbits + 7) / 8 + fixedbytes = common.BitLenToByteLen(fixedbits) shiftbits := fixedbits & 7 mask = byte(0xff) if shiftbits != 0 { diff --git a/kv/iter/iter_test.go b/kv/iter/iter_test.go index 3cad8a467..2c0a02a1b 100644 --- a/kv/iter/iter_test.go +++ b/kv/iter/iter_test.go @@ -85,12 +85,12 @@ func TestUnionPairs(t *testing.T) { require := require.New(t) tx, _ := db.BeginRw(ctx) defer tx.Rollback() - _ = tx.Put(kv.AccountsHistory, []byte{1}, []byte{1}) - _ = tx.Put(kv.AccountsHistory, []byte{3}, []byte{1}) - _ = tx.Put(kv.AccountsHistory, []byte{4}, []byte{1}) + _ = tx.Put(kv.E2AccountsHistory, []byte{1}, []byte{1}) + _ = tx.Put(kv.E2AccountsHistory, []byte{3}, []byte{1}) + _ = tx.Put(kv.E2AccountsHistory, []byte{4}, []byte{1}) _ = tx.Put(kv.PlainState, []byte{2}, []byte{9}) _ = tx.Put(kv.PlainState, []byte{3}, []byte{9}) - it, _ := tx.Range(kv.AccountsHistory, nil, nil) + it, _ := tx.Range(kv.E2AccountsHistory, nil, nil) it2, _ := tx.Range(kv.PlainState, nil, nil) keys, values, err := iter.ToKVArray(iter.UnionKV(it, it2, -1)) require.NoError(err) @@ -103,7 +103,7 @@ func TestUnionPairs(t *testing.T) { defer tx.Rollback() _ = tx.Put(kv.PlainState, []byte{2}, []byte{9}) _ = tx.Put(kv.PlainState, []byte{3}, []byte{9}) - it, _ := tx.Range(kv.AccountsHistory, nil, nil) + it, _ := tx.Range(kv.E2AccountsHistory, nil, nil) it2, _ := tx.Range(kv.PlainState, nil, nil) keys, _, err := iter.ToKVArray(iter.UnionKV(it, it2, -1)) require.NoError(err) @@ -113,10 +113,10 @@ func TestUnionPairs(t *testing.T) { require := require.New(t) tx, _ := db.BeginRw(ctx) defer tx.Rollback() - _ = tx.Put(kv.AccountsHistory, []byte{1}, []byte{1}) - _ = tx.Put(kv.AccountsHistory, []byte{3}, []byte{1}) - _ = tx.Put(kv.AccountsHistory, []byte{4}, []byte{1}) - it, _ := tx.Range(kv.AccountsHistory, nil, nil) + _ = tx.Put(kv.E2AccountsHistory, []byte{1}, []byte{1}) + _ = tx.Put(kv.E2AccountsHistory, []byte{3}, []byte{1}) + _ = tx.Put(kv.E2AccountsHistory, []byte{4}, []byte{1}) + it, _ := tx.Range(kv.E2AccountsHistory, nil, nil) it2, _ := tx.Range(kv.PlainState, nil, nil) keys, _, err := iter.ToKVArray(iter.UnionKV(it, it2, -1)) require.NoError(err) @@ -126,7 +126,7 @@ func TestUnionPairs(t *testing.T) { require := require.New(t) tx, _ := db.BeginRw(ctx) defer tx.Rollback() - it, _ := tx.Range(kv.AccountsHistory, nil, nil) + it, _ := tx.Range(kv.E2AccountsHistory, nil, nil) it2, _ := tx.Range(kv.PlainState, nil, nil) m := iter.UnionKV(it, it2, -1) require.False(m.HasNext()) diff --git a/kv/kv_interface.go b/kv/kv_interface.go index 3f27762ab..bd51347cd 100644 --- a/kv/kv_interface.go +++ b/kv/kv_interface.go @@ -469,6 +469,21 @@ type RwCursor interface { DeleteCurrent() error } +// CursorDupSort +// +// Example: +// +// for k, v, err = cursor.First(); k != nil; k, v, err = cursor.NextNoDup() { +// if err != nil { +// return err +// } +// for ; v != nil; _, v, err = cursor.NextDup() { +// if err != nil { +// return err +// } +// +// } +// } type CursorDupSort interface { Cursor diff --git a/kv/mdbx/kv_mdbx.go b/kv/mdbx/kv_mdbx.go index 13ea83546..45eebc919 100644 --- a/kv/mdbx/kv_mdbx.go +++ b/kv/mdbx/kv_mdbx.go @@ -489,9 +489,6 @@ func (db *MdbxKV) BeginRo(ctx context.Context) (txn kv.Tx, err error) { } defer func() { - if err == nil { - db.wg.Add(1) - } if txn == nil { // on error, or if there is whatever reason that we don't return a tx, // we need to free up the limiter slot, otherwise it could lead to deadlocks @@ -503,6 +500,7 @@ func (db *MdbxKV) BeginRo(ctx context.Context) (txn kv.Tx, err error) { if err != nil { return nil, fmt.Errorf("%w, label: %s, trace: %s", err, db.opts.label.String(), stack2.Trace().String()) } + db.wg.Add(1) return &MdbxTx{ ctx: ctx, db: db, @@ -530,17 +528,12 @@ func (db *MdbxKV) beginRw(ctx context.Context, flags uint) (txn kv.RwTx, err err return nil, fmt.Errorf("db closed") } runtime.LockOSThread() - defer func() { - if err == nil { - db.wg.Add(1) - } - }() - tx, err := db.env.BeginTxn(nil, flags) if err != nil { runtime.UnlockOSThread() // unlock only in case of error. normal flow is "defer .Rollback()" return nil, fmt.Errorf("%w, lable: %s, trace: %s", err, db.opts.label.String(), stack2.Trace().String()) } + db.wg.Add(1) return &MdbxTx{ db: db, tx: tx, @@ -554,7 +547,7 @@ type MdbxTx struct { db *MdbxKV cursors map[uint64]*mdbx.Cursor streams []kv.Closer - statelessCursors map[string]kv.Cursor + statelessCursors map[string]kv.RwCursor readOnly bool cursorID uint64 ctx context.Context @@ -897,18 +890,18 @@ func (tx *MdbxTx) closeCursors() { func (tx *MdbxTx) statelessCursor(bucket string) (kv.RwCursor, error) { if tx.statelessCursors == nil { - tx.statelessCursors = make(map[string]kv.Cursor) + tx.statelessCursors = make(map[string]kv.RwCursor) } c, ok := tx.statelessCursors[bucket] if !ok { var err error - c, err = tx.Cursor(bucket) + c, err = tx.RwCursor(bucket) if err != nil { return nil, err } tx.statelessCursors[bucket] = c } - return c.(kv.RwCursor), nil + return c, nil } func (tx *MdbxTx) Put(table string, k, v []byte) error { @@ -1099,9 +1092,6 @@ func (c *MdbxCursor) delAllDupData() error { return c.c.Del(mdbx func (c *MdbxCursor) put(k, v []byte) error { return c.c.Put(k, v, 0) } func (c *MdbxCursor) putCurrent(k, v []byte) error { return c.c.Put(k, v, mdbx.Current) } func (c *MdbxCursor) putNoOverwrite(k, v []byte) error { return c.c.Put(k, v, mdbx.NoOverwrite) } -func (c *MdbxCursor) putNoDupData(k, v []byte) error { return c.c.Put(k, v, mdbx.NoDupData) } -func (c *MdbxCursor) append(k, v []byte) error { return c.c.Put(k, v, mdbx.Append) } -func (c *MdbxCursor) appendDup(k, v []byte) error { return c.c.Put(k, v, mdbx.AppendDup) } func (c *MdbxCursor) getBoth(k, v []byte) ([]byte, error) { _, v, err := c.c.Get(k, v, mdbx.GetBoth) return v, err @@ -1447,11 +1437,8 @@ func (c *MdbxCursor) SeekExact(key []byte) ([]byte, []byte, error) { // Cast your cursor to *MdbxCursor to use this method. // Return error - if provided data will not sorted (or bucket have old records which mess with new in sorting manner). func (c *MdbxCursor) Append(k []byte, v []byte) error { - if len(k) == 0 { - return fmt.Errorf("mdbx doesn't support empty keys. bucket: %s", c.bucketName) - } - b := c.bucketCfg - if b.AutoDupSortKeysConversion { + if c.bucketCfg.AutoDupSortKeysConversion { + b := c.bucketCfg from, to := b.DupFromLen, b.DupToLen if len(k) != from && len(k) >= to { return fmt.Errorf("append dupsort bucket: %s, can have keys of len==%d and len<%d. key: %x,%d", c.bucketName, from, to, k, len(k)) @@ -1463,13 +1450,14 @@ func (c *MdbxCursor) Append(k []byte, v []byte) error { } } - if b.Flags&mdbx.DupSort != 0 { - if err := c.appendDup(k, v); err != nil { + if c.bucketCfg.Flags&mdbx.DupSort != 0 { + if err := c.c.Put(k, v, mdbx.AppendDup); err != nil { return fmt.Errorf("bucket: %s, %w", c.bucketName, err) } return nil } - if err := c.append(k, v); err != nil { + + if err := c.c.Put(k, v, mdbx.Append); err != nil { return fmt.Errorf("bucket: %s, %w", c.bucketName, err) } return nil @@ -1601,14 +1589,14 @@ func (c *MdbxDupSortCursor) Append(k []byte, v []byte) error { } func (c *MdbxDupSortCursor) AppendDup(k []byte, v []byte) error { - if err := c.appendDup(k, v); err != nil { + if err := c.c.Put(k, v, mdbx.AppendDup); err != nil { return fmt.Errorf("in AppendDup: bucket=%s, %w", c.bucketName, err) } return nil } -func (c *MdbxDupSortCursor) PutNoDupData(key, value []byte) error { - if err := c.putNoDupData(key, value); err != nil { +func (c *MdbxDupSortCursor) PutNoDupData(k, v []byte) error { + if err := c.c.Put(k, v, mdbx.NoDupData); err != nil { return fmt.Errorf("in PutNoDupData: %w", err) } diff --git a/kv/tables.go b/kv/tables.go index ea10f9fc9..652950dae 100644 --- a/kv/tables.go +++ b/kv/tables.go @@ -158,8 +158,8 @@ StorageHistory key - address + storage_key + shard_id_u64 value - roaring bitmap - list of block where it changed */ -const AccountsHistory = "AccountHistory" -const StorageHistory = "StorageHistory" +const E2AccountsHistory = "AccountHistory" +const E2StorageHistory = "StorageHistory" const ( @@ -365,43 +365,46 @@ const ( BittorrentCompletion = "BittorrentCompletion" BittorrentInfo = "BittorrentInfo" - // Domains and Inverted Indices - AccountKeys = "AccountKeys" - AccountVals = "AccountVals" - AccountHistoryKeys = "AccountHistoryKeys" - AccountHistoryVals = "AccountHistoryVals" - AccountIdx = "AccountIdx" - - StorageKeys = "StorageKeys" - StorageVals = "StorageVals" - StorageHistoryKeys = "StorageHistoryKeys" - StorageHistoryVals = "StorageHistoryVals" - StorageIdx = "StorageIdx" - - CodeKeys = "CodeKeys" - CodeVals = "CodeVals" - CodeHistoryKeys = "CodeHistoryKeys" - CodeHistoryVals = "CodeHistoryVals" - CodeIdx = "CodeIdx" - - CommitmentKeys = "CommitmentKeys" - CommitmentVals = "CommitmentVals" - CommitmentHistoryKeys = "CommitmentHistoryKeys" - CommitmentHistoryVals = "CommitmentHistoryVals" - CommitmentIdx = "CommitmentIdx" - - LogAddressKeys = "LogAddressKeys" - LogAddressIdx = "LogAddressIdx" - LogTopicsKeys = "LogTopicsKeys" - LogTopicsIdx = "LogTopicsIdx" - - TracesFromKeys = "TracesFromKeys" - TracesFromIdx = "TracesFromIdx" - TracesToKeys = "TracesToKeys" - TracesToIdx = "TracesToIdx" + // Domains/Histry/InvertedIndices + // Contants have "Tbl" prefix, to avoid collision with actual Domain names + // This constants is very rarely used in APP, but Domain/History/Idx names are widely used + TblAccountKeys = "AccountKeys" + TblAccountVals = "AccountVals" + TblAccountHistoryKeys = "AccountHistoryKeys" + TblAccountHistoryVals = "AccountHistoryVals" + TblAccountIdx = "AccountIdx" + + TblStorageKeys = "StorageKeys" + TblStorageVals = "StorageVals" + TblStorageHistoryKeys = "StorageHistoryKeys" + TblStorageHistoryVals = "StorageHistoryVals" + TblStorageIdx = "StorageIdx" + + TblCodeKeys = "CodeKeys" + TblCodeVals = "CodeVals" + TblCodeHistoryKeys = "CodeHistoryKeys" + TblCodeHistoryVals = "CodeHistoryVals" + TblCodeIdx = "CodeIdx" + + TblCommitmentKeys = "CommitmentKeys" + TblCommitmentVals = "CommitmentVals" + TblCommitmentHistoryKeys = "CommitmentHistoryKeys" + TblCommitmentHistoryVals = "CommitmentHistoryVals" + TblCommitmentIdx = "CommitmentIdx" + + TblLogAddressKeys = "LogAddressKeys" + TblLogAddressIdx = "LogAddressIdx" + TblLogTopicsKeys = "LogTopicsKeys" + TblLogTopicsIdx = "LogTopicsIdx" + + TblTracesFromKeys = "TracesFromKeys" + TblTracesFromIdx = "TracesFromIdx" + TblTracesToKeys = "TracesToKeys" + TblTracesToIdx = "TracesToIdx" Snapshots = "Snapshots" // name -> hash + //State Reconstitution RAccountKeys = "RAccountKeys" RAccountIdx = "RAccountIdx" RStorageKeys = "RStorageKeys" @@ -472,8 +475,8 @@ var ( // This list will be sorted in `init` method. // ChaindataTablesCfg - can be used to find index in sorted version of ChaindataTables list by name var ChaindataTables = []string{ - AccountsHistory, - StorageHistory, + E2AccountsHistory, + E2StorageHistory, Code, ContractCode, HeaderNumber, @@ -527,39 +530,39 @@ var ChaindataTables = []string{ BorReceipts, BorTxLookup, BorSeparate, - AccountKeys, - AccountVals, - AccountHistoryKeys, - AccountHistoryVals, - AccountIdx, - - StorageKeys, - StorageVals, - StorageHistoryKeys, - StorageHistoryVals, - StorageIdx, - - CodeKeys, - CodeVals, - CodeHistoryKeys, - CodeHistoryVals, - CodeIdx, - - CommitmentKeys, - CommitmentVals, - CommitmentHistoryKeys, - CommitmentHistoryVals, - CommitmentIdx, - - LogAddressKeys, - LogAddressIdx, - LogTopicsKeys, - LogTopicsIdx, - - TracesFromKeys, - TracesFromIdx, - TracesToKeys, - TracesToIdx, + TblAccountKeys, + TblAccountVals, + TblAccountHistoryKeys, + TblAccountHistoryVals, + TblAccountIdx, + + TblStorageKeys, + TblStorageVals, + TblStorageHistoryKeys, + TblStorageHistoryVals, + TblStorageIdx, + + TblCodeKeys, + TblCodeVals, + TblCodeHistoryKeys, + TblCodeHistoryVals, + TblCodeIdx, + + TblCommitmentKeys, + TblCommitmentVals, + TblCommitmentHistoryKeys, + TblCommitmentHistoryVals, + TblCommitmentIdx, + + TblLogAddressKeys, + TblLogAddressIdx, + TblLogTopicsKeys, + TblLogTopicsIdx, + + TblTracesFromKeys, + TblTracesFromIdx, + TblTracesToKeys, + TblTracesToIdx, Snapshots, MaxTxNum, @@ -665,34 +668,34 @@ var ChaindataTablesCfg = TableCfg{ }, CallTraceSet: {Flags: DupSort}, - AccountKeys: {Flags: DupSort}, - AccountHistoryKeys: {Flags: DupSort}, - AccountHistoryVals: {Flags: DupSort}, - AccountIdx: {Flags: DupSort}, - StorageKeys: {Flags: DupSort}, - StorageHistoryKeys: {Flags: DupSort}, - StorageHistoryVals: {Flags: DupSort}, - StorageIdx: {Flags: DupSort}, - CodeKeys: {Flags: DupSort}, - CodeHistoryKeys: {Flags: DupSort}, - CodeIdx: {Flags: DupSort}, - CommitmentKeys: {Flags: DupSort}, - CommitmentHistoryKeys: {Flags: DupSort}, - CommitmentIdx: {Flags: DupSort}, - LogAddressKeys: {Flags: DupSort}, - LogAddressIdx: {Flags: DupSort}, - LogTopicsKeys: {Flags: DupSort}, - LogTopicsIdx: {Flags: DupSort}, - TracesFromKeys: {Flags: DupSort}, - TracesFromIdx: {Flags: DupSort}, - TracesToKeys: {Flags: DupSort}, - TracesToIdx: {Flags: DupSort}, - RAccountKeys: {Flags: DupSort}, - RAccountIdx: {Flags: DupSort}, - RStorageKeys: {Flags: DupSort}, - RStorageIdx: {Flags: DupSort}, - RCodeKeys: {Flags: DupSort}, - RCodeIdx: {Flags: DupSort}, + TblAccountKeys: {Flags: DupSort}, + TblAccountHistoryKeys: {Flags: DupSort}, + TblAccountHistoryVals: {Flags: DupSort}, + TblAccountIdx: {Flags: DupSort}, + TblStorageKeys: {Flags: DupSort}, + TblStorageHistoryKeys: {Flags: DupSort}, + TblStorageHistoryVals: {Flags: DupSort}, + TblStorageIdx: {Flags: DupSort}, + TblCodeKeys: {Flags: DupSort}, + TblCodeHistoryKeys: {Flags: DupSort}, + TblCodeIdx: {Flags: DupSort}, + TblCommitmentKeys: {Flags: DupSort}, + TblCommitmentHistoryKeys: {Flags: DupSort}, + TblCommitmentIdx: {Flags: DupSort}, + TblLogAddressKeys: {Flags: DupSort}, + TblLogAddressIdx: {Flags: DupSort}, + TblLogTopicsKeys: {Flags: DupSort}, + TblLogTopicsIdx: {Flags: DupSort}, + TblTracesFromKeys: {Flags: DupSort}, + TblTracesFromIdx: {Flags: DupSort}, + TblTracesToKeys: {Flags: DupSort}, + TblTracesToIdx: {Flags: DupSort}, + RAccountKeys: {Flags: DupSort}, + RAccountIdx: {Flags: DupSort}, + RStorageKeys: {Flags: DupSort}, + RStorageIdx: {Flags: DupSort}, + RCodeKeys: {Flags: DupSort}, + RCodeIdx: {Flags: DupSort}, } var TxpoolTablesCfg = TableCfg{} @@ -776,3 +779,28 @@ func reinit() { } } } + +// Temporal + +const ( + AccountsDomain Domain = "AccountsDomain" + StorageDomain Domain = "StorageDomain" + CodeDomain Domain = "CodeDomain" +) + +const ( + AccountsHistory History = "AccountsHistory" + StorageHistory History = "StorageHistory" + CodeHistory History = "CodeHistory" +) + +const ( + AccountsHistoryIdx InvertedIdx = "AccountsHistoryIdx" + StorageHistoryIdx InvertedIdx = "StorageHistoryIdx" + CodeHistoryIdx InvertedIdx = "CodeHistoryIdx" + + LogTopicIdx InvertedIdx = "LogTopicIdx" + LogAddrIdx InvertedIdx = "LogAddrIdx" + TracesFromIdx InvertedIdx = "TracesFromIdx" + TracesToIdx InvertedIdx = "TracesToIdx" +) diff --git a/kv/temporal/historyv2/changeset.go b/kv/temporal/historyv2/changeset.go index 8bb203917..3fb716ec7 100644 --- a/kv/temporal/historyv2/changeset.go +++ b/kv/temporal/historyv2/changeset.go @@ -223,7 +223,7 @@ type CSMapper struct { var Mapper = map[string]CSMapper{ kv.AccountChangeSet: { - IndexBucket: kv.AccountsHistory, + IndexBucket: kv.E2AccountsHistory, IndexChunkKey: AccountIndexChunkKey, New: NewAccountChangeSet, Find: FindAccount, @@ -231,7 +231,7 @@ var Mapper = map[string]CSMapper{ Decode: DecodeAccounts, }, kv.StorageChangeSet: { - IndexBucket: kv.StorageHistory, + IndexBucket: kv.E2StorageHistory, IndexChunkKey: StorageIndexChunkKey, Find: FindStorage, New: NewStorageChangeSet, diff --git a/recsplit/index.go b/recsplit/index.go index cfcba6216..73c6452eb 100644 --- a/recsplit/index.go +++ b/recsplit/index.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Erigon contributors + Copyright 2022 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,10 +28,12 @@ import ( "time" "unsafe" + "github.com/ledgerwatch/log/v3" + + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/mmap" "github.com/ledgerwatch/erigon-lib/recsplit/eliasfano16" "github.com/ledgerwatch/erigon-lib/recsplit/eliasfano32" - "github.com/ledgerwatch/log/v3" ) // Index implements index lookup from the file created by the RecSplit @@ -299,7 +301,7 @@ func (idx *Index) RewriteWithOffsets(w *bufio.Writer, m map[uint64]uint64) error maxOffset = offset } } - bytesPerRec := (bits.Len64(maxOffset) + 7) / 8 + bytesPerRec := common.BitLenToByteLen(bits.Len64(maxOffset)) var numBuf [8]byte // Write baseDataID binary.BigEndian.PutUint64(numBuf[:], idx.baseDataID) diff --git a/recsplit/recsplit.go b/recsplit/recsplit.go index d7b90ddac..0129bc633 100644 --- a/recsplit/recsplit.go +++ b/recsplit/recsplit.go @@ -1,5 +1,5 @@ /* - Copyright 2021 Erigon contributors + Copyright 2021 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -28,12 +28,14 @@ import ( "path/filepath" "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/log/v3" + "github.com/spaolacci/murmur3" + + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/assert" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/recsplit/eliasfano16" "github.com/ledgerwatch/erigon-lib/recsplit/eliasfano32" - "github.com/ledgerwatch/log/v3" - "github.com/spaolacci/murmur3" ) var ErrCollision = fmt.Errorf("duplicate key") @@ -556,7 +558,7 @@ func (rs *RecSplit) Build() error { return fmt.Errorf("write number of keys: %w", err) } // Write number of bytes per index record - rs.bytesPerRec = (bits.Len64(rs.maxOffset) + 7) / 8 + rs.bytesPerRec = common.BitLenToByteLen(bits.Len64(rs.maxOffset)) if err = rs.indexW.WriteByte(byte(rs.bytesPerRec)); err != nil { return fmt.Errorf("write bytes per record: %w", err) } diff --git a/rlp/encodel.go b/rlp/encodel.go index 390e48eb9..e4527b039 100644 --- a/rlp/encodel.go +++ b/rlp/encodel.go @@ -1,5 +1,5 @@ /* - Copyright 2021 Erigon contributors + Copyright 2021 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ package rlp import ( "encoding/binary" "math/bits" + + "github.com/ledgerwatch/erigon-lib/common" ) // General design: @@ -38,14 +40,14 @@ import ( func ListPrefixLen(dataLen int) int { if dataLen >= 56 { - return 1 + (bits.Len64(uint64(dataLen))+7)/8 + return 1 + common.BitLenToByteLen(bits.Len64(uint64(dataLen))) } return 1 } func EncodeListPrefix(dataLen int, to []byte) int { if dataLen >= 56 { _ = to[9] - beLen := (bits.Len64(uint64(dataLen)) + 7) / 8 + beLen := common.BitLenToByteLen(bits.Len64(uint64(dataLen))) binary.BigEndian.PutUint64(to[1:], uint64(dataLen)) to[8-beLen] = 247 + byte(beLen) copy(to, to[8-beLen:9]) @@ -59,14 +61,14 @@ func U32Len(i uint32) int { if i < 128 { return 1 } - return 1 + (bits.Len32(i)+7)/8 + return 1 + common.BitLenToByteLen(bits.Len32(i)) } func U64Len(i uint64) int { if i < 128 { return 1 } - return 1 + (bits.Len64(i)+7)/8 + return 1 + common.BitLenToByteLen(bits.Len64(i)) } func EncodeU32(i uint32, to []byte) int { @@ -185,7 +187,7 @@ func StringLen(s []byte) int { sLen := len(s) switch { case sLen > 56: - beLen := (bits.Len(uint(sLen)) + 7) / 8 + beLen := common.BitLenToByteLen(bits.Len(uint(sLen))) return 1 + beLen + sLen case sLen == 0: return 1 @@ -201,7 +203,7 @@ func StringLen(s []byte) int { func EncodeString(s []byte, to []byte) int { switch { case len(s) > 56: - beLen := (bits.Len(uint(len(s))) + 7) / 8 + beLen := common.BitLenToByteLen(bits.Len(uint(len(s)))) binary.BigEndian.PutUint64(to[1:], uint64(len(s))) _ = to[beLen+len(s)] diff --git a/rlp/parse.go b/rlp/parse.go index d611a94a9..4710d0cd9 100644 --- a/rlp/parse.go +++ b/rlp/parse.go @@ -1,5 +1,5 @@ /* - Copyright 2021 Erigon contributors + Copyright 2021 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ import ( "fmt" "github.com/holiman/uint256" + + "github.com/ledgerwatch/erigon-lib/common" ) var ( @@ -223,7 +225,7 @@ func U256Len(z *uint256.Int) int { if nBits <= 7 { return 1 } - return 1 + (nBits+7)/8 + return 1 + common.BitLenToByteLen(nBits) } func ParseHash(payload []byte, pos int, hashbuf []byte) (int, error) { diff --git a/state/aggregator.go b/state/aggregator.go index bae315256..0f7199246 100644 --- a/state/aggregator.go +++ b/state/aggregator.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Erigon contributors + Copyright 2022 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,15 +29,16 @@ import ( "github.com/VictoriaMetrics/metrics" "github.com/holiman/uint256" - "github.com/ledgerwatch/erigon-lib/common/background" - "github.com/ledgerwatch/erigon-lib/kv/iter" - "github.com/ledgerwatch/erigon-lib/kv/order" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" "github.com/ledgerwatch/erigon-lib/commitment" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/background" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/iter" + "github.com/ledgerwatch/erigon-lib/kv/order" ) // StepsInBiggestFile - files of this size are completely frozen/immutable. @@ -120,32 +121,32 @@ func NewAggregator(dir, tmpdir string, aggregationStep uint64, commitmentMode Co if err != nil { return nil, err } - if a.accounts, err = NewDomain(dir, tmpdir, aggregationStep, "accounts", kv.AccountKeys, kv.AccountVals, kv.AccountHistoryKeys, kv.AccountHistoryVals, kv.AccountIdx, false, false, logger); err != nil { + if a.accounts, err = NewDomain(dir, tmpdir, aggregationStep, "accounts", kv.TblAccountKeys, kv.TblAccountVals, kv.TblAccountHistoryKeys, kv.TblAccountHistoryVals, kv.TblAccountIdx, false, false, logger); err != nil { return nil, err } - if a.storage, err = NewDomain(dir, tmpdir, aggregationStep, "storage", kv.StorageKeys, kv.StorageVals, kv.StorageHistoryKeys, kv.StorageHistoryVals, kv.StorageIdx, false, false, logger); err != nil { + if a.storage, err = NewDomain(dir, tmpdir, aggregationStep, "storage", kv.TblStorageKeys, kv.TblStorageVals, kv.TblStorageHistoryKeys, kv.TblStorageHistoryVals, kv.TblStorageIdx, false, false, logger); err != nil { return nil, err } - if a.code, err = NewDomain(dir, tmpdir, aggregationStep, "code", kv.CodeKeys, kv.CodeVals, kv.CodeHistoryKeys, kv.CodeHistoryVals, kv.CodeIdx, true, true, logger); err != nil { + if a.code, err = NewDomain(dir, tmpdir, aggregationStep, "code", kv.TblCodeKeys, kv.TblCodeVals, kv.TblCodeHistoryKeys, kv.TblCodeHistoryVals, kv.TblCodeIdx, true, true, logger); err != nil { return nil, err } - commitd, err := NewDomain(dir, tmpdir, aggregationStep, "commitment", kv.CommitmentKeys, kv.CommitmentVals, kv.CommitmentHistoryKeys, kv.CommitmentHistoryVals, kv.CommitmentIdx, false, true, logger) + commitd, err := NewDomain(dir, tmpdir, aggregationStep, "commitment", kv.TblCommitmentKeys, kv.TblCommitmentVals, kv.TblCommitmentHistoryKeys, kv.TblCommitmentHistoryVals, kv.TblCommitmentIdx, false, true, logger) if err != nil { return nil, err } a.commitment = NewCommittedDomain(commitd, commitmentMode, commitTrieVariant, logger) - if a.logAddrs, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "logaddrs", kv.LogAddressKeys, kv.LogAddressIdx, false, nil, logger); err != nil { + if a.logAddrs, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "logaddrs", kv.TblLogAddressKeys, kv.TblLogAddressIdx, false, nil, logger); err != nil { return nil, err } - if a.logTopics, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "logtopics", kv.LogTopicsKeys, kv.LogTopicsIdx, false, nil, logger); err != nil { + if a.logTopics, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "logtopics", kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, false, nil, logger); err != nil { return nil, err } - if a.tracesFrom, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "tracesfrom", kv.TracesFromKeys, kv.TracesFromIdx, false, nil, logger); err != nil { + if a.tracesFrom, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "tracesfrom", kv.TblTracesFromKeys, kv.TblTracesFromIdx, false, nil, logger); err != nil { return nil, err } - if a.tracesTo, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "tracesto", kv.TracesToKeys, kv.TracesToIdx, false, nil, logger); err != nil { + if a.tracesTo, err = NewInvertedIndex(dir, tmpdir, aggregationStep, "tracesto", kv.TblTracesToKeys, kv.TblTracesToIdx, false, nil, logger); err != nil { return nil, err } closeAgg = false @@ -1293,7 +1294,7 @@ func DecodeAccountBytes(enc []byte) (nonce uint64, balance *uint256.Int, hash [] func EncodeAccountBytes(nonce uint64, balance *uint256.Int, hash []byte, incarnation uint64) []byte { l := int(1) if nonce > 0 { - l += (bits.Len64(nonce) + 7) / 8 + l += common.BitLenToByteLen(bits.Len64(nonce)) } l++ if !balance.IsZero() { @@ -1305,7 +1306,7 @@ func EncodeAccountBytes(nonce uint64, balance *uint256.Int, hash []byte, incarna } l++ if incarnation > 0 { - l += (bits.Len64(incarnation) + 7) / 8 + l += common.BitLenToByteLen(bits.Len64(incarnation)) } value := make([]byte, l) pos := 0 @@ -1314,7 +1315,7 @@ func EncodeAccountBytes(nonce uint64, balance *uint256.Int, hash []byte, incarna value[pos] = 0 pos++ } else { - nonceBytes := (bits.Len64(nonce) + 7) / 8 + nonceBytes := common.BitLenToByteLen(bits.Len64(nonce)) value[pos] = byte(nonceBytes) var nonce = nonce for i := nonceBytes; i > 0; i-- { @@ -1345,7 +1346,7 @@ func EncodeAccountBytes(nonce uint64, balance *uint256.Int, hash []byte, incarna if incarnation == 0 { value[pos] = 0 } else { - incBytes := (bits.Len64(incarnation) + 7) / 8 + incBytes := common.BitLenToByteLen(bits.Len64(incarnation)) value[pos] = byte(incBytes) var inc = incarnation for i := incBytes; i > 0; i-- { diff --git a/state/aggregator_v3.go b/state/aggregator_v3.go index 9e91f7dc5..065b59701 100644 --- a/state/aggregator_v3.go +++ b/state/aggregator_v3.go @@ -105,25 +105,25 @@ func NewAggregatorV3(ctx context.Context, dir, tmpdir string, aggregationStep ui logger: logger, } var err error - if a.accounts, err = NewHistory(dir, a.tmpdir, aggregationStep, "accounts", kv.AccountHistoryKeys, kv.AccountIdx, kv.AccountHistoryVals, false, nil, false, logger); err != nil { + if a.accounts, err = NewHistory(dir, a.tmpdir, aggregationStep, "accounts", kv.TblAccountHistoryKeys, kv.TblAccountIdx, kv.TblAccountHistoryVals, false, nil, false, logger); err != nil { return nil, err } - if a.storage, err = NewHistory(dir, a.tmpdir, aggregationStep, "storage", kv.StorageHistoryKeys, kv.StorageIdx, kv.StorageHistoryVals, false, nil, false, logger); err != nil { + if a.storage, err = NewHistory(dir, a.tmpdir, aggregationStep, "storage", kv.TblStorageHistoryKeys, kv.TblStorageIdx, kv.TblStorageHistoryVals, false, nil, false, logger); err != nil { return nil, err } - if a.code, err = NewHistory(dir, a.tmpdir, aggregationStep, "code", kv.CodeHistoryKeys, kv.CodeIdx, kv.CodeHistoryVals, true, nil, true, logger); err != nil { + if a.code, err = NewHistory(dir, a.tmpdir, aggregationStep, "code", kv.TblCodeHistoryKeys, kv.TblCodeIdx, kv.TblCodeHistoryVals, true, nil, true, logger); err != nil { return nil, err } - if a.logAddrs, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "logaddrs", kv.LogAddressKeys, kv.LogAddressIdx, false, nil, logger); err != nil { + if a.logAddrs, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "logaddrs", kv.TblLogAddressKeys, kv.TblLogAddressIdx, false, nil, logger); err != nil { return nil, err } - if a.logTopics, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "logtopics", kv.LogTopicsKeys, kv.LogTopicsIdx, false, nil, logger); err != nil { + if a.logTopics, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "logtopics", kv.TblLogTopicsKeys, kv.TblLogTopicsIdx, false, nil, logger); err != nil { return nil, err } - if a.tracesFrom, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "tracesfrom", kv.TracesFromKeys, kv.TracesFromIdx, false, nil, logger); err != nil { + if a.tracesFrom, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "tracesfrom", kv.TblTracesFromKeys, kv.TblTracesFromIdx, false, nil, logger); err != nil { return nil, err } - if a.tracesTo, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "tracesto", kv.TracesToKeys, kv.TracesToIdx, false, nil, logger); err != nil { + if a.tracesTo, err = NewInvertedIndex(dir, a.tmpdir, aggregationStep, "tracesto", kv.TblTracesToKeys, kv.TblTracesToIdx, false, nil, logger); err != nil { return nil, err } a.recalcMaxTxNum() @@ -656,7 +656,7 @@ func (a *AggregatorV3) integrateFiles(sf AggV3StaticFiles, txNumFrom, txNumTo ui a.tracesTo.integrateFiles(sf.tracesTo, txNumFrom, txNumTo) } -func (a *AggregatorV3) NeedSaveFilesListInDB() bool { +func (a *AggregatorV3) HasNewFrozenFiles() bool { return a.needSaveFilesListInDB.CompareAndSwap(true, false) } @@ -804,8 +804,8 @@ func (a *AggregatorV3) CanPrune(tx kv.Tx) bool { return a.CanPruneFrom(tx) < a.minimaxTxNumInFiles.Load() } func (a *AggregatorV3) CanPruneFrom(tx kv.Tx) uint64 { - fst, _ := kv.FirstKey(tx, kv.TracesToKeys) - fst2, _ := kv.FirstKey(tx, kv.StorageHistoryKeys) + fst, _ := kv.FirstKey(tx, kv.TblTracesToKeys) + fst2, _ := kv.FirstKey(tx, kv.TblStorageHistoryKeys) if len(fst) > 0 && len(fst2) > 0 { fstInDb := binary.BigEndian.Uint64(fst) fstInDb2 := binary.BigEndian.Uint64(fst2) @@ -1293,11 +1293,11 @@ func (a *AggregatorV3) AddCodePrev(addr []byte, prev []byte) error { func (a *AggregatorV3) PutIdx(idx kv.InvertedIdx, key []byte) error { switch idx { - case kv.TracesFromIdx: + case kv.TblTracesFromIdx: return a.tracesFrom.Add(key) - case kv.TracesToIdx: + case kv.TblTracesToIdx: return a.tracesTo.Add(key) - case kv.LogAddressIdx: + case kv.TblLogAddressIdx: return a.logAddrs.Add(key) case kv.LogTopicIndex: return a.logTopics.Add(key) diff --git a/state/btree_index.go b/state/btree_index.go index 713f38b50..7ddcaaee3 100644 --- a/state/btree_index.go +++ b/state/btree_index.go @@ -720,7 +720,7 @@ func (btw *BtIndexWriter) Build() error { return fmt.Errorf("write number of keys: %w", err) } // Write number of bytes per index record - btw.bytesPerRec = (bits.Len64(btw.maxOffset) + 7) / 8 + btw.bytesPerRec = common.BitLenToByteLen(bits.Len64(btw.maxOffset)) if err = btw.indexW.WriteByte(byte(btw.bytesPerRec)); err != nil { return fmt.Errorf("write bytes per record: %w", err) } diff --git a/txpool/mocks_test.go b/txpool/mocks_test.go index 4b7767220..98d280d21 100644 --- a/txpool/mocks_test.go +++ b/txpool/mocks_test.go @@ -7,6 +7,7 @@ import ( "context" "github.com/ledgerwatch/erigon-lib/gointerfaces/remote" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/txpool/txpoolcfg" types2 "github.com/ledgerwatch/erigon-lib/types" "sync" ) @@ -53,7 +54,7 @@ var _ Pool = &PoolMock{} // } type PoolMock struct { // AddLocalTxsFunc mocks the AddLocalTxs method. - AddLocalTxsFunc func(ctx context.Context, newTxs types2.TxSlots, tx kv.Tx) ([]DiscardReason, error) + AddLocalTxsFunc func(ctx context.Context, newTxs types2.TxSlots, tx kv.Tx) ([]txpoolcfg.DiscardReason, error) // AddNewGoodPeerFunc mocks the AddNewGoodPeer method. AddNewGoodPeerFunc func(peerID types2.PeerID) @@ -146,7 +147,7 @@ type PoolMock struct { } // AddLocalTxs calls AddLocalTxsFunc. -func (mock *PoolMock) AddLocalTxs(ctx context.Context, newTxs types2.TxSlots, tx kv.Tx) ([]DiscardReason, error) { +func (mock *PoolMock) AddLocalTxs(ctx context.Context, newTxs types2.TxSlots, tx kv.Tx) ([]txpoolcfg.DiscardReason, error) { callInfo := struct { Ctx context.Context NewTxs types2.TxSlots @@ -161,7 +162,7 @@ func (mock *PoolMock) AddLocalTxs(ctx context.Context, newTxs types2.TxSlots, tx mock.lockAddLocalTxs.Unlock() if mock.AddLocalTxsFunc == nil { var ( - discardReasonsOut []DiscardReason + discardReasonsOut []txpoolcfg.DiscardReason errOut error ) return discardReasonsOut, errOut diff --git a/txpool/pool.go b/txpool/pool.go index cc1ba39d8..7282b4d84 100644 --- a/txpool/pool.go +++ b/txpool/pool.go @@ -49,7 +49,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/fixedgas" - emath "github.com/ledgerwatch/erigon-lib/common/math" "github.com/ledgerwatch/erigon-lib/common/u256" "github.com/ledgerwatch/erigon-lib/gointerfaces" "github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil" @@ -82,7 +81,7 @@ type Pool interface { // Handle 3 main events - new remote txs from p2p, new local txs from RPC, new blocks from execution layer AddRemoteTxs(ctx context.Context, newTxs types.TxSlots) - AddLocalTxs(ctx context.Context, newTxs types.TxSlots, tx kv.Tx) ([]DiscardReason, error) + AddLocalTxs(ctx context.Context, newTxs types.TxSlots, tx kv.Tx) ([]txpoolcfg.DiscardReason, error) OnNewBlock(ctx context.Context, stateChanges *remote.StateChangeBatch, unwindTxs, minedTxs types.TxSlots, tx kv.Tx) error // IdHashKnown check whether transaction with given Id hash is known to the pool @@ -116,90 +115,6 @@ const ( QueuedPoolBits = EnoughFeeCapProtocol ) -type DiscardReason uint8 - -const ( - NotSet DiscardReason = 0 // analog of "nil-value", means it will be set in future - Success DiscardReason = 1 - AlreadyKnown DiscardReason = 2 - Mined DiscardReason = 3 - ReplacedByHigherTip DiscardReason = 4 - UnderPriced DiscardReason = 5 - ReplaceUnderpriced DiscardReason = 6 // if a transaction is attempted to be replaced with a different one without the required price bump. - FeeTooLow DiscardReason = 7 - OversizedData DiscardReason = 8 - InvalidSender DiscardReason = 9 - NegativeValue DiscardReason = 10 // ensure no one is able to specify a transaction with a negative value. - Spammer DiscardReason = 11 - PendingPoolOverflow DiscardReason = 12 - BaseFeePoolOverflow DiscardReason = 13 - QueuedPoolOverflow DiscardReason = 14 - GasUintOverflow DiscardReason = 15 - IntrinsicGas DiscardReason = 16 - RLPTooLong DiscardReason = 17 - NonceTooLow DiscardReason = 18 - InsufficientFunds DiscardReason = 19 - NotReplaced DiscardReason = 20 // There was an existing transaction with the same sender and nonce, not enough price bump to replace - DuplicateHash DiscardReason = 21 // There was an existing transaction with the same hash - InitCodeTooLarge DiscardReason = 22 // EIP-3860 - transaction init code is too large - TxTypeNotSupported DiscardReason = 23 -) - -func (r DiscardReason) String() string { - switch r { - case NotSet: - return "not set" - case Success: - return "success" - case AlreadyKnown: - return "already known" - case Mined: - return "mined" - case ReplacedByHigherTip: - return "replaced by transaction with higher tip" - case UnderPriced: - return "underpriced" - case ReplaceUnderpriced: - return "replacement transaction underpriced" - case FeeTooLow: - return "fee too low" - case OversizedData: - return "oversized data" - case InvalidSender: - return "invalid sender" - case NegativeValue: - return "negative value" - case Spammer: - return "spammer" - case PendingPoolOverflow: - return "pending sub-pool is full" - case BaseFeePoolOverflow: - return "baseFee sub-pool is full" - case QueuedPoolOverflow: - return "queued sub-pool is full" - case GasUintOverflow: - return "GasUintOverflow" - case IntrinsicGas: - return "IntrinsicGas" - case RLPTooLong: - return "RLPTooLong" - case NonceTooLow: - return "nonce too low" - case InsufficientFunds: - return "insufficient funds" - case NotReplaced: - return "could not replace existing tx" - case DuplicateHash: - return "existing tx with same hash" - case InitCodeTooLarge: - return "initcode too large" - case TxTypeNotSupported: - return types.ErrTxTypeNotSupported.Error() - default: - panic(fmt.Sprintf("discard reason: %d", r)) - } -} - // metaTx holds transaction and some metadata type metaTx struct { Tx *types.TxSlot @@ -286,9 +201,9 @@ type TxPool struct { // - batch notifications about new txs (reduce P2P spam to other nodes about txs propagation) // - and as a result reducing lock contention unprocessedRemoteTxs *types.TxSlots - unprocessedRemoteByHash map[string]int // to reject duplicates - byHash map[string]*metaTx // tx_hash => tx : only not committed to db yet records - discardReasonsLRU *simplelru.LRU[string, DiscardReason] // tx_hash => discard_reason : non-persisted + unprocessedRemoteByHash map[string]int // to reject duplicates + byHash map[string]*metaTx // tx_hash => tx : only not committed to db yet records + discardReasonsLRU *simplelru.LRU[string, txpoolcfg.DiscardReason] // tx_hash => discard_reason : non-persisted pending *PendingPool baseFee *SubPool queued *SubPool @@ -322,7 +237,7 @@ func New(newTxs chan types.Announcements, coreDB kv.RoDB, cfg txpoolcfg.Config, if err != nil { return nil, err } - discardHistory, err := simplelru.NewLRU[string, DiscardReason](10_000, nil) + discardHistory, err := simplelru.NewLRU[string, txpoolcfg.DiscardReason](10_000, nil) if err != nil { return nil, err } @@ -729,7 +644,7 @@ func (p *TxPool) best(n uint16, txs *types.TxsRlp, tx kv.Tx, onTopOf, availableG // make sure we have enough gas in the caller to add this transaction. // not an exact science using intrinsic gas but as close as we could hope for at // this stage - intrinsicGas, _ := CalcIntrinsicGas(uint64(mt.Tx.DataLen), uint64(mt.Tx.DataNonZeroLen), nil, mt.Tx.Creation, true, true, isShanghai) + intrinsicGas, _ := txpoolcfg.CalcIntrinsicGas(uint64(mt.Tx.DataLen), uint64(mt.Tx.DataNonZeroLen), nil, mt.Tx.Creation, true, true, isShanghai) if intrinsicGas > availableGas { // we might find another TX with a low enough intrinsic gas to include so carry on continue @@ -793,18 +708,18 @@ func (p *TxPool) AddRemoteTxs(_ context.Context, newTxs types.TxSlots) { } } -func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache.CacheView) DiscardReason { +func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache.CacheView) txpoolcfg.DiscardReason { // No unauthenticated deposits allowed in the transaction pool. // This is for spam protection, not consensus, // as the external engine-API user authenticates deposits. if txn.Type == types.DepositTxType { - return TxTypeNotSupported + return txpoolcfg.TxTypeNotSupported } isShanghai := p.isShanghai() if isShanghai { if txn.DataLen > fixedgas.MaxInitCodeSize { - return InitCodeTooLarge + return txpoolcfg.InitCodeTooLarge } } @@ -813,13 +728,13 @@ func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache. if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx underpriced idHash=%x local=%t, feeCap=%d, cfg.MinFeeCap=%d", txn.IDHash, isLocal, txn.FeeCap, p.cfg.MinFeeCap)) } - return UnderPriced + return txpoolcfg.UnderPriced } - gas, reason := CalcIntrinsicGas(uint64(txn.DataLen), uint64(txn.DataNonZeroLen), nil, txn.Creation, true, true, isShanghai) + gas, reason := txpoolcfg.CalcIntrinsicGas(uint64(txn.DataLen), uint64(txn.DataNonZeroLen), nil, txn.Creation, true, true, isShanghai) if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx intrinsic gas idHash=%x gas=%d", txn.IDHash, gas)) } - if reason != Success { + if reason != txpoolcfg.Success { if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx intrinsic gas calculated failed idHash=%x reason=%s", txn.IDHash, reason)) } @@ -829,13 +744,13 @@ func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache. if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx intrinsic gas > txn.gas idHash=%x gas=%d, txn.gas=%d", txn.IDHash, gas, txn.Gas)) } - return IntrinsicGas + return txpoolcfg.IntrinsicGas } if !isLocal && uint64(p.all.count(txn.SenderID)) > p.cfg.AccountSlots { if txn.Traced { log.Info(fmt.Sprintf("TX TRACING: validateTx marked as spamming idHash=%x slots=%d, limit=%d", txn.IDHash, p.all.count(txn.SenderID), p.cfg.AccountSlots)) } - return Spammer + return txpoolcfg.Spammer } // check nonce and balance @@ -844,7 +759,7 @@ func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache. if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx nonce too low idHash=%x nonce in state=%d, txn.nonce=%d", txn.IDHash, senderNonce, txn.Nonce)) } - return NonceTooLow + return txpoolcfg.NonceTooLow } // Transactor should have enough funds to cover the costs total := uint256.NewInt(txn.Gas) @@ -854,9 +769,9 @@ func (p *TxPool) validateTx(txn *types.TxSlot, isLocal bool, stateCache kvcache. if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: validateTx insufficient funds idHash=%x balance in state=%d, txn.gas*txn.tip=%d", txn.IDHash, senderBalance, total)) } - return InsufficientFunds + return txpoolcfg.InsufficientFunds } - return Success + return txpoolcfg.Success } func (p *TxPool) isShanghai() bool { @@ -903,10 +818,10 @@ func (p *TxPool) ValidateSerializedTxn(serializedTxn []byte) error { } return nil } -func (p *TxPool) validateTxs(txs *types.TxSlots, stateCache kvcache.CacheView) (reasons []DiscardReason, goodTxs types.TxSlots, err error) { +func (p *TxPool) validateTxs(txs *types.TxSlots, stateCache kvcache.CacheView) (reasons []txpoolcfg.DiscardReason, goodTxs types.TxSlots, err error) { // reasons is pre-sized for direct indexing, with the default zero // value DiscardReason of NotSet - reasons = make([]DiscardReason, len(txs.Txs)) + reasons = make([]txpoolcfg.DiscardReason, len(txs.Txs)) if err := txs.Valid(); err != nil { return reasons, goodTxs, err @@ -915,12 +830,12 @@ func (p *TxPool) validateTxs(txs *types.TxSlots, stateCache kvcache.CacheView) ( goodCount := 0 for i, txn := range txs.Txs { reason := p.validateTx(txn, txs.IsLocal[i], stateCache) - if reason == Success { + if reason == txpoolcfg.Success { goodCount++ // Success here means no DiscardReason yet, so leave it NotSet continue } - if reason == Spammer { + if reason == txpoolcfg.Spammer { p.punishSpammer(txn.SenderID) } reasons[i] = reason @@ -930,7 +845,7 @@ func (p *TxPool) validateTxs(txs *types.TxSlots, stateCache kvcache.CacheView) ( j := 0 for i, txn := range txs.Txs { - if reasons[i] == NotSet { + if reasons[i] == txpoolcfg.NotSet { goodTxs.Txs[j] = txn goodTxs.IsLocal[j] = txs.IsLocal[i] copy(goodTxs.Senders.At(j), txs.Senders.At(i)) @@ -951,27 +866,27 @@ func (p *TxPool) punishSpammer(spammer uint64) { return count > 0 }) for _, mt := range txsToDelete { - p.discardLocked(mt, Spammer) // can't call it while iterating by all + p.discardLocked(mt, txpoolcfg.Spammer) // can't call it while iterating by all } } } -func fillDiscardReasons(reasons []DiscardReason, newTxs types.TxSlots, discardReasonsLRU *simplelru.LRU[string, DiscardReason]) []DiscardReason { +func fillDiscardReasons(reasons []txpoolcfg.DiscardReason, newTxs types.TxSlots, discardReasonsLRU *simplelru.LRU[string, txpoolcfg.DiscardReason]) []txpoolcfg.DiscardReason { for i := range reasons { - if reasons[i] != NotSet { + if reasons[i] != txpoolcfg.NotSet { continue } reason, ok := discardReasonsLRU.Get(string(newTxs.Txs[i].IDHash[:])) if ok { reasons[i] = reason } else { - reasons[i] = Success + reasons[i] = txpoolcfg.Success } } return reasons } -func (p *TxPool) AddLocalTxs(ctx context.Context, newTransactions types.TxSlots, tx kv.Tx) ([]DiscardReason, error) { +func (p *TxPool) AddLocalTxs(ctx context.Context, newTransactions types.TxSlots, tx kv.Tx) ([]txpoolcfg.DiscardReason, error) { coreTx, err := p.coreDB().BeginRo(ctx) if err != nil { return nil, err @@ -1008,7 +923,7 @@ func (p *TxPool) AddLocalTxs(ctx context.Context, newTransactions types.TxSlots, p.pendingBaseFee.Load(), p.blockGasLimit.Load(), p.pending, p.baseFee, p.queued, p.all, p.byHash, p.addLocked, p.discardLocked, true, p.l1Cost, p.logger) if err == nil { for i, reason := range addReasons { - if reason != NotSet { + if reason != txpoolcfg.NotSet { reasons[i] = reason } } @@ -1020,7 +935,7 @@ func (p *TxPool) AddLocalTxs(ctx context.Context, newTransactions types.TxSlots, reasons = fillDiscardReasons(reasons, newTxs, p.discardReasonsLRU) for i, reason := range reasons { - if reason == Success { + if reason == txpoolcfg.Success { txn := newTxs.Txs[i] if txn.Traced { p.logger.Info(fmt.Sprintf("TX TRACING: AddLocalTxs promotes idHash=%x, senderId=%d", txn.IDHash, txn.SenderID)) @@ -1052,8 +967,9 @@ func (p *TxPool) cache() kvcache.Cache { func addTxs(blockNum uint64, cacheView kvcache.CacheView, senders *sendersBatch, newTxs types.TxSlots, pendingBaseFee, blockGasLimit uint64, pending *PendingPool, baseFee, queued *SubPool, - byNonce *BySenderAndNonce, byHash map[string]*metaTx, add func(*metaTx, *types.Announcements) DiscardReason, discard func(*metaTx, DiscardReason), collect bool, l1CostFn L1CostFn, - logger log.Logger) (types.Announcements, []DiscardReason, error) { + byNonce *BySenderAndNonce, byHash map[string]*metaTx, add func(*metaTx, *types.Announcements) txpoolcfg.DiscardReason, discard func(*metaTx, txpoolcfg.DiscardReason), collect bool, + l1CostFn L1CostFn, + logger log.Logger) (types.Announcements, []txpoolcfg.DiscardReason, error) { protocolBaseFee := calcProtocolBaseFee(pendingBaseFee) if assert.Enable { for _, txn := range newTxs.Txs { @@ -1072,11 +988,11 @@ func addTxs(blockNum uint64, cacheView kvcache.CacheView, senders *sendersBatch, // somehow the fact that certain transactions were local, needs to be remembered for some // time (up to some "immutability threshold"). sendersWithChangedState := map[uint64]struct{}{} - discardReasons := make([]DiscardReason, len(newTxs.Txs)) + discardReasons := make([]txpoolcfg.DiscardReason, len(newTxs.Txs)) announcements := types.Announcements{} for i, txn := range newTxs.Txs { if found, ok := byHash[string(txn.IDHash[:])]; ok { - discardReasons[i] = DuplicateHash + discardReasons[i] = txpoolcfg.DuplicateHash // In case if the transation is stuck, "poke" it to rebroadcast if collect && newTxs.IsLocal[i] && (found.currentSubPool == PendingSubPool || found.currentSubPool == BaseFeeSubPool) { announcements.Append(found.Tx.Type, found.Tx.Size, found.Tx.IDHash[:]) @@ -1084,11 +1000,11 @@ func addTxs(blockNum uint64, cacheView kvcache.CacheView, senders *sendersBatch, continue } mt := newMetaTx(txn, newTxs.IsLocal[i], blockNum) - if reason := add(mt, &announcements); reason != NotSet { + if reason := add(mt, &announcements); reason != txpoolcfg.NotSet { discardReasons[i] = reason continue } - discardReasons[i] = NotSet + discardReasons[i] = txpoolcfg.NotSet if txn.Traced { logger.Info(fmt.Sprintf("TX TRACING: schedule sendersWithChangedState idHash=%x senderId=%d", txn.IDHash, mt.Tx.SenderID)) } @@ -1112,7 +1028,8 @@ func addTxs(blockNum uint64, cacheView kvcache.CacheView, senders *sendersBatch, func addTxsOnNewBlock(blockNum uint64, cacheView kvcache.CacheView, stateChanges *remote.StateChangeBatch, senders *sendersBatch, newTxs types.TxSlots, pendingBaseFee uint64, blockGasLimit uint64, pending *PendingPool, baseFee, queued *SubPool, - byNonce *BySenderAndNonce, byHash map[string]*metaTx, add func(*metaTx, *types.Announcements) DiscardReason, discard func(*metaTx, DiscardReason), l1CostFn L1CostFn, + byNonce *BySenderAndNonce, byHash map[string]*metaTx, add func(*metaTx, *types.Announcements) txpoolcfg.DiscardReason, discard func(*metaTx, txpoolcfg.DiscardReason), + l1CostFn L1CostFn, logger log.Logger) (types.Announcements, error) { protocolBaseFee := calcProtocolBaseFee(pendingBaseFee) if assert.Enable { @@ -1138,7 +1055,7 @@ func addTxsOnNewBlock(blockNum uint64, cacheView kvcache.CacheView, stateChanges continue } mt := newMetaTx(txn, newTxs.IsLocal[i], blockNum) - if reason := add(mt, &announcements); reason != NotSet { + if reason := add(mt, &announcements); reason != txpoolcfg.NotSet { discard(mt, reason) continue } @@ -1183,7 +1100,7 @@ func (p *TxPool) setBaseFee(baseFee uint64) (uint64, bool) { return p.pendingBaseFee.Load(), changed } -func (p *TxPool) addLocked(mt *metaTx, announcements *types.Announcements) DiscardReason { +func (p *TxPool) addLocked(mt *metaTx, announcements *types.Announcements) txpoolcfg.DiscardReason { // Insert to pending pool, if pool doesn't have txn with same Nonce and bigger Tip found := p.all.get(mt.Tx.SenderID, mt.Tx.Nonce) if found != nil { @@ -1200,9 +1117,9 @@ func (p *TxPool) addLocked(mt *metaTx, announcements *types.Announcements) Disca announcements.Append(found.Tx.Type, found.Tx.Size, found.Tx.IDHash[:]) } if bytes.Equal(found.Tx.IDHash[:], mt.Tx.IDHash[:]) { - return NotSet + return txpoolcfg.NotSet } - return NotReplaced + return txpoolcfg.NotReplaced } switch found.currentSubPool { @@ -1216,7 +1133,7 @@ func (p *TxPool) addLocked(mt *metaTx, announcements *types.Announcements) Disca //already removed } - p.discardLocked(found, ReplacedByHigherTip) + p.discardLocked(found, txpoolcfg.ReplacedByHigherTip) } p.byHash[string(mt.Tx.IDHash[:])] = mt @@ -1232,12 +1149,12 @@ func (p *TxPool) addLocked(mt *metaTx, announcements *types.Announcements) Disca } // All transactions are first added to the queued pool and then immediately promoted from there if required p.queued.Add(mt, p.logger) - return NotSet + return txpoolcfg.NotSet } // dropping transaction from all sub-structures and from db // Important: don't call it while iterating by all -func (p *TxPool) discardLocked(mt *metaTx, reason DiscardReason) { +func (p *TxPool) discardLocked(mt *metaTx, reason txpoolcfg.DiscardReason) { delete(p.byHash, string(mt.Tx.IDHash[:])) p.deletedTxs = append(p.deletedTxs, mt) p.all.delete(mt) @@ -1261,7 +1178,7 @@ func (p *TxPool) NonceFromAddress(addr [20]byte) (nonce uint64, inPool bool) { // modify state_balance and state_nonce, potentially remove some elements (if transaction with some nonce is // included into a block), and finally, walk over the transaction records and update SubPool fields depending on // the actual presence of nonce gaps and what the balance is. -func removeMined(byNonce *BySenderAndNonce, minedTxs []*types.TxSlot, pending *PendingPool, baseFee, queued *SubPool, discard func(*metaTx, DiscardReason), logger log.Logger) error { +func removeMined(byNonce *BySenderAndNonce, minedTxs []*types.TxSlot, pending *PendingPool, baseFee, queued *SubPool, discard func(*metaTx, txpoolcfg.DiscardReason), logger log.Logger) error { noncesToRemove := map[uint64]uint64{} for _, txn := range minedTxs { nonce, ok := noncesToRemove[txn.SenderID] @@ -1300,7 +1217,7 @@ func removeMined(byNonce *BySenderAndNonce, minedTxs []*types.TxSlot, pending *P }) for _, mt := range toDel { - discard(mt, Mined) + discard(mt, txpoolcfg.Mined) } toDel = toDel[:0] } @@ -1312,7 +1229,7 @@ func removeMined(byNonce *BySenderAndNonce, minedTxs []*types.TxSlot, pending *P // nonces, and also affect other transactions from the same sender with higher nonce, it loops through all transactions // for a given senderID func onSenderStateChange(senderID uint64, senderNonce uint64, senderBalance uint256.Int, byNonce *BySenderAndNonce, - protocolBaseFee, blockGasLimit uint64, pending *PendingPool, baseFee, queued *SubPool, discard func(*metaTx, DiscardReason), l1CostFn L1CostFn, logger log.Logger) { + protocolBaseFee, blockGasLimit uint64, pending *PendingPool, baseFee, queued *SubPool, discard func(*metaTx, txpoolcfg.DiscardReason), l1CostFn L1CostFn, logger log.Logger) { noGapsNonce := senderNonce cumulativeRequiredBalance := uint256.NewInt(0) minFeeCap := uint256.NewInt(0).SetAllOne() @@ -1425,13 +1342,13 @@ func onSenderStateChange(senderID uint64, senderNonce uint64, senderBalance uint return true }) for _, mt := range toDel { - discard(mt, NonceTooLow) + discard(mt, txpoolcfg.NonceTooLow) } } // promote reasserts invariants of the subpool and returns the list of transactions that ended up // being promoted to the pending or basefee pool, for re-broadcasting -func promote(pending *PendingPool, baseFee, queued *SubPool, pendingBaseFee uint64, discard func(*metaTx, DiscardReason), announcements *types.Announcements, +func promote(pending *PendingPool, baseFee, queued *SubPool, pendingBaseFee uint64, discard func(*metaTx, txpoolcfg.DiscardReason), announcements *types.Announcements, logger log.Logger) { // Demote worst transactions that do not qualify for pending sub pool anymore, to other sub pools, or discard for worst := pending.Worst(); pending.Len() > 0 && (worst.subPool < BaseFeePoolBits || worst.minFeeCap.Cmp(uint256.NewInt(pendingBaseFee)) < 0); worst = pending.Worst() { @@ -1442,7 +1359,7 @@ func promote(pending *PendingPool, baseFee, queued *SubPool, pendingBaseFee uint } else if worst.subPool >= QueuedPoolBits { queued.Add(pending.PopWorst(), logger) } else { - discard(pending.PopWorst(), FeeTooLow) + discard(pending.PopWorst(), txpoolcfg.FeeTooLow) } } @@ -1458,7 +1375,7 @@ func promote(pending *PendingPool, baseFee, queued *SubPool, pendingBaseFee uint if worst.subPool >= QueuedPoolBits { queued.Add(baseFee.PopWorst(), logger) } else { - discard(baseFee.PopWorst(), FeeTooLow) + discard(baseFee.PopWorst(), txpoolcfg.FeeTooLow) } } @@ -1475,22 +1392,22 @@ func promote(pending *PendingPool, baseFee, queued *SubPool, pendingBaseFee uint // Discard worst transactions from the queued sub pool if they do not qualify for worst := queued.Worst(); queued.Len() > 0 && worst.subPool < QueuedPoolBits; worst = queued.Worst() { - discard(queued.PopWorst(), FeeTooLow) + discard(queued.PopWorst(), txpoolcfg.FeeTooLow) } // Discard worst transactions from pending pool until it is within capacity limit for pending.Len() > pending.limit { - discard(pending.PopWorst(), PendingPoolOverflow) + discard(pending.PopWorst(), txpoolcfg.PendingPoolOverflow) } // Discard worst transactions from pending sub pool until it is within capacity limits for baseFee.Len() > baseFee.limit { - discard(baseFee.PopWorst(), BaseFeePoolOverflow) + discard(baseFee.PopWorst(), txpoolcfg.BaseFeePoolOverflow) } // Discard worst transactions from the queued sub pool until it is within its capacity limits for _ = queued.Worst(); queued.Len() > queued.limit; _ = queued.Worst() { - discard(queued.PopWorst(), QueuedPoolOverflow) + discard(queued.PopWorst(), txpoolcfg.QueuedPoolOverflow) } } @@ -1801,7 +1718,7 @@ func (p *TxPool) fromDB(ctx context.Context, tx kv.Tx, coreTx kv.Tx) error { isLocalTx := p.isLocalLRU.Contains(string(k)) - if reason := p.validateTx(txn, isLocalTx, cacheView); reason != NotSet && reason != Success { + if reason := p.validateTx(txn, isLocalTx, cacheView); reason != txpoolcfg.NotSet && reason != txpoolcfg.Success { return nil } txs.Resize(uint(i + 1)) @@ -1953,87 +1870,6 @@ func (p *TxPoolDropRemote) AddRemoteTxs(ctx context.Context, newTxs types.TxSlot // consume remote tx from fetch } -// CalcIntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func CalcIntrinsicGas(dataLen, dataNonZeroLen uint64, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028, isShanghai bool) (uint64, DiscardReason) { - // Set the starting gas for the raw transaction - var gas uint64 - if isContractCreation && isHomestead { - gas = fixedgas.TxGasContractCreation - } else { - gas = fixedgas.TxGas - } - // Bump the required gas by the amount of transactional data - if dataLen > 0 { - // Zero and non-zero bytes are priced differently - nz := dataNonZeroLen - // Make sure we don't exceed uint64 for all data combinations - nonZeroGas := fixedgas.TxDataNonZeroGasFrontier - if isEIP2028 { - nonZeroGas = fixedgas.TxDataNonZeroGasEIP2028 - } - - product, overflow := emath.SafeMul(nz, nonZeroGas) - if overflow { - return 0, GasUintOverflow - } - gas, overflow = emath.SafeAdd(gas, product) - if overflow { - return 0, GasUintOverflow - } - - z := dataLen - nz - - product, overflow = emath.SafeMul(z, fixedgas.TxDataZeroGas) - if overflow { - return 0, GasUintOverflow - } - gas, overflow = emath.SafeAdd(gas, product) - if overflow { - return 0, GasUintOverflow - } - - if isContractCreation && isShanghai { - numWords := toWordSize(dataLen) - product, overflow = emath.SafeMul(numWords, fixedgas.InitCodeWordGas) - if overflow { - return 0, GasUintOverflow - } - gas, overflow = emath.SafeAdd(gas, product) - if overflow { - return 0, GasUintOverflow - } - } - } - if accessList != nil { - product, overflow := emath.SafeMul(uint64(len(accessList)), fixedgas.TxAccessListAddressGas) - if overflow { - return 0, GasUintOverflow - } - gas, overflow = emath.SafeAdd(gas, product) - if overflow { - return 0, GasUintOverflow - } - - product, overflow = emath.SafeMul(uint64(accessList.StorageKeys()), fixedgas.TxAccessListStorageKeyGas) - if overflow { - return 0, GasUintOverflow - } - gas, overflow = emath.SafeAdd(gas, product) - if overflow { - return 0, GasUintOverflow - } - } - return gas, Success -} - -// toWordSize returns the ceiled word size required for memory expansion. -func toWordSize(size uint64) uint64 { - if size > math.MaxUint64-31 { - return math.MaxUint64/32 + 1 - } - return (size + 31) / 32 -} - var PoolChainConfigKey = []byte("chain_config") var PoolLastSeenBlockKey = []byte("last_seen_block") var PoolPendingBaseFeeKey = []byte("pending_base_fee") diff --git a/txpool/pool_test.go b/txpool/pool_test.go index 6cd524c81..95c4c313c 100644 --- a/txpool/pool_test.go +++ b/txpool/pool_test.go @@ -93,7 +93,7 @@ func TestNonceFromAddress(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } @@ -118,7 +118,7 @@ func TestNonceFromAddress(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -138,7 +138,7 @@ func TestNonceFromAddress(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(InsufficientFunds, reason, reason.String()) + assert.Equal(txpoolcfg.InsufficientFunds, reason, reason.String()) } } @@ -156,7 +156,7 @@ func TestNonceFromAddress(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(NonceTooLow, reason, reason.String()) + assert.Equal(txpoolcfg.NonceTooLow, reason, reason.String()) } } } @@ -213,7 +213,7 @@ func TestReplaceWithHigherFee(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } // Bumped only feeCap, transaction not accepted @@ -230,7 +230,7 @@ func TestReplaceWithHigherFee(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(NotReplaced, reason, reason.String()) + assert.Equal(txpoolcfg.NotReplaced, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -250,7 +250,7 @@ func TestReplaceWithHigherFee(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(NotReplaced, reason, reason.String()) + assert.Equal(txpoolcfg.NotReplaced, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -270,7 +270,7 @@ func TestReplaceWithHigherFee(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -330,7 +330,7 @@ func TestReverseNonces(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } fmt.Printf("AFTER TX 1\n") @@ -358,7 +358,7 @@ func TestReverseNonces(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } fmt.Printf("AFTER TX 2\n") @@ -386,7 +386,7 @@ func TestReverseNonces(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } fmt.Printf("AFTER TX 3\n") @@ -459,7 +459,7 @@ func TestTxPoke(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } var promoted types.Announcements @@ -485,7 +485,7 @@ func TestTxPoke(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(DuplicateHash, reason, reason.String()) + assert.Equal(txpoolcfg.DuplicateHash, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -514,7 +514,7 @@ func TestTxPoke(t *testing.T) { reasons, err := pool.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(NotReplaced, reason, reason.String()) + assert.Equal(txpoolcfg.NotReplaced, reason, reason.String()) } nonce, ok := pool.NonceFromAddress(addr) assert.True(ok) @@ -623,8 +623,8 @@ func TestShanghaiIntrinsicGas(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { - gas, reason := CalcIntrinsicGas(c.dataLen, c.dataNonZeroLen, nil, c.creation, true, true, c.isShanghai) - if reason != Success { + gas, reason := txpoolcfg.CalcIntrinsicGas(c.dataLen, c.dataNonZeroLen, nil, c.creation, true, true, c.isShanghai) + if reason != txpoolcfg.Success { t.Errorf("expected success but got reason %v", reason) } if gas != c.expected { @@ -637,27 +637,27 @@ func TestShanghaiIntrinsicGas(t *testing.T) { func TestShanghaiValidateTx(t *testing.T) { asrt := assert.New(t) tests := map[string]struct { - expected DiscardReason + expected txpoolcfg.DiscardReason dataLen int isShanghai bool }{ "no shanghai": { - expected: Success, + expected: txpoolcfg.Success, dataLen: 32, isShanghai: false, }, "shanghai within bounds": { - expected: Success, + expected: txpoolcfg.Success, dataLen: 32, isShanghai: true, }, "shanghai exactly on bound": { - expected: Success, + expected: txpoolcfg.Success, dataLen: fixedgas.MaxInitCodeSize, isShanghai: true, }, "shanghai one over bound": { - expected: InitCodeTooLarge, + expected: txpoolcfg.InitCodeTooLarge, dataLen: fixedgas.MaxInitCodeSize + 1, isShanghai: true, }, @@ -760,8 +760,8 @@ func TestDepositTxValidateTx(t *testing.T) { reason := pool.validateTx(txn, false, view) - if reason != TxTypeNotSupported { - t.Errorf("expected %v, got %v", TxTypeNotSupported, reason) + if reason != txpoolcfg.TxTypeNotSupported { + t.Errorf("expected %v, got %v", txpoolcfg.TxTypeNotSupported, reason) } } @@ -822,7 +822,7 @@ func TestDropRemote(t *testing.T) { reasons, err := txPoolDropRemote.AddLocalTxs(ctx, txSlots, tx) assert.NoError(err) for _, reason := range reasons { - assert.Equal(Success, reason, reason.String()) + assert.Equal(txpoolcfg.Success, reason, reason.String()) } } select { diff --git a/txpool/txpool_grpc_server.go b/txpool/txpool_grpc_server.go index ec539cab2..a15080aa0 100644 --- a/txpool/txpool_grpc_server.go +++ b/txpool/txpool_grpc_server.go @@ -28,6 +28,7 @@ import ( grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/txpool/txpoolcfg" "github.com/ledgerwatch/log/v3" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -53,7 +54,7 @@ type txPool interface { PeekBest(n uint16, txs *types.TxsRlp, tx kv.Tx, onTopOf, availableGas uint64) (bool, error) GetRlp(tx kv.Tx, hash []byte) ([]byte, error) - AddLocalTxs(ctx context.Context, newTxs types.TxSlots, tx kv.Tx) ([]DiscardReason, error) + AddLocalTxs(ctx context.Context, newTxs types.TxSlots, tx kv.Tx) ([]txpoolcfg.DiscardReason, error) deprecatedForEach(_ context.Context, f func(rlp []byte, sender common.Address, t SubPoolType), tx kv.Tx) CountContent() (int, int, int) IdHashKnown(tx kv.Tx, hash []byte) (bool, error) @@ -198,10 +199,10 @@ func (s *GrpcServer) Add(ctx context.Context, in *txpool_proto.AddRequest) (*txp return nil }); err != nil { if errors.Is(err, types.ErrAlreadyKnown) { // Noop, but need to handle to not count these - reply.Errors[i] = AlreadyKnown.String() + reply.Errors[i] = txpoolcfg.AlreadyKnown.String() reply.Imported[i] = txpool_proto.ImportResult_ALREADY_EXISTS } else if errors.Is(err, types.ErrRlpTooBig) { // Noop, but need to handle to not count these - reply.Errors[i] = RLPTooLong.String() + reply.Errors[i] = txpoolcfg.RLPTooLong.String() reply.Imported[i] = txpool_proto.ImportResult_INVALID } else { reply.Errors[i] = err.Error() @@ -231,15 +232,15 @@ func (s *GrpcServer) Add(ctx context.Context, in *txpool_proto.AddRequest) (*txp return reply, nil } -func mapDiscardReasonToProto(reason DiscardReason) txpool_proto.ImportResult { +func mapDiscardReasonToProto(reason txpoolcfg.DiscardReason) txpool_proto.ImportResult { switch reason { - case Success: + case txpoolcfg.Success: return txpool_proto.ImportResult_SUCCESS - case AlreadyKnown: + case txpoolcfg.AlreadyKnown: return txpool_proto.ImportResult_ALREADY_EXISTS - case UnderPriced, ReplaceUnderpriced, FeeTooLow: + case txpoolcfg.UnderPriced, txpoolcfg.ReplaceUnderpriced, txpoolcfg.FeeTooLow: return txpool_proto.ImportResult_FEE_TOO_LOW - case InvalidSender, NegativeValue, OversizedData, InitCodeTooLarge, RLPTooLong, TxTypeNotSupported: + case txpoolcfg.InvalidSender, txpoolcfg.NegativeValue, txpoolcfg.OversizedData, txpoolcfg.InitCodeTooLarge, txpoolcfg.RLPTooLong, txpoolcfg.TxTypeNotSupported: return txpool_proto.ImportResult_INVALID default: return txpool_proto.ImportResult_INTERNAL_ERROR diff --git a/txpool/txpoolcfg/txpoolcfg.go b/txpool/txpoolcfg/txpoolcfg.go index e62586950..e256d7d52 100644 --- a/txpool/txpoolcfg/txpoolcfg.go +++ b/txpool/txpoolcfg/txpoolcfg.go @@ -1,8 +1,14 @@ package txpoolcfg import ( + "fmt" + "math" "math/big" "time" + + "github.com/ledgerwatch/erigon-lib/common/fixedgas" + emath "github.com/ledgerwatch/erigon-lib/common/math" + "github.com/ledgerwatch/erigon-lib/types" ) type Config struct { @@ -42,3 +48,168 @@ var DefaultConfig = Config{ Optimism: false, NoTxGossip: false, } + +type DiscardReason uint8 + +const ( + NotSet DiscardReason = 0 // analog of "nil-value", means it will be set in future + Success DiscardReason = 1 + AlreadyKnown DiscardReason = 2 + Mined DiscardReason = 3 + ReplacedByHigherTip DiscardReason = 4 + UnderPriced DiscardReason = 5 + ReplaceUnderpriced DiscardReason = 6 // if a transaction is attempted to be replaced with a different one without the required price bump. + FeeTooLow DiscardReason = 7 + OversizedData DiscardReason = 8 + InvalidSender DiscardReason = 9 + NegativeValue DiscardReason = 10 // ensure no one is able to specify a transaction with a negative value. + Spammer DiscardReason = 11 + PendingPoolOverflow DiscardReason = 12 + BaseFeePoolOverflow DiscardReason = 13 + QueuedPoolOverflow DiscardReason = 14 + GasUintOverflow DiscardReason = 15 + IntrinsicGas DiscardReason = 16 + RLPTooLong DiscardReason = 17 + NonceTooLow DiscardReason = 18 + InsufficientFunds DiscardReason = 19 + NotReplaced DiscardReason = 20 // There was an existing transaction with the same sender and nonce, not enough price bump to replace + DuplicateHash DiscardReason = 21 // There was an existing transaction with the same hash + InitCodeTooLarge DiscardReason = 22 // EIP-3860 - transaction init code is too large + TxTypeNotSupported DiscardReason = 23 +) + +func (r DiscardReason) String() string { + switch r { + case NotSet: + return "not set" + case Success: + return "success" + case AlreadyKnown: + return "already known" + case Mined: + return "mined" + case ReplacedByHigherTip: + return "replaced by transaction with higher tip" + case UnderPriced: + return "underpriced" + case ReplaceUnderpriced: + return "replacement transaction underpriced" + case FeeTooLow: + return "fee too low" + case OversizedData: + return "oversized data" + case InvalidSender: + return "invalid sender" + case NegativeValue: + return "negative value" + case Spammer: + return "spammer" + case PendingPoolOverflow: + return "pending sub-pool is full" + case BaseFeePoolOverflow: + return "baseFee sub-pool is full" + case QueuedPoolOverflow: + return "queued sub-pool is full" + case GasUintOverflow: + return "GasUintOverflow" + case IntrinsicGas: + return "IntrinsicGas" + case RLPTooLong: + return "RLPTooLong" + case NonceTooLow: + return "nonce too low" + case InsufficientFunds: + return "insufficient funds" + case NotReplaced: + return "could not replace existing tx" + case DuplicateHash: + return "existing tx with same hash" + case InitCodeTooLarge: + return "initcode too large" + case TxTypeNotSupported: + return types.ErrTxTypeNotSupported.Error() + default: + panic(fmt.Sprintf("discard reason: %d", r)) + } +} + +// CalcIntrinsicGas computes the 'intrinsic gas' for a message with the given data. +func CalcIntrinsicGas(dataLen, dataNonZeroLen uint64, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028, isShanghai bool) (uint64, DiscardReason) { + // Set the starting gas for the raw transaction + var gas uint64 + if isContractCreation && isHomestead { + gas = fixedgas.TxGasContractCreation + } else { + gas = fixedgas.TxGas + } + // Bump the required gas by the amount of transactional data + if dataLen > 0 { + // Zero and non-zero bytes are priced differently + nz := dataNonZeroLen + // Make sure we don't exceed uint64 for all data combinations + nonZeroGas := fixedgas.TxDataNonZeroGasFrontier + if isEIP2028 { + nonZeroGas = fixedgas.TxDataNonZeroGasEIP2028 + } + + product, overflow := emath.SafeMul(nz, nonZeroGas) + if overflow { + return 0, GasUintOverflow + } + gas, overflow = emath.SafeAdd(gas, product) + if overflow { + return 0, GasUintOverflow + } + + z := dataLen - nz + + product, overflow = emath.SafeMul(z, fixedgas.TxDataZeroGas) + if overflow { + return 0, GasUintOverflow + } + gas, overflow = emath.SafeAdd(gas, product) + if overflow { + return 0, GasUintOverflow + } + + if isContractCreation && isShanghai { + numWords := toWordSize(dataLen) + product, overflow = emath.SafeMul(numWords, fixedgas.InitCodeWordGas) + if overflow { + return 0, GasUintOverflow + } + gas, overflow = emath.SafeAdd(gas, product) + if overflow { + return 0, GasUintOverflow + } + } + } + if accessList != nil { + product, overflow := emath.SafeMul(uint64(len(accessList)), fixedgas.TxAccessListAddressGas) + if overflow { + return 0, GasUintOverflow + } + gas, overflow = emath.SafeAdd(gas, product) + if overflow { + return 0, GasUintOverflow + } + + product, overflow = emath.SafeMul(uint64(accessList.StorageKeys()), fixedgas.TxAccessListStorageKeyGas) + if overflow { + return 0, GasUintOverflow + } + gas, overflow = emath.SafeAdd(gas, product) + if overflow { + return 0, GasUintOverflow + } + } + return gas, Success +} + +// toWordSize returns the ceiled word size required for memory expansion. +func toWordSize(size uint64) uint64 { + if size > math.MaxUint64-31 { + return math.MaxUint64/32 + 1 + } + return (size + 31) / 32 +} diff --git a/types/ssz/ssz.go b/types/ssz/ssz.go index 72a6440bb..6ecae5a13 100644 --- a/types/ssz/ssz.go +++ b/types/ssz/ssz.go @@ -1,5 +1,5 @@ /* - Copyright 2021 Erigon contributors + Copyright 2021 The Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,7 +19,7 @@ package ssz import ( "encoding/binary" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" "github.com/ledgerwatch/erigon-lib/types/clonable" ) @@ -144,7 +144,7 @@ func DecodeStaticList[T Unmarshaler](bytes []byte, start, end, bytesPerElement u return objs, nil } -func DecodeHashList(bytes []byte, start, end, max uint32) ([]libcommon.Hash, error) { +func DecodeHashList(bytes []byte, start, end, max uint32) ([]common.Hash, error) { if start > end || len(bytes) < int(end) { return nil, ErrBadOffset } @@ -157,7 +157,7 @@ func DecodeHashList(bytes []byte, start, end, max uint32) ([]libcommon.Hash, err if elementsNum > max { return nil, ErrTooBigList } - objs := make([]libcommon.Hash, elementsNum) + objs := make([]common.Hash, elementsNum) for i := range objs { copy(objs[i][:], buf[i*length.Hash:]) } diff --git a/types/txn.go b/types/txn.go index 3c62236b7..b178436a1 100644 --- a/types/txn.go +++ b/types/txn.go @@ -430,8 +430,8 @@ func (ctx *TxParseContext) ParseTransaction(payload []byte, pos int, slot *TxSlo if chainIDBits <= 7 { chainIDLen = 1 } else { - chainIDLen = (chainIDBits + 7) / 8 // It is always < 56 bytes - sigHashLen++ // For chainId len Prefix + chainIDLen = common.BitLenToByteLen(chainIDBits) // It is always < 56 bytes + sigHashLen++ // For chainId len Prefix } sigHashLen += uint(chainIDLen) // For chainId sigHashLen += 2 // For two extra zeros @@ -493,7 +493,7 @@ func (ctx *TxParseContext) ParseTransaction(payload []byte, pos int, slot *TxSlo return 0, fmt.Errorf("%w: computing signHash (hashing len Prefix): %s", ErrParseTxn, err) //nolint } } else { - beLen := (bits.Len(sigHashLen) + 7) / 8 + beLen := common.BitLenToByteLen(bits.Len(sigHashLen)) binary.BigEndian.PutUint64(ctx.buf[1:], uint64(sigHashLen)) ctx.buf[8-beLen] = byte(beLen) + 247 if _, err := ctx.Keccak2.Write(ctx.buf[8-beLen : 9]); err != nil { @@ -819,7 +819,7 @@ func EncodeSenderLengthForStorage(nonce uint64, balance uint256.Int) uint { structLength += uint(balance.ByteLen()) + 1 } if nonce > 0 { - structLength += uint((bits.Len64(nonce)+7)/8) + 1 + structLength += uint(common.BitLenToByteLen(bits.Len64(nonce))) + 1 } return structLength } @@ -829,7 +829,7 @@ func EncodeSender(nonce uint64, balance uint256.Int, buffer []byte) { var pos = 1 if nonce > 0 { fieldSet = 1 - nonceBytes := (bits.Len64(nonce) + 7) / 8 + nonceBytes := common.BitLenToByteLen(bits.Len64(nonce)) buffer[pos] = byte(nonceBytes) var nonce = nonce for i := nonceBytes; i > 0; i-- {