Upstream v2.56.2

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
     if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -113,7 +113,7 @@ jobs:
   docker-build-check:
     # don't run this on devel - the PR must have run it to be merged and it misleads that this pushes the docker image
     if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/devel' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/checkout@v3
@@ -129,7 +129,7 @@ jobs:
 
 #  automated-tests:
 #    runs-on:
-#      ubuntu-20.04
+#      ubuntu-22.04
 #    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
 #    steps:
 #      - uses: actions/checkout@v3

.github/workflows/qa-clean-exit.yml

@@ -0,0 +1,89 @@
+name: QA - Clean exit on Ctrl-C
+
+on:
+  push:
+    branches:
+      - devel
+      - alpha
+      - 'release/**'
+  pull_request:
+    branches:
+      - devel
+      - alpha
+      - 'release/**'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+jobs:
+  long-running-test:
+    #if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    #strategy:
+    #  matrix:
+    #    os: [ ubuntu-22.04, macos-13-xlarge ]
+    #runs-on: ${{ matrix.os }}
+    runs-on: self-hosted
+    env:
+      ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.20'
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    #- name: Install dependencies
+    #  run: |
+    #    sudo apt-get update
+    #    sudo apt-get install -y build-essential make gcc
+
+    - name: Clean Erigon Build Directory
+      run: |
+        make clean
+
+    - name: Build Erigon
+      run: |
+        make erigon
+      working-directory: ${{ github.workspace }}
+
+    #- name: Download Python Script for Logs Checking
+    #  run: |
+    #    curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'
+
+    - name: Run Erigon, send ctrl-c and check for clean exiting
+      run: |
+        # Run Erigon, send ctrl-c and check logs
+        python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR
+
+        # Capture monitoring script exit status
+        monitoring_exit_status=$?
+
+        # Clean up Erigon process if it's still running
+        if kill -0 $ERIGON_PID 2> /dev/null; then
+          echo "Terminating Erigon"
+          kill $ERIGON_PID
+          wait $ERIGON_PID
+        else
+          echo "Erigon has already terminated"
+        fi
+
+        # Clean up Erigon build and data directories
+        rm -rf $ERIGON_DATA_DIR
+
+        # Check monitoring script exit status
+        if [ $monitoring_exit_status -eq 0 ]; then
+          echo "Monitoring completed successfully"
+        else
+          echo "Error detected in Erigon logs or monitoring script exited unexpectedly"
+          exit 1
+        fi

.github/workflows/test-integration-caplin.yml

@@ -21,8 +21,7 @@ jobs:
     strategy:
       matrix:
 #        disable macos-11 until https://github.com/ledgerwatch/erigon/issues/8789
-#        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
-        os: [ ubuntu-20.04 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:

.github/workflows/test-integration.yml

@@ -14,7 +14,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:

.gitignore

@@ -95,3 +95,9 @@ jwt.hex
 *__debug_bin*
 yarn.lock
 node_modules
+
+/config.toml
+/config.yaml
+/config.yml
+
+vendor

.goreleaser.yml

@@ -1,4 +1,4 @@
-project_name: op-erigon
+project_name: erigon
 
 release:
   disable: false
@@ -14,7 +14,7 @@ builds:
     env:
       - CC=o64-clang
       - CXX=o64-clang++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w
 
   - id: darwin-arm64
@@ -25,7 +25,7 @@ builds:
     env:
       - CC=oa64-clang
       - CXX=oa64-clang++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w
 
   - id: linux-amd64
@@ -36,7 +36,7 @@ builds:
     env:
       - CC=x86_64-linux-gnu-gcc
       - CXX=x86_64-linux-gnu-g++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w -extldflags "-static" # We need to build a static binary because we are building in a glibc based system and running in a musl container
 
   - id: linux-arm64
@@ -47,27 +47,28 @@ builds:
     env:
       - CC=aarch64-linux-gnu-gcc
       - CXX=aarch64-linux-gnu-g++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w -extldflags "-static" # We need to build a static binary because we are building in a glibc based system and running in a musl container
 
-#  - id: windows-amd64
-#    main: ./cmd/erigon
-#    binary: erigon
-#    goos: [ windows ]
-#    goarch: [ amd64 ]
-#    env:
-#      - CC=x86_64-w64-mingw32-gcc
-#      - CXX=x86_64-w64-mingw32-g++
-#    tags: [ nosqlite, noboltdb ]
-#    ldflags: -s -w
+  - id: windows-amd64
+    main: ./cmd/erigon
+    binary: erigon
+    goos: [ windows ]
+    goarch: [ amd64 ]
+    env:
+      - CC=x86_64-w64-mingw32-gcc
+      - CXX=x86_64-w64-mingw32-g++
+    tags: [ nosqlite, noboltdb, nosilkworm ]
+    ldflags: -s -w
 
 
 snapshot:
   name_template: "{{ .Tag }}.next"
 
 dockers:
   - image_templates:
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-amd64
+      - thorax/{{ .ProjectName }}:{{ .Version }}-amd64
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-amd64
     dockerfile: Dockerfile.release
     use: buildx
     skip_push: true
@@ -78,7 +79,8 @@ dockers:
       - --platform=linux/amd64
 
   - image_templates:
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-arm64
+      - thorax/{{ .ProjectName }}:{{ .Version }}-arm64
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-arm64
     dockerfile: Dockerfile.release
     skip_push: true
     use: buildx
@@ -89,17 +91,29 @@ dockers:
       - --platform=linux/arm64/v8
 
 docker_manifests:
-  - name_template: testinprod/{{ .ProjectName }}:{{ .Version }}
+  - name_template: thorax/{{ .ProjectName }}:{{ .Version }}
+    skip_push: true
+    image_templates:
+      - thorax/{{ .ProjectName }}:{{ .Version }}-amd64
+      - thorax/{{ .ProjectName }}:{{ .Version }}-arm64
+
+  - name_template: ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}
+    skip_push: true
+    image_templates:
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-amd64
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-arm64
+
+  - name_template: thorax/{{ .ProjectName }}:latest
     skip_push: true
     image_templates:
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-amd64
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-arm64
+      - thorax/{{ .ProjectName }}:{{ .Version }}-amd64
+      - thorax/{{ .ProjectName }}:{{ .Version }}-arm64
 
-  - name_template: testinprod/{{ .ProjectName }}:latest
+  - name_template: ghcr.io/ledgerwatch/{{ .ProjectName }}:latest
     skip_push: true
     image_templates:
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-amd64
-      - testinprod/{{ .ProjectName }}:{{ .Version }}-arm64
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-amd64
+      - ghcr.io/ledgerwatch/{{ .ProjectName }}:{{ .Version }}-arm64
 
 announce:
   slack:

Dockerfile

@@ -15,7 +15,7 @@ ADD . .
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/tmp/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    make all
+    make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all
 
 
 FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder

Makefile

@@ -26,16 +26,28 @@ CGO_CFLAGS += -O
 CGO_CFLAGS += -D__BLST_PORTABLE__
 CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes -Wno-unused-but-set-variable
 
+CGO_LDFLAGS := $(shell $(GO) env CGO_LDFLAGS 2> /dev/null)
+ifeq ($(shell uname -s), Darwin)
+	ifeq ($(filter-out 13.%,$(shell sw_vers --productVersion)),)
+		CGO_LDFLAGS += -mmacosx-version-min=13.3
+	endif
+endif
+
 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757
 BUILD_TAGS = nosqlite,noboltdb
+
+ifneq ($(shell "$(CURDIR)/turbo/silkworm/silkworm_compat_check.sh"),)
+	BUILD_TAGS := $(BUILD_TAGS),nosilkworm
+endif
+
 PACKAGE = github.com/testinprod-io/op-erigon
 
 GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false
 GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}"
 
-GOBUILD = CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GO_FLAGS)
-GO_DBG_BUILD = CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l"  # see delve docs
-GOTEST = CGO_CFLAGS="$(CGO_CFLAGS)" GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2
+GOBUILD = CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build $(GO_FLAGS)
+GO_DBG_BUILD = CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l"  # see delve docs
+GOTEST = CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2
 
 default: all
 
@@ -122,6 +134,8 @@ COMMANDS += sentinel
 COMMANDS += caplin
 COMMANDS += caplin-regression
 COMMANDS += tooling
+COMMANDS += snapshots
+
 
 
 
@@ -226,6 +240,16 @@ git-submodules:
 	@git submodule sync --quiet --recursive || true
 	@git submodule update --quiet --init --recursive --force || true
 
+## install:                            copies binaries and libraries to DIST
+DIST ?= $(CURDIR)/build/dist
+.PHONY: install
+install:
+	mkdir -p "$(DIST)"
+	cp -f "$$($(CURDIR)/turbo/silkworm/silkworm_lib_path.sh)" "$(DIST)"
+	cp -f "$(GOBIN)/"* "$(DIST)"
+	@echo "Copied files to $(DIST):"
+	@ls -al "$(DIST)"
+
 PACKAGE_NAME          := github.com/testinprod-io/op-erigon
 GOLANG_CROSS_VERSION  ?= v1.20.7
 

README.md

@@ -224,15 +224,15 @@ in `erigon --help`). We don't allow change this flag after first start.
 System Requirements
 ===================
 
-* For an Archive node of Ethereum Mainnet we recommend >=3TB storage space: 1.8TB state (as of March 2022),
-  200GB temp files (can symlink or mount folder `<datadir>/temp` to another disk). Ethereum Mainnet Full node (
+* For an Archive node of Ethereum Mainnet we recommend >=3.5TB storage space: 2.2TB state (as of December 2023),
+  470GB snapshots (can symlink or mount folder `<datadir>/snapshots` to another disk), 200GB temp files (can symlink or mount folder `<datadir>/temp` to another disk). Ethereum Mainnet Full node (
   see `--prune*` flags): 400Gb (April 2022).
 
 * Goerli Full node (see `--prune*` flags): 189GB on Beta, 114GB on Alpha (April 2022).
 
 * Gnosis Chain Archive: 600GB (October 2023).
 
-* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive:
+* Polygon Mainnet Archive: 8.5TiB (December 2023). `--prune.*.older 15768000`: 5.1Tb (September 2023). Polygon Mumbai Archive:
   1TB. (April 2022).
 
 SSD or NVMe. Do not recommend HDD - on HDD Erigon will always stay N blocks behind chain tip, but not fall behind.
@@ -369,13 +369,7 @@ Support only remote-miners.
   , `--miner.gastarget`
 * JSON-RPC supports methods: eth_coinbase , eth_hashrate, eth_mining, eth_getWork, eth_submitWork, eth_submitHashrate
 * JSON-RPC supports websocket methods: newPendingTransaction
-* TODO:
-    + we don't broadcast mined blocks to p2p-network
-      yet, [but it's easy to accomplish](https://github.com/ledgerwatch/erigon/blob/9b8cdc0f2289a7cef78218a15043de5bdff4465e/eth/downloader/downloader.go#L673)
-    + eth_newPendingTransactionFilter
-    + eth_newBlockFilter
-    + eth_newFilter
-    + websocket Logs
+
 
 <code> 🔬 Detailed explanation is [here](/docs/mining.md).</code>
 
@@ -439,7 +433,7 @@ of the yaml configuration file and sets the chain to goerli
 Example of setting up TOML config file
 
 ```
-`datadir = 'your datadir'
+datadir = 'your datadir'
 port = 1111
 chain = "mainnet"
 http = true

TESTING.md

@@ -2,7 +2,7 @@ Testing of new releases of Erigon should ideally include these checks.
 
 ## Incremental Sync
 
-This check requires having the Erigon database synced previously. Lets assume (for command line examples) it is in the
+This check requires having the Erigon database synced previously. Let's assume (for command line examples) it is in the
 directory `~/mainnet/erigon/chaindata`.
 Using `git pull` or `git checkout`, update the code to the version that is to be released (or very close to it). Then,
 build erigon executable:
@@ -306,7 +306,7 @@ Example of recording command:
 ./build/bin/rpctest bench8 --erigonUrl http://192.168.1.2:8545 --gethUrl http://192.168.1.1:8545 --needCompare --blockFrom 9000000 --blockTo 9000100 --recordFile req.txt
 ```
 
-The file format is plain text, with requests and responses are written in separate lines, and delimited by the tripple
+The file format is plain text, with requests and responses are written in separate lines, and delimited by the triple
 line breaks, like this:
 
 ```