new(fw,tests,github): Add automatic feature-based fixture building in CI, add EOF feature (eip7692), fix outstanding EOF test failures

.github/actions/build-evm/action.yaml

@@ -8,31 +8,41 @@ inputs:
 outputs:
   impl:
     description: "Implementation of EVM binary to build"
-    value: ${{ steps.evm-config-reader.outputs.impl }}
+    value: ${{ steps.config-evm-reader.outputs.impl }}
   repo:
     description: "Repository to use to build the EVM binary"
-    value: ${{ steps.evm-config-reader.outputs.repo }}
+    value: ${{ steps.config-evm-reader.outputs.repo }}
   ref:
     description: "Reference to branch, commit, or tag to use to build the EVM binary"
-    value: ${{ steps.evm-config-reader.outputs.ref }}
+    value: ${{ steps.config-evm-reader.outputs.ref }}
+  evm-bin:
+    description: "Binary name of the evm tool to use"
+    value: ${{ steps.config-evm-reader.outputs.evm-bin }}
 runs:
   using: "composite"
   steps:
-    - name: Get the selected EVM version from the evm-config.yaml
-      id: evm-config-reader
+    - name: Get the selected EVM version from the configs/evm.yaml
+      id: config-evm-reader
       shell: bash
       run: |
-        awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./evm-config.yaml \
+        awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./configs/evm.yaml \
         | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT"
     - name: Print Variables for the selected EVM type
       shell: bash
       run: |
-        echo "Implementation: ${{ steps.evm-config-reader.outputs.impl }}"
-        echo "Repository: ${{ steps.evm-config-reader.outputs.repo }}"
-        echo "Reference: ${{ steps.evm-config-reader.outputs.ref }}"
+        echo "Implementation: ${{ steps.config-evm-reader.outputs.impl }}"
+        echo "Repository: ${{ steps.config-evm-reader.outputs.repo }}"
+        echo "Reference: ${{ steps.config-evm-reader.outputs.ref }}"
+        echo "EVM Binary: ${{ steps.config-evm-reader.outputs.evm-bin }}"
     - name: Build the EVM using Geth action
-      if: steps.evm-config-reader.outputs.impl == 'geth'
+      if: steps.config-evm-reader.outputs.impl == 'geth'
       uses: ./.github/actions/build-geth-evm
       with:
-        repo: ${{ steps.evm-config-reader.outputs.repo }}
-        ref: ${{ steps.evm-config-reader.outputs.ref }}
+        repo: ${{ steps.config-evm-reader.outputs.repo }}
+        ref: ${{ steps.config-evm-reader.outputs.ref }}
+    - name: Build the EVM using EVMONE action
+      if: steps.config-evm-reader.outputs.impl == 'evmone'
+      uses: ./.github/actions/build-evmone-evm
+      with:
+        repo: ${{ steps.config-evm-reader.outputs.repo }}
+        ref: ${{ steps.config-evm-reader.outputs.ref }}

.github/actions/build-evmone-evm/action.yaml

@@ -0,0 +1,31 @@
+name: 'Build evmone EVM'
+description: 'Builds the evmone EVM binary'
+inputs:
+  repo:
+    description: 'Source repository to use to build the EVM binary'
+    required: true
+    default: 'ethereum/evmone'
+  ref:
+    description: 'Reference to branch, commit, or tag to use to build the EVM binary'
+    required: true
+    default: 'master'
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout evmone
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repo }}
+        ref: ${{ inputs.ref }}
+        path: evmone
+        submodules: true
+    - name: Setup cmake
+      uses: jwlawson/actions-setup-cmake@v2
+    - name: Build evmone binary
+      shell: bash
+      run: |
+        mkdir -p $GITHUB_WORKSPACE/bin
+        cd $GITHUB_WORKSPACE/evmone
+        cmake -S . -B build -DEVMONE_TESTING=ON
+        cmake --build build --parallel
+        echo $GITHUB_WORKSPACE/evmone/build/bin/ >> $GITHUB_PATH

.github/actions/build-fixtures/action.yaml

@@ -0,0 +1,54 @@
+name: Build and Package Fixtures
+inputs:
+  name:
+    description: 'Name of the fixture package'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - name: Install yq
+      shell: bash
+      run: |
+        pip install yq
+    - name: Extract fixture properties
+      id: properties
+      shell: bash
+      run: |
+        yq -r --arg feature "${{ inputs.name }}" '.[$feature] | to_entries | map("\(.key)=\(.value)")[]' ./configs/feature.yaml >> "$GITHUB_OUTPUT"
+    - uses: ./.github/actions/build-evm
+      id: evm-builder
+      with:
+        type: ${{ steps.properties.outputs.evm-type }}
+    - name: Install solc compiler
+      shell: bash
+      run: |
+        if [ "$RUNNER_OS" == "Linux" ]; then PLATFORM="linux-amd64"; else PLATFORM="macosx-amd64"; fi
+        RELEASE_NAME=$(curl https://binaries.soliditylang.org/${PLATFORM}/list.json | jq -r --arg SOLC_VERSION "${{ steps.properties.outputs.solc }}" '.releases[$SOLC_VERSION]')
+        wget -O $GITHUB_WORKSPACE/bin/solc https://binaries.soliditylang.org/${PLATFORM}/$RELEASE_NAME
+        chmod a+x $GITHUB_WORKSPACE/bin/solc
+        echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH
+    - name: Run fixtures fill
+      shell: bash
+      run: |
+        pip install --upgrade pip
+        python -m venv env
+        source env/bin/activate
+        pip install -e .
+        fill -n auto --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }}
+    - name: Create fixtures info file
+      shell: bash
+      run: |
+        echo -e "ref: $GITHUB_REF \ncommit: $GITHUB_SHA\nbuild: $(date +"%Y-%m-%dT%H:%M:%SZ")" \
+        > fixtures/info.txt
+    - name: Tar fixtures output
+      shell: bash
+      run: |
+        tar -czvf fixtures_${{ inputs.name }}.tar.gz ./fixtures
+    - uses: actions/upload-artifact@v4
+      with:
+        name: fixtures_${{ inputs.name }}
+        path: fixtures_${{ inputs.name }}.tar.gz

.github/workflows/fixtures.yaml

@@ -5,66 +5,34 @@ on:
     branches:
       - main
     tags:
-      - 'v*'
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
   workflow_dispatch:
 
 jobs:
+  features:
+    runs-on: ubuntu-latest
+    outputs:
+      features: ${{ steps.parse.outputs.features }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get names from configs/feature.yaml
+        id: parse
+        shell: bash
+        run: |
+          echo "features=$(grep -Po "^[0-9a-zA-Z_\-]+" ./configs/feature.yaml | jq -R . | jq -cs .)" >> "$GITHUB_OUTPUT"
   build:
+    needs: features
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - name: 'fixtures'
-            evm-type: 'main'
-            fill-params: ''
-            solc: '0.8.21'
-            python: '3.11'
-          - name: 'fixtures_develop'
-            evm-type: 'develop'
-            fill-params: '--until=Prague --ignore=./tests/prague/eip7692_eof_v1'
-            solc: '0.8.21'
-            python: '3.11'
+        name: ${{ fromJson(needs.features.outputs.features) }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: ./.github/actions/build-evm
-        id: evm-builder
-        with:
-          type: ${{ matrix.evm-type }}
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-      - name: Install solc compiler
-        shell: bash
-        run: |
-          if [ "$RUNNER_OS" == "Linux" ]; then PLATFORM="linux-amd64"; else PLATFORM="macosx-amd64"; fi
-          RELEASE_NAME=$(curl https://binaries.soliditylang.org/${PLATFORM}/list.json | jq -r --arg SOLC_VERSION "${{ matrix.solc }}" '.releases[$SOLC_VERSION]')
-          wget -O $GITHUB_WORKSPACE/bin/solc https://binaries.soliditylang.org/${PLATFORM}/$RELEASE_NAME
-          chmod a+x $GITHUB_WORKSPACE/bin/solc
-          echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH
-      - name: Run fixtures fill
-        shell: bash
-        run: |
-          pip install --upgrade pip
-          python -m venv env
-          source env/bin/activate
-          pip install -e .
-          fill ${{ matrix.fill-params }}
-      - name: Create fixtures info file
-        shell: bash
-        run: |
-          echo -e "ref: $GITHUB_REF \ncommit: $GITHUB_SHA\nbuild: $(date +"%Y-%m-%dT%H:%M:%SZ")" \
-          > fixtures/info.txt
-      - name: Tar fixtures output
-        shell: bash
-        run: |
-          tar -czvf ${{ matrix.name }}.tar.gz ./fixtures
-      - uses: actions/upload-artifact@v4
+      - uses: ./.github/actions/build-fixtures
         with:
           name: ${{ matrix.name }}
-          path: ${{ matrix.name }}.tar.gz
   release:
     runs-on: ubuntu-latest
     needs: build

.github/workflows/fixtures_feature.yaml

@@ -0,0 +1,42 @@
+name: Build and Package Fixtures for a feature
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*@v*'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Get feature name
+        id: feature-name
+        shell: bash
+        run: |
+          echo name=${GITHUB_REF_NAME//@*/} >> "$GITHUB_OUTPUT"
+      - uses: ./.github/actions/build-fixtures
+        with:
+          name: ${{ steps.feature-name.outputs.name }}
+  release:
+    runs-on: ubuntu-latest
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .
+      - name: Draft Pre-release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: './**'
+          draft: true
+          prerelease: true
+          generate_release_notes: true
+          fail_on_unmatched_files: true

.github/workflows/test.yaml

@@ -11,22 +11,28 @@ jobs:
           - os: ubuntu-latest
             python: '3.10'
             solc: '0.8.20'
-            evm-type: 'main'
+            evm-type: 'stable'
             tox-cmd: 'tox run-parallel --parallel-no-spinner'
           - os: ubuntu-latest
             python: '3.12'
             solc: '0.8.23'
-            evm-type: 'main'
+            evm-type: 'stable'
             tox-cmd: 'tox run-parallel --parallel-no-spinner'
           - os: ubuntu-latest
             python: '3.11'
             solc: '0.8.21'
             evm-type: 'develop'
             tox-cmd: 'tox -e tests-develop'
+          # Disabled to not be gated by evmone implementation
+          # - os: ubuntu-latest
+          #   python: '3.11'
+          #   solc: '0.8.21'
+          #   evm-type: 'eip7692'
+          #   tox-cmd: 'tox -e tests-eip7692'
           - os: macos-latest
             python: '3.11'
             solc: '0.8.22'
-            evm-type: 'main'
+            evm-type: 'stable'
             tox-cmd: 'tox run-parallel --parallel-no-spinner'
     steps:
       - uses: actions/checkout@v4

README.md

@@ -72,7 +72,7 @@ Generally, specific `t8n` implementations and branches must be used when develop
 
 We use named reference tags to point to the specific version of the `t8n` implementation that needs to be used fill the tests.
 
-All current tags, their t8n implementation and branch they point to, are listed in [evm-config.yaml](evm-config.yaml).
+All current tags, their t8n implementation and branch they point to, are listed in [configs/evm.yaml](configs/evm.yaml).
 
 ## Getting Started
 

configs/evm.yaml

@@ -0,0 +1,15 @@
+stable:
+  impl: geth
+  repo: ethereum/go-ethereum
+  ref: master
+  evm-bin: evm
+develop:
+  impl: geth
+  repo: lightclient/go-ethereum
+  ref: prague-devnet-0
+  evm-bin: evm
+eip7692:
+  impl: evmone
+  repo: ethereum/evmone
+  ref: master
+  evm-bin: evmone-t8n

configs/feature.yaml

@@ -0,0 +1,12 @@
+stable:
+  evm-type: stable
+  fill-params: ''
+  solc: 0.8.21
+develop:
+  evm-type: develop
+  fill-params: --until=Prague
+  solc: 0.8.21
+eip7692:
+  evm-type: eip7692
+  fill-params: --fork=CancunEIP7692 ./tests/prague
+  solc: 0.8.21

docs/CHANGELOG.md

@@ -47,10 +47,12 @@ Test fixtures for use by clients are available for each release on the [Github r
   - Fixtures' hashes (in the `_info` field) are now calculated by removing the "_info" field entirely instead of it being set to an empty dict.
 - 🐞 Relax minor and patch dependency requirements to avoid conflicting package dependencies ([#510](https://github.com/ethereum/execution-spec-tests/pull/510)).
 - 🔀 Update all CI actions to use their respective Node.js 20 versions, ahead of their Node.js 16 version deprecations ([#527](https://github.com/ethereum/execution-spec-tests/pull/527)).
+- ✨ Releases now contain a `fixtures_eip7692.tar.gz` which contains all EOF fixtures ([#573](https://github.com/ethereum/execution-spec-tests/pull/573)).
 
 ### 💥 Breaking Change
 
 - Cancun is now the latest deployed fork, and the development fork is now Prague ([#489](https://github.com/ethereum/execution-spec-tests/pull/489)).
+- Stable fixtures artifact `fixtures.tar.gz` has been renamed to `fixtures_stable.tar.gz` ([#573](https://github.com/ethereum/execution-spec-tests/pull/573))
 
 ## 🔜 [v2.1.1](https://github.com/ethereum/execution-spec-tests/releases/tag/v2.1.1) - 2024-03-09
 

docs/gen_test_case_reference.py

@@ -186,21 +186,24 @@ def run_collect_only(test_path: Path = source_directory) -> Tuple[str, str]:
         str: The command used to collect the tests.
         str: A list of the collected tests.
     """
-    buffer = io.StringIO()
-    with contextlib.redirect_stdout(buffer):
-        pytest.main(["--collect-only", "-q", "--until", DEV_FORKS[-1], str(test_path)])
-    output = buffer.getvalue()
-    collect_only_command = f"fill --collect-only -q --until {DEV_FORKS[-1]} {test_path}"
-    # strip out the test module
-    output_lines = [
-        line.split("::")[1]
-        for line in output.split("\n")
-        if line.startswith("tests/") and "::" in line
-    ]
-    # prefix with required indent for admonition in MARKDOWN_TEST_CASES_TEMPLATE
-    collect_only_output = "\n".join("    " + line for line in output_lines)
-    collect_only_output = collect_only_output[4:]  # strip out indent for first line
-    return collect_only_command, collect_only_output
+    for fork in DEV_FORKS:
+        collect_only_args = ["--collect-only", "-q", "--until", fork, str(test_path)]
+        buffer = io.StringIO()
+        with contextlib.redirect_stdout(buffer):
+            pytest.main(collect_only_args)
+        output = buffer.getvalue()
+        # strip out the test module
+        output_lines = [
+            line.split("::")[1]
+            for line in output.split("\n")
+            if line.startswith("tests/") and "::" in line
+        ]
+        # prefix with required indent for admonition in MARKDOWN_TEST_CASES_TEMPLATE
+        collect_only_output = "\n".join("    " + line for line in output_lines)
+        collect_only_output = collect_only_output[4:]  # strip out indent for first line
+        if collect_only_output:
+            break
+    return f'fill {" ".join(collect_only_args)}', collect_only_output
 
 
 def generate_github_url(file_path, branch_or_commit_or_tag="main"):