evm tests / fixed heuristic

.github/workflows/main.yaml

@@ -83,7 +83,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job: [evm-byzantium, evm-petersburg, evm-istanbul]
+        job: [evm-byzantium, evm-petersburg, evm-istanbul, evm-latest]
 
     steps:
     - uses: actions/checkout@v1

brownie/project/compiler/solidity.py

@@ -314,6 +314,7 @@ def _generate_coverage_data(
     active_source_node: Optional[NodeBase] = None
     active_fn_node: Optional[NodeBase] = None
     active_fn_name: Optional[str] = None
+    first_source = source_map[0]
 
     while source_map:
         # format of source_map is [start, stop, contract_id, jump code]
@@ -340,12 +341,13 @@ def _generate_coverage_data(
             pc += int(pc_list[-1]["op"][4:])
 
         # for REVERT opcodes without an source offset, try to infer one
-        if source[2] == -1:
+        if source[2] == -1 or source == first_source:
             if pc_list[-1]["op"] == "REVERT":
                 _find_revert_offset(
                     pc_list, source_map, active_source_node, active_fn_node, active_fn_name
                 )
-            continue
+            if source[2] == -1:
+                continue
 
         # set contract path (-1 means none)
         contract_id = str(source[2])
@@ -448,8 +450,8 @@ def _find_revert_offset(
 
     # attempt to infer a source offset for reverts that do not have one
 
-    if source_map and source_map[0][2] == -1:
-        # this is not the last instruction and the following instruction also has no source
+    if source_map:
+        # is not the last instruction
         if len(pc_list) >= 8 and pc_list[-8]["op"] == "CALLVALUE":
             # reference to CALLVALUE 8 instructions previous is a nonpayable function check
             pc_list[-1].update(
@@ -458,7 +460,7 @@ def _find_revert_offset(
                 offset=pc_list[-8]["offset"],
                 path=pc_list[-8]["path"],
             )
-        return
+            return
 
     # if there is active function, we are still in the function selector table
     if not fn_node:

tests/conftest.py

@@ -10,6 +10,7 @@
 from typing import Dict, List
 
 import pytest
+import solcx
 from _pytest.monkeypatch import MonkeyPatch
 from ethpm._utils.ipfs import dummy_ipfs_pin
 from ethpm.backends.ipfs import BaseIPFSBackend
@@ -23,9 +24,6 @@
 
 TARGET_OPTS = {
     "evm": "evmtester",
-    "evm-byzantium": "evmtester",
-    "evm-petersburg": "evmtester",
-    "evm-istanbul": "evmtester",
     "pm": "package_test",
     "plugin": "plugintester",
 }
@@ -34,19 +32,27 @@
 def pytest_addoption(parser):
     parser.addoption(
         "--target",
-        choices=["all", "core"] + list(TARGET_OPTS),
+        choices=["core", "pm", "plugin"],
         default="core",
-        help="Target a specific component of the tests. Use 'all' to run the full suite.",
+        help="Target a specific component of the tests.",
+    )
+    parser.addoption(
+        "--evm",
+        nargs=3,
+        metavar=("solc_versions", "evm_rulesets", "optimizer_runs"),
+        help="Run evm tests against a matrix of solc versions, evm versions, and compiler runs.",
     )
 
 
 # remove tests based on config flags and fixture names
 def pytest_collection_modifyitems(config, items):
-    target = config.getoption("--target").split("-")[0]
-    if target == "all":
-        return
+    if config.getoption("--evm"):
+        target = "evm"
+    else:
+        target = config.getoption("--target")
+
     for flag, fixture in TARGET_OPTS.items():
-        if "-" in flag or target == flag:
+        if target == flag:
             continue
         for test in [i for i in items if fixture in i.fixturenames]:
             items.remove(test)
@@ -57,31 +63,25 @@ def pytest_collection_modifyitems(config, items):
 
 
 def pytest_configure(config):
-    if config.getoption("--target") in ("all", "plugin") and config.getoption("numprocesses"):
+    if config.getoption("--target") == "plugin" and config.getoption("numprocesses"):
         raise pytest.UsageError("Cannot use xdist with plugin tests, try adding the '-n 0' flag")
 
+    if config.getoption("--evm"):
+        # reformat evm options - only do this once to avoid repeat queries for latest solc version
+        solc_versions, evm_verions, runs = [i.split(",") for i in config.option.evm]
+        runs = [int(i) for i in runs]
+        if "latest" in solc_versions:
+            latest_version = solcx.get_available_solc_versions()[0]
+            solc_versions.remove("latest")
+            solc_versions.append(latest_version)
+        config.option.evm = (evm_verions, runs, solc_versions)
+
 
 def pytest_generate_tests(metafunc):
     # parametrize the evmtester fixture
-    target = metafunc.config.getoption("--target")
-    if "evmtester" in metafunc.fixturenames and (target == "all" or target.startswith("evm")):
-
-        if target.startswith("evm-"):
-            target = target[4:]
-        runs = [0, 10000]
-
-        versions = [target]
-        if target in ("all", "evm"):
-            versions = ["byzantium", "petersburg", "istanbul"]
-        params = list(itertools.product(versions, runs, ["0.6.3", "0.5.15"]))
-
-        if target != "istanbul":
-            if target in ("all", "evm"):
-                versions = ["byzantium", "constantinople", "istanbul"]
-            elif target == "petersburg":
-                versions = ["constantinople"]
-            params += list(itertools.product(versions, runs, ["0.5.0", "0.4.25", "0.4.22"]))
-
+    evm_config = metafunc.config.getoption("--evm")
+    if "evmtester" in metafunc.fixturenames and evm_config:
+        params = list(itertools.product(*evm_config))
         metafunc.parametrize("evmtester", params, indirect=True)
 
 

tox.ini

@@ -4,27 +4,28 @@ envlist =
     docs-{local,external}
     py{36,37,38}
     {pm,evm,plugin}test
-    evm-{byzantium,petersburg,istanbul}
+    evm-{byzantium,petersburg,istanbul,latest}
 
 [testenv]
 passenv =
     ETHERSCAN_TOKEN
     GITHUB_TOKEN
     WEB3_INFURA_PROJECT_ID
 deps =
-    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul}: coverage==5.1
-    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul}: pytest==5.4.3
-    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul}: pytest-cov==2.9.0
-    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul}: pytest-mock==3.1.1
-    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul}: pytest-xdist==1.32.0
+    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul,latest}: coverage==5.1
+    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul,latest}: pytest==5.4.3
+    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul,latest}: pytest-cov==2.9.0
+    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul,latest}: pytest-mock==3.1.1
+    py{36,37,38},{pm,plugin}test,evm-{byzantium,petersburg,istanbul,latest}: pytest-xdist==1.32.0
     docs-{local,external}: sphinx
     docs-{local,external}: sphinx_rtd_theme
     docs-{local,external}: pygments_lexer_solidity
 commands =
     py{36,37,38}: python -m pytest tests/
-    evm-byzantium: python -m pytest tests/ --target evm-byzantium
-    evm-petersburg: python -m pytest tests/ --target evm-petersburg
-    evm-istanbul: python -m pytest tests/ --target evm-istanbul
+    evm-byzantium: python -m pytest tests/ --evm 0.4.22,0.4.26,0.5.0,0.5.17,0.6.3,0.6.9 byzantium 0,10000
+    evm-petersburg: python -m pytest tests/ --evm 0.5.5,0.5.17,0.6.3,0.6.9 petersburg 0,10000
+    evm-istanbul: python -m pytest tests/ --evm 0.5.13,0.5.17,0.6.3,0.6.9 istanbul 0,10000
+    evm-latest: python -m pytest tests/ --evm latest byzantium,petersburg,istanbul 0,200,10000
     pmtest: python -m pytest tests/ --target pm -n 0
     plugintest: python -m pytest tests/ --target plugin -n 0
     docs-local: sphinx-build {posargs:-E} -b html docs dist/docs -n -q --color