From 4024ef48612f3c2d20f320ee19b8724d19704bc0 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 20 Mar 2023 07:00:18 -0400 Subject: [PATCH 01/11] docs: more details, and current output, in contributing.rst --- doc/contributing.rst | 121 ++++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 43 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index e9d2c3a40..410d0d4cc 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -67,57 +67,91 @@ Running the tests The tests are written mostly as standard unittest-style tests, and are run with pytest running under `tox`_:: - $ tox - py37 create: /Users/nedbat/coverage/trunk/.tox/py37 - py37 installdeps: -rrequirements/pip.pip, -rrequirements/pytest.pip, eventlet==0.25.1, greenlet==0.4.15 - py37 develop-inst: /Users/nedbat/coverage/trunk - py37 installed: apipkg==1.5,appdirs==1.4.4,attrs==20.3.0,backports.functools-lru-cache==1.6.4,-e git+git@github.com:nedbat/coveragepy.git@36ef0e03c0439159c2245d38de70734fa08cddb4#egg=coverage,decorator==5.0.7,distlib==0.3.1,dnspython==2.1.0,eventlet==0.25.1,execnet==1.8.0,filelock==3.0.12,flaky==3.7.0,future==0.18.2,greenlet==0.4.15,hypothesis==6.10.1,importlib-metadata==4.0.1,iniconfig==1.1.1,monotonic==1.6,packaging==20.9,pluggy==0.13.1,py==1.10.0,PyContracts @ git+https://github.com/slorg1/contracts@c5a6da27d4dc9985f68e574d20d86000880919c3,pyparsing==2.4.7,pytest==6.2.3,pytest-forked==1.3.0,pytest-xdist==2.2.1,qualname==0.1.0,six==1.15.0,sortedcontainers==2.3.0,toml==0.10.2,typing-extensions==3.10.0.0,virtualenv==20.4.4,zipp==3.4.1 - py37 run-test-pre: PYTHONHASHSEED='376882681' - py37 run-test: commands[0] | python setup.py --quiet clean develop - py37 run-test: commands[1] | python igor.py zip_mods remove_extension - py37 run-test: commands[2] | python igor.py test_with_tracer py - === CPython 3.7.10 with Python tracer (.tox/py37/bin/python) === + % tox + ROOT: tox-gh won't override envlist because tox is not running in GitHub Actions + .pkg: _optional_hooks> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + .pkg: get_requires_for_build_editable> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + .pkg: build_editable> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + py37: install_package> python -m pip install -U --force-reinstall --no-deps .tox/.tmp/package/87/coverage-7.2.3a0.dev1-0.editable-cp37-cp37m-macosx_10_15_x86_64.whl + py37: commands[0]> python igor.py zip_mods + py37: commands[1]> python setup.py --quiet build_ext --inplace + py37: commands[2]> python -m pip install -q -e . + py37: commands[3]> python igor.py test_with_tracer c + === CPython 3.7.15 with C tracer (.tox/py37/bin/python) === bringing up nodes... - ........................................................................................................................................................... [ 15%] - ........................................................................................................................................................... [ 31%] - ...........................................................................................................................................s............... [ 47%] - ...........................................s...................................................................................sss.sssssssssssssssssss..... [ 63%] - ........................................................................................................................................................s.. [ 79%] - ......................................s..................................s................................................................................. [ 95%] - ........................................ss...... [100%] - 949 passed, 29 skipped in 40.56s - py37 run-test: commands[3] | python setup.py --quiet build_ext --inplace - py37 run-test: commands[4] | python igor.py test_with_tracer c - === CPython 3.7.10 with C tracer (.tox/py37/bin/python) === + .........................................................................................................................x.................s....s....... [ 11%] + ..s.....x.............................................s................................................................................................. [ 22%] + ........................................................................................................................................................ [ 34%] + ........................................................................................................................................................ [ 45%] + ........................................................................................................................................................ [ 57%] + .........s....................................................................................................................s......................... [ 68%] + .................................s..............................s...............s..................................s.................................... [ 80%] + ........................................................s............................................................................................... [ 91%] + ......................................s......................................................................... [100%] + 1316 passed, 12 skipped, 2 xfailed in 36.42s + py37: commands[4]> python igor.py remove_extension + py37: commands[5]> python igor.py test_with_tracer py + === CPython 3.7.15 with Python tracer (.tox/py37/bin/python) === bringing up nodes... - ........................................................................................................................................................... [ 15%] - ........................................................................................................................................................... [ 31%] - ......................................................................s.................................................................................... [ 47%] - ........................................................................................................................................................... [ 63%] - ..........................s................................................s............................................................................... [ 79%] - .................................................................................s......................................................................... [ 95%] - ......................................s......... [100%] - 973 passed, 5 skipped in 41.36s - ____________________________________________________________________________ summary _____________________________________________________________________________ - py37: commands succeeded - congratulations :) + ................................................................................................x...........................x.................s......... [ 11%] + .....s.............s.s.....................................................s..............ss............................s.ss....ss.ss................... [ 22%] + ......................................................................................................................................s................. [ 34%] + ..................................................................................................................s..................................... [ 45%] + ...................s.ss.....................................................................................s....................s.ss................... [ 57%] + ..................s.s................................................................................................................................... [ 68%] + ..........................s.........................................ssss...............s.................s...sss..................s...ss...ssss.s....... [ 80%] + .......................................................................................................................................................s [ 91%] + .........................................................................s.................................ss.... [100%] + 1281 passed, 47 skipped, 2 xfailed in 33.86s + .pkg: _exit> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + py37: OK (82.38=setup[2.80]+cmd[0.20,0.35,7.30,37.20,0.21,34.32] seconds) + congratulations :) (83.61 seconds) Tox runs the complete test suite twice for each version of Python you have -installed. The first run uses the Python implementation of the trace function, -the second uses the C implementation. +installed. The first run uses the C implementation of the trace function, +the second uses the Python implementation. To limit tox to just a few versions of Python, use the ``-e`` switch:: $ tox -e py37,py39 -To run just a few tests, you can use `pytest test selectors`_:: - - $ tox tests/test_misc.py - $ tox tests/test_misc.py::HasherTest - $ tox tests/test_misc.py::HasherTest::test_string_hashing - -These command run the tests in one file, one class, and just one test, -respectively. +On the tox command line, options after ``--`` are passed to pytest. To run +just a few tests, you can use `pytest test selectors`_:: + + $ tox -- tests/test_misc.py + $ tox -- tests/test_misc.py::HasherTest + $ tox -- tests/test_misc.py::HasherTest::test_string_hashing + +These commands run the tests in one file, one class, and just one test, +respectively. The pytest ``-k`` option selects tests based on a word in their +name, which can be very convenient for ad-hoc test selection. Of course you +can combine tox and pytest options:: + + $ tox -q -e py37 -- -n 0 -vv -k hash + === CPython 3.7.15 with C tracer (.tox/py37/bin/python) === + ======================================= test session starts ======================================== + platform darwin -- Python 3.7.15, pytest-7.2.2, pluggy-1.0.0 -- /Users/nedbat/coverage/.tox/py37/bin/python + cachedir: .tox/py37/.pytest_cache + rootdir: /Users/nedbat/coverage, configfile: setup.cfg + plugins: flaky-3.7.0, hypothesis-6.70.0, xdist-3.2.1 + collected 1330 items / 1320 deselected / 10 selected + run-last-failure: no previously failed tests, not deselecting items. + + tests/test_data.py::CoverageDataTest::test_add_to_hash_with_lines PASSED [ 10%] + tests/test_data.py::CoverageDataTest::test_add_to_hash_with_arcs PASSED [ 20%] + tests/test_data.py::CoverageDataTest::test_add_to_lines_hash_with_missing_file PASSED [ 30%] + tests/test_data.py::CoverageDataTest::test_add_to_arcs_hash_with_missing_file PASSED [ 40%] + tests/test_execfile.py::RunPycFileTest::test_running_hashed_pyc PASSED [ 50%] + tests/test_misc.py::HasherTest::test_string_hashing PASSED [ 60%] + tests/test_misc.py::HasherTest::test_bytes_hashing PASSED [ 70%] + tests/test_misc.py::HasherTest::test_unicode_hashing PASSED [ 80%] + tests/test_misc.py::HasherTest::test_dict_hashing PASSED [ 90%] + tests/test_misc.py::HasherTest::test_dict_collision PASSED [100%] + + =============================== 10 passed, 1320 deselected in 1.88s ================================ + Skipping tests with Python tracer: Only one tracer: no Python tracer for CPython + py37: OK (12.22=setup[2.19]+cmd[0.20,0.36,6.57,2.51,0.20,0.19] seconds) + congratulations :) (13.10 seconds) You can also affect the test runs with environment variables. Define any of these as 1 to use them: @@ -156,7 +190,8 @@ some warnings. Please try to keep it that way, but don't let pylint warnings keep you from sending patches. I can clean them up. Lines should be kept to a 100-character maximum length. I recommend an -`editorconfig.org`_ plugin for your editor of choice. +`editorconfig.org`_ plugin for your editor of choice, which will also help with +indentation, line endings and so on. Other style questions are best answered by looking at the existing code. Formatting of docstrings, comments, long lines, and so on, should match the From 67b9e6c9b8422ddb20ebc6c637f058de5519df01 Mon Sep 17 00:00:00 2001 From: "neiljp (Neil Pilgrim)" Date: Mon, 20 Mar 2023 10:08:32 -0700 Subject: [PATCH 02/11] build: Migrate pytest configuration to pyproject.toml. --- pyproject.toml | 26 ++++++++++++++++++++++++++ setup.cfg | 23 ----------------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e11a5af1d..6b02c6a47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,32 @@ exclude = """(?x)( | ^tests/balance_xdist_plugin\\.py$ # not part of our test suite. )""" +[tool.pytest.ini_options] +addopts = "-q -n auto -p no:legacypath --strict-markers --no-flaky-report -rfEX --failed-first" +python_classes = "*Test" +markers = [ + "expensive: too slow to run during \"make smoke\"", +] + +# How come these warnings are suppressed successfully here, but not in conftest.py?? +filterwarnings = [ + "ignore:the imp module is deprecated in favour of importlib:DeprecationWarning", + "ignore:distutils Version classes are deprecated:DeprecationWarning", + "ignore:The distutils package is deprecated and slated for removal in Python 3.12:DeprecationWarning", +] + +# xfail tests that pass should fail the test suite +xfail_strict = true + +balanced_clumps = [ + # Because of expensive session-scoped fixture: + "VirtualenvTest", + # Because of shared-file manipulations (~/tests/actual/testing): + "CompareTest", + # No idea why this one fails if run on separate workers: + "GetZipBytesTest", +] + [tool.scriv] # Changelog management: https://pypi.org/project/scriv/ format = "rst" diff --git a/setup.cfg b/setup.cfg index adbdfb11a..db3fdce8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,28 +1,5 @@ # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt -[tool:pytest] -addopts = -q -n auto -p no:legacypath --strict-markers --no-flaky-report -rfEX --failed-first -python_classes = *Test -markers = - expensive: too slow to run during "make smoke" - -# How come these warnings are suppressed successfully here, but not in conftest.py?? -filterwarnings = - ignore:the imp module is deprecated in favour of importlib:DeprecationWarning - ignore:distutils Version classes are deprecated:DeprecationWarning - ignore:The distutils package is deprecated and slated for removal in Python 3.12:DeprecationWarning - -# xfail tests that pass should fail the test suite -xfail_strict = true - -balanced_clumps = - ; Because of expensive session-scoped fixture: - VirtualenvTest - ; Because of shared-file manipulations (~/tests/actual/testing): - CompareTest - ; No idea why this one fails if run on separate workers: - GetZipBytesTest - [metadata] license_files = LICENSE.txt From ea6db5e1aa9efd31b2569eb2f35597733358a388 Mon Sep 17 00:00:00 2001 From: Kassandra Keeton Date: Mon, 20 Mar 2023 17:01:18 -0500 Subject: [PATCH 03/11] docs: additional clarity for getting started + contributing (#1590) * additional clarity for getting started + contributing * Apply suggestions from code review Updating formatting (limitation of 79 characters for linting),. Co-authored-by: Ned Batchelder --------- Co-authored-by: Ned Batchelder --- README.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 25239393e..8e3ffd940 100644 --- a/README.rst +++ b/README.rst @@ -70,7 +70,8 @@ For Enterprise Getting Started --------------- -See the `Quick Start section`_ of the docs. +Looking to run ``coverage`` on your test suite? See the `Quick Start section`_ +of the docs. .. _Quick Start section: https://coverage.readthedocs.io/#quick-start @@ -96,7 +97,8 @@ Community Code of Conduct`_. Contributing ------------ -See the `Contributing section`_ of the docs. +Found a bug? Want to help improve the code or documentation? See the +`Contributing section`_ of the docs. .. _Contributing section: https://coverage.readthedocs.io/en/latest/contributing.html From 14beae2e44155b4d833fed3c4e16084924db7fd5 Mon Sep 17 00:00:00 2001 From: polkapolka <7219835+polkapolka@users.noreply.github.com> Date: Mon, 20 Mar 2023 16:09:22 -0700 Subject: [PATCH 04/11] docs: update contribution instructions (#1591) * Updating contribution to include fork instructions * Fixing urls * Fix urls --- doc/contributing.rst | 45 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 410d0d4cc..c7b755364 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -33,17 +33,27 @@ you frustration. Getting the code ---------------- -The coverage.py code is hosted on a GitHub repository at -https://github.com/nedbat/coveragepy. To get a working environment, follow -these steps: +Fork the repo into your own github repository. If you have not forked a +repository before then please follow the `fork instructions`_ provided by +github. + +.. _fork instructions: https://docs.github.com/en/get-started/quickstart/fork-a-repo + +The coverage.py code will then be copied into a GitHub repository at +https://github.com/GITHUB_USER/coveragepy where GITHUB_USER is your github +username. To get a working development environment, follow these steps: .. minimum of PYVERSIONS: -#. Create a Python 3.7 virtualenv to work in, and activate it. +#. Create a virtualenv using one of the common virtualenv formats (conda, +venv, pipenv, etc) and a version of python greater than 3.7 to work in, and +activate it. Please follow your virtualenv format specific creation and +activation instructions. + #. Clone the repository:: - $ git clone https://github.com/nedbat/coveragepy + $ git clone https://github.com/GITHUB_USER/coveragepy $ cd coveragepy #. Install the requirements:: @@ -54,20 +64,23 @@ these steps: ``dev.in`` instead:: $ python3 -m pip install -r requirements/dev.in + + Note: You may need to upgrade pip to install the requirements. -#. Install a number of versions of Python. Coverage.py supports a range - of Python versions. The more you can test with, the more easily your code - can be used as-is. If you only have one version, that's OK too, but may - mean more work integrating your contribution. +#. (Optional) Install more versions of Python. By default, Coverage.py won't + run the tests on a version of python that isn't installed. Coverage.py + supports a range of Python versions. The more you can test with, the more + easily your code can be used as-is. If you only have one version, that's + OK too, but may mean more work integrating your contribution. Running the tests ----------------- The tests are written mostly as standard unittest-style tests, and are run with -pytest running under `tox`_:: +pytest running under `tox`_ by using the command `python3 -m tox`:: - % tox + % python3 -m tox ROOT: tox-gh won't override envlist because tox is not running in GitHub Actions .pkg: _optional_hooks> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta .pkg: get_requires_for_build_editable> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta @@ -113,21 +126,21 @@ the second uses the Python implementation. To limit tox to just a few versions of Python, use the ``-e`` switch:: - $ tox -e py37,py39 + $ python3 -m tox -e py37,py39 On the tox command line, options after ``--`` are passed to pytest. To run just a few tests, you can use `pytest test selectors`_:: - $ tox -- tests/test_misc.py - $ tox -- tests/test_misc.py::HasherTest - $ tox -- tests/test_misc.py::HasherTest::test_string_hashing + $ python3 -m tox -- tests/test_misc.py + $ python3 -m tox -- tests/test_misc.py::HasherTest + $ python3 -m tox -- tests/test_misc.py::HasherTest::test_string_hashing These commands run the tests in one file, one class, and just one test, respectively. The pytest ``-k`` option selects tests based on a word in their name, which can be very convenient for ad-hoc test selection. Of course you can combine tox and pytest options:: - $ tox -q -e py37 -- -n 0 -vv -k hash + $ python3 -m tox -q -e py37 -- -n 0 -vv -k hash === CPython 3.7.15 with C tracer (.tox/py37/bin/python) === ======================================= test session starts ======================================== platform darwin -- Python 3.7.15, pytest-7.2.2, pluggy-1.0.0 -- /Users/nedbat/coverage/.tox/py37/bin/python From 1469085b50e887f3f0a592bfff55c525321619f9 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 20 Mar 2023 19:11:58 -0400 Subject: [PATCH 05/11] docs: further slimming down of contribution instructions --- doc/contributing.rst | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index c7b755364..7cba0859e 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -33,23 +33,19 @@ you frustration. Getting the code ---------------- -Fork the repo into your own github repository. If you have not forked a -repository before then please follow the `fork instructions`_ provided by -github. +The coverage.py code is hosted on a GitHub repository at +https://github.com/nedbat/coveragepy. To get a working environment, follow +these steps: -.. _fork instructions: https://docs.github.com/en/get-started/quickstart/fork-a-repo +#. `Fork the repo`__ into your own GitHub account. The coverage.py code +will then be copied into a GitHub repository at +https://github.com/GITHUB_USER/coveragepy where GITHUB_USER is your GitHub +username. -The coverage.py code will then be copied into a GitHub repository at -https://github.com/GITHUB_USER/coveragepy where GITHUB_USER is your github -username. To get a working development environment, follow these steps: - -.. minimum of PYVERSIONS: - -#. Create a virtualenv using one of the common virtualenv formats (conda, -venv, pipenv, etc) and a version of python greater than 3.7 to work in, and -activate it. Please follow your virtualenv format specific creation and -activation instructions. +__ https://docs.github.com/en/get-started/quickstart/fork-a-repo +#. (Optional) Create a virtualenv to work in, and activate it. There +are a number of ways to do this. Use the method you are comfortable with. #. Clone the repository:: @@ -58,27 +54,16 @@ activation instructions. #. Install the requirements:: - $ python3 -m pip install -r requirements/dev.pip - - If this fails due to incorrect or missing hashes, use - ``dev.in`` instead:: - $ python3 -m pip install -r requirements/dev.in - - Note: You may need to upgrade pip to install the requirements. -#. (Optional) Install more versions of Python. By default, Coverage.py won't - run the tests on a version of python that isn't installed. Coverage.py - supports a range of Python versions. The more you can test with, the more - easily your code can be used as-is. If you only have one version, that's - OK too, but may mean more work integrating your contribution. + Note: You may need to upgrade pip to install the requirements. Running the tests ----------------- The tests are written mostly as standard unittest-style tests, and are run with -pytest running under `tox`_ by using the command `python3 -m tox`:: +pytest running under `tox`_:: % python3 -m tox ROOT: tox-gh won't override envlist because tox is not running in GitHub Actions From e73c858a603f23da9076a40f89c923ed86af9da2 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 20 Mar 2023 19:15:17 -0400 Subject: [PATCH 06/11] docs: correct name alphabetization --- CONTRIBUTORS.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a50138f85..2c8a92e52 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -125,15 +125,15 @@ Michał Bultrowicz Michał Górny Mickie Betz Mike Fiedler -Naveen Yadav Nathan Land +Naveen Yadav Nikita Bloshchanevich Nils Kattenbeck Noel O'Boyle Olivier Grisel Ori Avtalion -Pankaj Pandey Pablo Carballo +Pankaj Pandey Patrick Mezard Peter Baughman Peter Ebden @@ -168,8 +168,8 @@ Thijs Triemstra Thomas Grainger Titus Brown Valentin Lab -Vince Salvino Ville Skyttä +Vince Salvino Xie Yanbo Yilei "Dolee" Yang Yury Selivanov From 5b39a88baf651628621510a60a3b8d6e4e183cab Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Mon, 20 Mar 2023 19:17:39 -0400 Subject: [PATCH 07/11] docs: three PRs from the PyCascades sprint --- CONTRIBUTORS.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 2c8a92e52..06a0605a5 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -99,6 +99,7 @@ Judson Neer Julian Berman Julien Voisin Justas Sadzevičius +Kassandra Keeton Kjell Braden Krystian Kichewko Kyle Altendorf @@ -127,6 +128,7 @@ Mickie Betz Mike Fiedler Nathan Land Naveen Yadav +Neil Pilgrim Nikita Bloshchanevich Nils Kattenbeck Noel O'Boyle @@ -138,6 +140,7 @@ Patrick Mezard Peter Baughman Peter Ebden Peter Portante +Phebe Polk Reya B Rodrigue Cloutier Roger Hu From 3ca4a7047a17a651e4eed9ca053735192634ece1 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 21 Mar 2023 06:51:51 -0400 Subject: [PATCH 08/11] build: make sure build can be deleted If the virtualenv tests fail, they somehow leave behind build directories with 000 permissions. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index bcca29ea7..565f0ab66 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ clean: clean_platform ## Remove artifacts of test execution, installation, etc @echo "Cleaning..." @-pip uninstall -yq coverage @rm -f *.pyd */*.pyd + @chmod -R 777 build @rm -rf build coverage.egg-info dist htmlcov @rm -f *.bak */*.bak */*/*.bak */*/*/*.bak */*/*/*/*.bak */*/*/*/*/*.bak @rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class */*/*/*/*$$py.class */*/*/*/*/*$$py.class From 220e6a724ed49fde399ab6c9ecbdf72367ba36a8 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 21 Mar 2023 07:00:47 -0400 Subject: [PATCH 09/11] build: tidy up the clean action clean_platform was added in 7379c811580cd27e8695763ed5eb15746b133275 so we could remove platform-specific pyc files to run Linux tests. Those were removed in 76e80108c11b042e6e929641fe3f38975f5765ba, but it couldn't hurt to keep things organized this way. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 565f0ab66..1adda37b8 100644 --- a/Makefile +++ b/Makefile @@ -11,18 +11,18 @@ clean_platform: @rm -f *.so */*.so + @rm -f *.pyd */*.pyd @rm -rf __pycache__ */__pycache__ */*/__pycache__ */*/*/__pycache__ */*/*/*/__pycache__ */*/*/*/*/__pycache__ @rm -f *.pyc */*.pyc */*/*.pyc */*/*/*.pyc */*/*/*/*.pyc */*/*/*/*/*.pyc @rm -f *.pyo */*.pyo */*/*.pyo */*/*/*.pyo */*/*/*/*.pyo */*/*/*/*/*.pyo + @rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class */*/*/*/*$$py.class */*/*/*/*/*$$py.class clean: clean_platform ## Remove artifacts of test execution, installation, etc. @echo "Cleaning..." @-pip uninstall -yq coverage - @rm -f *.pyd */*.pyd @chmod -R 777 build @rm -rf build coverage.egg-info dist htmlcov @rm -f *.bak */*.bak */*/*.bak */*/*/*.bak */*/*/*/*.bak */*/*/*/*/*.bak - @rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class */*/*/*/*$$py.class */*/*/*/*/*$$py.class @rm -f coverage/*,cover @rm -f MANIFEST @rm -f .coverage .coverage.* coverage.xml coverage.json .metacov* From 019f0996ec3dcfd6c8237b336ac8d81256e58239 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 21 Mar 2023 08:36:25 -0400 Subject: [PATCH 10/11] test: provide more guidance about working with gold files And make it easier to add new gold tests. --- tests/gold/README.rst | 18 ++++++++++++++---- tests/gold/html/Makefile | 2 +- tests/goldtest.py | 6 +++++- tests/test_goldtest.py | 6 +++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/gold/README.rst b/tests/gold/README.rst index aec1d6370..ec4081825 100644 --- a/tests/gold/README.rst +++ b/tests/gold/README.rst @@ -9,16 +9,26 @@ these comparisons is in tests/goldtest.py. If gold tests are failing, you may need to update the gold files by copying the current output of the tests into the gold files. When a test fails, the actual -output is in the tests/actual directory. Do not commit those files to git. +output is in the tests/actual directory. Those files are ignored by git. -You can run just the failed tests again with:: +There's a Makefile in the html directory for working with gold files and their +associated support files. If actual outputs have changed so that comparisons +are failing, but the new output is correct, you can use "make update-gold" to +copy the actual output as the new gold files. + +If you have added a gold test, you'll need to manually copy the tests/actual +files to tests/gold. + +Once you've copied the actual results to the gold files, or to check your work +again, you can run just the failed tests again with:: tox -e py39 -- -n 0 --lf The saved HTML files in the html directories can't be viewed properly without the supporting CSS and Javascript files. But we don't want to save copies of -those files in every subdirectory. There's a Makefile in the html directory -for working with the saved copies of the support files. +those files in every subdirectory. The make target "make complete" in +tests/gold/html will copy the support file so you can open the HTML files to +see how they look. If the output files are correct, you can update the gold files with "make update-gold". If there are version-specific gold files (for example, diff --git a/tests/gold/html/Makefile b/tests/gold/html/Makefile index 7be71f841..1b75f73d5 100644 --- a/tests/gold/html/Makefile +++ b/tests/gold/html/Makefile @@ -17,7 +17,7 @@ complete: ## Copy support files into directories so the HTML can be viewed prop clean: ## Remove the effects of this Makefile. @git clean -fq . -update-gold: ## Copy output files from latest tests to gold files. +update-gold: ## Copy actual output files from latest tests to gold files. @for sub in ../../actual/html/*; do \ rsync --verbose --existing --recursive $$sub/ $$(basename $$sub) ; \ done ; \ diff --git a/tests/goldtest.py b/tests/goldtest.py index 12a04af66..f7bd2e9b6 100644 --- a/tests/goldtest.py +++ b/tests/goldtest.py @@ -46,6 +46,8 @@ def compare( """ __tracebackhide__ = True # pytest, please don't show me this function. assert os_sep("/gold/") in expected_dir + assert os.path.exists(actual_dir) + os.makedirs(expected_dir, exist_ok=True) dc = filecmp.dircmp(expected_dir, actual_dir) diff_files = _fnmatch_list(dc.diff_files, file_pattern) @@ -56,9 +58,11 @@ def save_mismatch(f: str) -> None: """Save a mismatched result to tests/actual.""" save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/")) os.makedirs(save_path, exist_ok=True) - with open(os.path.join(save_path, f), "w") as savef: + save_file = os.path.join(save_path, f) + with open(save_file, "w") as savef: with open(os.path.join(actual_dir, f)) as readf: savef.write(readf.read()) + print(os_sep(f"Saved actual output to '{save_file}': see tests/gold/README.rst")) # filecmp only compares in binary mode, but we want text mode. So # look through the list of different files, and compare them diff --git a/tests/test_goldtest.py b/tests/test_goldtest.py index def5ee90e..297f21b0e 100644 --- a/tests/test_goldtest.py +++ b/tests/test_goldtest.py @@ -13,7 +13,7 @@ from tests.coveragetest import CoverageTest, TESTS_DIR from tests.goldtest import compare, gold_path from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain -from tests.helpers import re_line, remove_tree +from tests.helpers import os_sep, re_line, remove_tree GOOD_GETTY = """\ Four score and seven years ago our fathers brought forth upon this continent, a @@ -73,6 +73,10 @@ def test_bad(self) -> None: assert "+ Five score" in stdout assert re_line(rf"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) assert re_line(rf"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) + assert ( + os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst") + in os_sep(stdout) + ) assert " D/D/D, Gxxx, Pennsylvania" in stdout # The actual file was saved. From eaa3c8adf13d3cfa0e83a6230b445a6ac54109de Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 21 Mar 2023 08:51:02 -0400 Subject: [PATCH 11/11] test: check the HTML for context output --- tests/gold/html/contexts/index.html | 102 ++++++++++++++++++ tests/gold/html/contexts/two_tests_py.html | 119 +++++++++++++++++++++ tests/test_html.py | 3 + 3 files changed, 224 insertions(+) create mode 100644 tests/gold/html/contexts/index.html create mode 100644 tests/gold/html/contexts/two_tests_py.html diff --git a/tests/gold/html/contexts/index.html b/tests/gold/html/contexts/index.html new file mode 100644 index 000000000..05d6a457f --- /dev/null +++ b/tests/gold/html/contexts/index.html @@ -0,0 +1,102 @@ + + + + + Coverage report + + + + + +
+
+

Coverage report: + 94% +

+ +
+ +
+

+ coverage.py v7.2.3a0.dev1, + created at 2023-03-21 08:44 -0400 +

+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Modulestatementsmissingexcludedcoverage
two_tests.py171094%
Total171094%
+

+ No items found using the specified filter. +

+
+ + + diff --git a/tests/gold/html/contexts/two_tests_py.html b/tests/gold/html/contexts/two_tests_py.html new file mode 100644 index 000000000..5e107b5eb --- /dev/null +++ b/tests/gold/html/contexts/two_tests_py.html @@ -0,0 +1,119 @@ + + + + + Coverage for two_tests.py: 94% + + + + + +
+
+

+ Coverage for two_tests.py: + 94% +

+ +

+ 17 statements   + + + +

+

+ « prev     + ^ index     + » next +       + coverage.py v7.2.3a0.dev1, + created at 2023-03-21 08:44 -0400 +

+ +
+
+
+

1def helper(lineno): 

+

2 x = 2 (empty)two_tests.test_onetwo_tests.test_two

+

3 

+

4def test_one(): 

+

5 a = 5 two_tests.test_one

+

6 helper(6) two_tests.test_one

+

7 

+

8def test_two(): 

+

9 a = 9 two_tests.test_two

+

10 b = 10 two_tests.test_two

+

11 if a > 11: two_tests.test_two

+

12 b = 12 

+

13 assert a == (13-4) two_tests.test_two

+

14 assert b == (14-4) two_tests.test_two

+

15 helper( two_tests.test_two

+

16 16 

+

17 ) 

+

18 

+

19test_one() 

+

20x = 20 

+

21helper(21) 

+

22test_two() 

+
+ + + diff --git a/tests/test_html.py b/tests/test_html.py index 5113cd06e..89ad8d1c2 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1199,6 +1199,9 @@ def test_dynamic_contexts(self) -> None: ] assert sorted(expected) == sorted(actual) + cov.html_report(mod, directory="out/contexts") + compare_html(gold_path("html/contexts"), "out/contexts") + def test_filtered_dynamic_contexts(self) -> None: self.make_file("two_tests.py", self.SOURCE) cov = coverage.Coverage(source=["."])