diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index a50138f85..06a0605a5 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -99,6 +99,7 @@ Judson Neer Julian Berman Julien Voisin Justas Sadzevičius +Kassandra Keeton Kjell Braden Krystian Kichewko Kyle Altendorf @@ -125,19 +126,21 @@ Michał Bultrowicz Michał Górny Mickie Betz Mike Fiedler -Naveen Yadav Nathan Land +Naveen Yadav +Neil Pilgrim Nikita Bloshchanevich Nils Kattenbeck Noel O'Boyle Olivier Grisel Ori Avtalion -Pankaj Pandey Pablo Carballo +Pankaj Pandey Patrick Mezard Peter Baughman Peter Ebden Peter Portante +Phebe Polk Reya B Rodrigue Cloutier Roger Hu @@ -168,8 +171,8 @@ Thijs Triemstra Thomas Grainger Titus Brown Valentin Lab -Vince Salvino Ville Skyttä +Vince Salvino Xie Yanbo Yilei "Dolee" Yang Yury Selivanov diff --git a/Makefile b/Makefile index bcca29ea7..1adda37b8 100644 --- a/Makefile +++ b/Makefile @@ -11,17 +11,18 @@ clean_platform: @rm -f *.so */*.so + @rm -f *.pyd */*.pyd @rm -rf __pycache__ */__pycache__ */*/__pycache__ */*/*/__pycache__ */*/*/*/__pycache__ */*/*/*/*/__pycache__ @rm -f *.pyc */*.pyc */*/*.pyc */*/*/*.pyc */*/*/*/*.pyc */*/*/*/*/*.pyc @rm -f *.pyo */*.pyo */*/*.pyo */*/*/*.pyo */*/*/*/*.pyo */*/*/*/*/*.pyo + @rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class */*/*/*/*$$py.class */*/*/*/*/*$$py.class clean: clean_platform ## Remove artifacts of test execution, installation, etc. @echo "Cleaning..." @-pip uninstall -yq coverage - @rm -f *.pyd */*.pyd + @chmod -R 777 build @rm -rf build coverage.egg-info dist htmlcov @rm -f *.bak */*.bak */*/*.bak */*/*/*.bak */*/*/*/*.bak */*/*/*/*/*.bak - @rm -f *$$py.class */*$$py.class */*/*$$py.class */*/*/*$$py.class */*/*/*/*$$py.class */*/*/*/*/*$$py.class @rm -f coverage/*,cover @rm -f MANIFEST @rm -f .coverage .coverage.* coverage.xml coverage.json .metacov* diff --git a/README.rst b/README.rst index 25239393e..8e3ffd940 100644 --- a/README.rst +++ b/README.rst @@ -70,7 +70,8 @@ For Enterprise Getting Started --------------- -See the `Quick Start section`_ of the docs. +Looking to run ``coverage`` on your test suite? See the `Quick Start section`_ +of the docs. .. _Quick Start section: https://coverage.readthedocs.io/#quick-start @@ -96,7 +97,8 @@ Community Code of Conduct`_. Contributing ------------ -See the `Contributing section`_ of the docs. +Found a bug? Want to help improve the code or documentation? See the +`Contributing section`_ of the docs. .. _Contributing section: https://coverage.readthedocs.io/en/latest/contributing.html diff --git a/doc/contributing.rst b/doc/contributing.rst index e9d2c3a40..7cba0859e 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -37,28 +37,26 @@ The coverage.py code is hosted on a GitHub repository at https://github.com/nedbat/coveragepy. To get a working environment, follow these steps: -.. minimum of PYVERSIONS: +#. `Fork the repo`__ into your own GitHub account. The coverage.py code +will then be copied into a GitHub repository at +https://github.com/GITHUB_USER/coveragepy where GITHUB_USER is your GitHub +username. + +__ https://docs.github.com/en/get-started/quickstart/fork-a-repo -#. Create a Python 3.7 virtualenv to work in, and activate it. +#. (Optional) Create a virtualenv to work in, and activate it. There +are a number of ways to do this. Use the method you are comfortable with. #. Clone the repository:: - $ git clone https://github.com/nedbat/coveragepy + $ git clone https://github.com/GITHUB_USER/coveragepy $ cd coveragepy #. Install the requirements:: - $ python3 -m pip install -r requirements/dev.pip - - If this fails due to incorrect or missing hashes, use - ``dev.in`` instead:: - $ python3 -m pip install -r requirements/dev.in -#. Install a number of versions of Python. Coverage.py supports a range - of Python versions. The more you can test with, the more easily your code - can be used as-is. If you only have one version, that's OK too, but may - mean more work integrating your contribution. + Note: You may need to upgrade pip to install the requirements. Running the tests @@ -67,57 +65,91 @@ Running the tests The tests are written mostly as standard unittest-style tests, and are run with pytest running under `tox`_:: - $ tox - py37 create: /Users/nedbat/coverage/trunk/.tox/py37 - py37 installdeps: -rrequirements/pip.pip, -rrequirements/pytest.pip, eventlet==0.25.1, greenlet==0.4.15 - py37 develop-inst: /Users/nedbat/coverage/trunk - py37 installed: apipkg==1.5,appdirs==1.4.4,attrs==20.3.0,backports.functools-lru-cache==1.6.4,-e git+git@github.com:nedbat/coveragepy.git@36ef0e03c0439159c2245d38de70734fa08cddb4#egg=coverage,decorator==5.0.7,distlib==0.3.1,dnspython==2.1.0,eventlet==0.25.1,execnet==1.8.0,filelock==3.0.12,flaky==3.7.0,future==0.18.2,greenlet==0.4.15,hypothesis==6.10.1,importlib-metadata==4.0.1,iniconfig==1.1.1,monotonic==1.6,packaging==20.9,pluggy==0.13.1,py==1.10.0,PyContracts @ git+https://github.com/slorg1/contracts@c5a6da27d4dc9985f68e574d20d86000880919c3,pyparsing==2.4.7,pytest==6.2.3,pytest-forked==1.3.0,pytest-xdist==2.2.1,qualname==0.1.0,six==1.15.0,sortedcontainers==2.3.0,toml==0.10.2,typing-extensions==3.10.0.0,virtualenv==20.4.4,zipp==3.4.1 - py37 run-test-pre: PYTHONHASHSEED='376882681' - py37 run-test: commands[0] | python setup.py --quiet clean develop - py37 run-test: commands[1] | python igor.py zip_mods remove_extension - py37 run-test: commands[2] | python igor.py test_with_tracer py - === CPython 3.7.10 with Python tracer (.tox/py37/bin/python) === + % python3 -m tox + ROOT: tox-gh won't override envlist because tox is not running in GitHub Actions + .pkg: _optional_hooks> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + .pkg: get_requires_for_build_editable> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + .pkg: build_editable> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + py37: install_package> python -m pip install -U --force-reinstall --no-deps .tox/.tmp/package/87/coverage-7.2.3a0.dev1-0.editable-cp37-cp37m-macosx_10_15_x86_64.whl + py37: commands[0]> python igor.py zip_mods + py37: commands[1]> python setup.py --quiet build_ext --inplace + py37: commands[2]> python -m pip install -q -e . + py37: commands[3]> python igor.py test_with_tracer c + === CPython 3.7.15 with C tracer (.tox/py37/bin/python) === bringing up nodes... - ........................................................................................................................................................... [ 15%] - ........................................................................................................................................................... [ 31%] - ...........................................................................................................................................s............... [ 47%] - ...........................................s...................................................................................sss.sssssssssssssssssss..... [ 63%] - ........................................................................................................................................................s.. [ 79%] - ......................................s..................................s................................................................................. [ 95%] - ........................................ss...... [100%] - 949 passed, 29 skipped in 40.56s - py37 run-test: commands[3] | python setup.py --quiet build_ext --inplace - py37 run-test: commands[4] | python igor.py test_with_tracer c - === CPython 3.7.10 with C tracer (.tox/py37/bin/python) === + .........................................................................................................................x.................s....s....... [ 11%] + ..s.....x.............................................s................................................................................................. [ 22%] + ........................................................................................................................................................ [ 34%] + ........................................................................................................................................................ [ 45%] + ........................................................................................................................................................ [ 57%] + .........s....................................................................................................................s......................... [ 68%] + .................................s..............................s...............s..................................s.................................... [ 80%] + ........................................................s............................................................................................... [ 91%] + ......................................s......................................................................... [100%] + 1316 passed, 12 skipped, 2 xfailed in 36.42s + py37: commands[4]> python igor.py remove_extension + py37: commands[5]> python igor.py test_with_tracer py + === CPython 3.7.15 with Python tracer (.tox/py37/bin/python) === bringing up nodes... - ........................................................................................................................................................... [ 15%] - ........................................................................................................................................................... [ 31%] - ......................................................................s.................................................................................... [ 47%] - ........................................................................................................................................................... [ 63%] - ..........................s................................................s............................................................................... [ 79%] - .................................................................................s......................................................................... [ 95%] - ......................................s......... [100%] - 973 passed, 5 skipped in 41.36s - ____________________________________________________________________________ summary _____________________________________________________________________________ - py37: commands succeeded - congratulations :) + ................................................................................................x...........................x.................s......... [ 11%] + .....s.............s.s.....................................................s..............ss............................s.ss....ss.ss................... [ 22%] + ......................................................................................................................................s................. [ 34%] + ..................................................................................................................s..................................... [ 45%] + ...................s.ss.....................................................................................s....................s.ss................... [ 57%] + ..................s.s................................................................................................................................... [ 68%] + ..........................s.........................................ssss...............s.................s...sss..................s...ss...ssss.s....... [ 80%] + .......................................................................................................................................................s [ 91%] + .........................................................................s.................................ss.... [100%] + 1281 passed, 47 skipped, 2 xfailed in 33.86s + .pkg: _exit> python /usr/local/virtualenvs/coverage/lib/python3.7/site-packages/pyproject_api/_backend.py True setuptools.build_meta + py37: OK (82.38=setup[2.80]+cmd[0.20,0.35,7.30,37.20,0.21,34.32] seconds) + congratulations :) (83.61 seconds) Tox runs the complete test suite twice for each version of Python you have -installed. The first run uses the Python implementation of the trace function, -the second uses the C implementation. +installed. The first run uses the C implementation of the trace function, +the second uses the Python implementation. To limit tox to just a few versions of Python, use the ``-e`` switch:: - $ tox -e py37,py39 - -To run just a few tests, you can use `pytest test selectors`_:: - - $ tox tests/test_misc.py - $ tox tests/test_misc.py::HasherTest - $ tox tests/test_misc.py::HasherTest::test_string_hashing - -These command run the tests in one file, one class, and just one test, -respectively. + $ python3 -m tox -e py37,py39 + +On the tox command line, options after ``--`` are passed to pytest. To run +just a few tests, you can use `pytest test selectors`_:: + + $ python3 -m tox -- tests/test_misc.py + $ python3 -m tox -- tests/test_misc.py::HasherTest + $ python3 -m tox -- tests/test_misc.py::HasherTest::test_string_hashing + +These commands run the tests in one file, one class, and just one test, +respectively. The pytest ``-k`` option selects tests based on a word in their +name, which can be very convenient for ad-hoc test selection. Of course you +can combine tox and pytest options:: + + $ python3 -m tox -q -e py37 -- -n 0 -vv -k hash + === CPython 3.7.15 with C tracer (.tox/py37/bin/python) === + ======================================= test session starts ======================================== + platform darwin -- Python 3.7.15, pytest-7.2.2, pluggy-1.0.0 -- /Users/nedbat/coverage/.tox/py37/bin/python + cachedir: .tox/py37/.pytest_cache + rootdir: /Users/nedbat/coverage, configfile: setup.cfg + plugins: flaky-3.7.0, hypothesis-6.70.0, xdist-3.2.1 + collected 1330 items / 1320 deselected / 10 selected + run-last-failure: no previously failed tests, not deselecting items. + + tests/test_data.py::CoverageDataTest::test_add_to_hash_with_lines PASSED [ 10%] + tests/test_data.py::CoverageDataTest::test_add_to_hash_with_arcs PASSED [ 20%] + tests/test_data.py::CoverageDataTest::test_add_to_lines_hash_with_missing_file PASSED [ 30%] + tests/test_data.py::CoverageDataTest::test_add_to_arcs_hash_with_missing_file PASSED [ 40%] + tests/test_execfile.py::RunPycFileTest::test_running_hashed_pyc PASSED [ 50%] + tests/test_misc.py::HasherTest::test_string_hashing PASSED [ 60%] + tests/test_misc.py::HasherTest::test_bytes_hashing PASSED [ 70%] + tests/test_misc.py::HasherTest::test_unicode_hashing PASSED [ 80%] + tests/test_misc.py::HasherTest::test_dict_hashing PASSED [ 90%] + tests/test_misc.py::HasherTest::test_dict_collision PASSED [100%] + + =============================== 10 passed, 1320 deselected in 1.88s ================================ + Skipping tests with Python tracer: Only one tracer: no Python tracer for CPython + py37: OK (12.22=setup[2.19]+cmd[0.20,0.36,6.57,2.51,0.20,0.19] seconds) + congratulations :) (13.10 seconds) You can also affect the test runs with environment variables. Define any of these as 1 to use them: @@ -156,7 +188,8 @@ some warnings. Please try to keep it that way, but don't let pylint warnings keep you from sending patches. I can clean them up. Lines should be kept to a 100-character maximum length. I recommend an -`editorconfig.org`_ plugin for your editor of choice. +`editorconfig.org`_ plugin for your editor of choice, which will also help with +indentation, line endings and so on. Other style questions are best answered by looking at the existing code. Formatting of docstrings, comments, long lines, and so on, should match the diff --git a/pyproject.toml b/pyproject.toml index e11a5af1d..6b02c6a47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,32 @@ exclude = """(?x)( | ^tests/balance_xdist_plugin\\.py$ # not part of our test suite. )""" +[tool.pytest.ini_options] +addopts = "-q -n auto -p no:legacypath --strict-markers --no-flaky-report -rfEX --failed-first" +python_classes = "*Test" +markers = [ + "expensive: too slow to run during \"make smoke\"", +] + +# How come these warnings are suppressed successfully here, but not in conftest.py?? +filterwarnings = [ + "ignore:the imp module is deprecated in favour of importlib:DeprecationWarning", + "ignore:distutils Version classes are deprecated:DeprecationWarning", + "ignore:The distutils package is deprecated and slated for removal in Python 3.12:DeprecationWarning", +] + +# xfail tests that pass should fail the test suite +xfail_strict = true + +balanced_clumps = [ + # Because of expensive session-scoped fixture: + "VirtualenvTest", + # Because of shared-file manipulations (~/tests/actual/testing): + "CompareTest", + # No idea why this one fails if run on separate workers: + "GetZipBytesTest", +] + [tool.scriv] # Changelog management: https://pypi.org/project/scriv/ format = "rst" diff --git a/setup.cfg b/setup.cfg index adbdfb11a..db3fdce8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,28 +1,5 @@ # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt -[tool:pytest] -addopts = -q -n auto -p no:legacypath --strict-markers --no-flaky-report -rfEX --failed-first -python_classes = *Test -markers = - expensive: too slow to run during "make smoke" - -# How come these warnings are suppressed successfully here, but not in conftest.py?? -filterwarnings = - ignore:the imp module is deprecated in favour of importlib:DeprecationWarning - ignore:distutils Version classes are deprecated:DeprecationWarning - ignore:The distutils package is deprecated and slated for removal in Python 3.12:DeprecationWarning - -# xfail tests that pass should fail the test suite -xfail_strict = true - -balanced_clumps = - ; Because of expensive session-scoped fixture: - VirtualenvTest - ; Because of shared-file manipulations (~/tests/actual/testing): - CompareTest - ; No idea why this one fails if run on separate workers: - GetZipBytesTest - [metadata] license_files = LICENSE.txt diff --git a/tests/gold/README.rst b/tests/gold/README.rst index aec1d6370..ec4081825 100644 --- a/tests/gold/README.rst +++ b/tests/gold/README.rst @@ -9,16 +9,26 @@ these comparisons is in tests/goldtest.py. If gold tests are failing, you may need to update the gold files by copying the current output of the tests into the gold files. When a test fails, the actual -output is in the tests/actual directory. Do not commit those files to git. +output is in the tests/actual directory. Those files are ignored by git. -You can run just the failed tests again with:: +There's a Makefile in the html directory for working with gold files and their +associated support files. If actual outputs have changed so that comparisons +are failing, but the new output is correct, you can use "make update-gold" to +copy the actual output as the new gold files. + +If you have added a gold test, you'll need to manually copy the tests/actual +files to tests/gold. + +Once you've copied the actual results to the gold files, or to check your work +again, you can run just the failed tests again with:: tox -e py39 -- -n 0 --lf The saved HTML files in the html directories can't be viewed properly without the supporting CSS and Javascript files. But we don't want to save copies of -those files in every subdirectory. There's a Makefile in the html directory -for working with the saved copies of the support files. +those files in every subdirectory. The make target "make complete" in +tests/gold/html will copy the support file so you can open the HTML files to +see how they look. If the output files are correct, you can update the gold files with "make update-gold". If there are version-specific gold files (for example, diff --git a/tests/gold/html/Makefile b/tests/gold/html/Makefile index 7be71f841..1b75f73d5 100644 --- a/tests/gold/html/Makefile +++ b/tests/gold/html/Makefile @@ -17,7 +17,7 @@ complete: ## Copy support files into directories so the HTML can be viewed prop clean: ## Remove the effects of this Makefile. @git clean -fq . -update-gold: ## Copy output files from latest tests to gold files. +update-gold: ## Copy actual output files from latest tests to gold files. @for sub in ../../actual/html/*; do \ rsync --verbose --existing --recursive $$sub/ $$(basename $$sub) ; \ done ; \ diff --git a/tests/gold/html/contexts/index.html b/tests/gold/html/contexts/index.html new file mode 100644 index 000000000..05d6a457f --- /dev/null +++ b/tests/gold/html/contexts/index.html @@ -0,0 +1,102 @@ + + + + + Coverage report + + + + + +
+
+

Coverage report: + 94% +

+ +
+ +
+

+ coverage.py v7.2.3a0.dev1, + created at 2023-03-21 08:44 -0400 +

+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Modulestatementsmissingexcludedcoverage
two_tests.py171094%
Total171094%
+

+ No items found using the specified filter. +

+
+ + + diff --git a/tests/gold/html/contexts/two_tests_py.html b/tests/gold/html/contexts/two_tests_py.html new file mode 100644 index 000000000..5e107b5eb --- /dev/null +++ b/tests/gold/html/contexts/two_tests_py.html @@ -0,0 +1,119 @@ + + + + + Coverage for two_tests.py: 94% + + + + + +
+
+

+ Coverage for two_tests.py: + 94% +

+ +

+ 17 statements   + + + +

+

+ « prev     + ^ index     + » next +       + coverage.py v7.2.3a0.dev1, + created at 2023-03-21 08:44 -0400 +

+ +
+
+
+

1def helper(lineno): 

+

2 x = 2 (empty)two_tests.test_onetwo_tests.test_two

+

3 

+

4def test_one(): 

+

5 a = 5 two_tests.test_one

+

6 helper(6) two_tests.test_one

+

7 

+

8def test_two(): 

+

9 a = 9 two_tests.test_two

+

10 b = 10 two_tests.test_two

+

11 if a > 11: two_tests.test_two

+

12 b = 12 

+

13 assert a == (13-4) two_tests.test_two

+

14 assert b == (14-4) two_tests.test_two

+

15 helper( two_tests.test_two

+

16 16 

+

17 ) 

+

18 

+

19test_one() 

+

20x = 20 

+

21helper(21) 

+

22test_two() 

+
+ + + diff --git a/tests/goldtest.py b/tests/goldtest.py index 12a04af66..f7bd2e9b6 100644 --- a/tests/goldtest.py +++ b/tests/goldtest.py @@ -46,6 +46,8 @@ def compare( """ __tracebackhide__ = True # pytest, please don't show me this function. assert os_sep("/gold/") in expected_dir + assert os.path.exists(actual_dir) + os.makedirs(expected_dir, exist_ok=True) dc = filecmp.dircmp(expected_dir, actual_dir) diff_files = _fnmatch_list(dc.diff_files, file_pattern) @@ -56,9 +58,11 @@ def save_mismatch(f: str) -> None: """Save a mismatched result to tests/actual.""" save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/")) os.makedirs(save_path, exist_ok=True) - with open(os.path.join(save_path, f), "w") as savef: + save_file = os.path.join(save_path, f) + with open(save_file, "w") as savef: with open(os.path.join(actual_dir, f)) as readf: savef.write(readf.read()) + print(os_sep(f"Saved actual output to '{save_file}': see tests/gold/README.rst")) # filecmp only compares in binary mode, but we want text mode. So # look through the list of different files, and compare them diff --git a/tests/test_goldtest.py b/tests/test_goldtest.py index def5ee90e..297f21b0e 100644 --- a/tests/test_goldtest.py +++ b/tests/test_goldtest.py @@ -13,7 +13,7 @@ from tests.coveragetest import CoverageTest, TESTS_DIR from tests.goldtest import compare, gold_path from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain -from tests.helpers import re_line, remove_tree +from tests.helpers import os_sep, re_line, remove_tree GOOD_GETTY = """\ Four score and seven years ago our fathers brought forth upon this continent, a @@ -73,6 +73,10 @@ def test_bad(self) -> None: assert "+ Five score" in stdout assert re_line(rf"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) assert re_line(rf"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) + assert ( + os_sep(f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst") + in os_sep(stdout) + ) assert " D/D/D, Gxxx, Pennsylvania" in stdout # The actual file was saved. diff --git a/tests/test_html.py b/tests/test_html.py index 5113cd06e..89ad8d1c2 100644 --- a/tests/test_html.py +++ b/tests/test_html.py @@ -1199,6 +1199,9 @@ def test_dynamic_contexts(self) -> None: ] assert sorted(expected) == sorted(actual) + cov.html_report(mod, directory="out/contexts") + compare_html(gold_path("html/contexts"), "out/contexts") + def test_filtered_dynamic_contexts(self) -> None: self.make_file("two_tests.py", self.SOURCE) cov = coverage.Coverage(source=["."])