From ede7478dcc69ef0d1526346813ca32fb34ce0343 Mon Sep 17 00:00:00 2001 From: Omar Kohl Date: Mon, 20 Jun 2016 15:05:50 +0200 Subject: [PATCH] Exit pytest on collection error (without executing tests) Add --continue-on-collection-errors option to restore the previous behaviour: Execute tests (that were successfully collected) even when collection errors happen. Some tests had to be modified e.g. because the return code changed to 2 (EXIT_INTERRUPTED) instead of 1 (EXIT_TESTSFAILED) because an Interrupted exception is raised on collection error. Implemented via pair programming with: Oleg Pidsadnyi closes #1421 --- AUTHORS | 1 + CHANGELOG.rst | 7 +++ _pytest/main.py | 8 +++ testing/acceptance_test.py | 10 ++-- testing/test_collection.py | 111 +++++++++++++++++++++++++++++++++++++ testing/test_doctest.py | 21 +++++-- testing/test_resultlog.py | 2 +- testing/test_terminal.py | 2 +- 8 files changed, 152 insertions(+), 10 deletions(-) diff --git a/AUTHORS b/AUTHORS index 0c40517cea8..d72ddc9f4ea 100644 --- a/AUTHORS +++ b/AUTHORS @@ -72,6 +72,7 @@ Michael Birtwell Michael Droettboom Mike Lundy Nicolas Delaby +Oleg Pidsadnyi Omar Kohl Pieter Mulder Piotr Banaszkiewicz diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26f78cec18e..0c87f9fe7c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -74,6 +74,10 @@ message to raise when no exception occurred. Thanks `@palaviv`_ for the complete PR (`#1616`_). +* Fix `#1421`_: Exit tests if a collection error occurs and add + ``--continue-on-collection-errors`` option to restore previous behaviour. + Thanks `@olegpidsadnyi`_ and `@omarkohl`_ for the complete PR (`#1628`_). + .. _@milliams: https://github.com/milliams .. _@csaftoiu: https://github.com/csaftoiu .. _@novas0x2a: https://github.com/novas0x2a @@ -83,7 +87,9 @@ .. _@palaviv: https://github.com/palaviv .. _@omarkohl: https://github.com/omarkohl .. _@mikofski: https://github.com/mikofski +.. _@olegpidsadnyi: https://github.com/olegpidsadnyi +.. _#1421: https://github.com/pytest-dev/pytest/issues/1421 .. _#1426: https://github.com/pytest-dev/pytest/issues/1426 .. _#1428: https://github.com/pytest-dev/pytest/pull/1428 .. _#1444: https://github.com/pytest-dev/pytest/pull/1444 @@ -98,6 +104,7 @@ .. _#372: https://github.com/pytest-dev/pytest/issues/372 .. _#1544: https://github.com/pytest-dev/pytest/issues/1544 .. _#1616: https://github.com/pytest-dev/pytest/pull/1616 +.. _#1628: https://github.com/pytest-dev/pytest/pull/1628 **Bug Fixes** diff --git a/_pytest/main.py b/_pytest/main.py index 47b6cb694c9..ed57c4f27c0 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -48,6 +48,9 @@ def pytest_addoption(parser): help="run pytest in strict mode, warnings become errors.") group._addoption("-c", metavar="file", type=str, dest="inifilename", help="load configuration from `file` instead of trying to locate one of the implicit configuration files.") + group._addoption("--continue-on-collection-errors", action="store_true", + default=False, dest="continue_on_collection_errors", + help="Force test execution even if collection errors occur.") group = parser.getgroup("collect", "collection") group.addoption('--collectonly', '--collect-only', action="store_true", @@ -133,6 +136,11 @@ def pytest_collection(session): return session.perform_collect() def pytest_runtestloop(session): + if (session.testsfailed and + not session.config.option.continue_on_collection_errors): + raise session.Interrupted( + "%d errors during collection" % session.testsfailed) + if session.config.option.collectonly: return True diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 0c9d5888565..4813ebdac93 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -120,7 +120,7 @@ def test_this(): "ImportError while importing test module*", "'No module named *does_not_work*", ]) - assert result.ret == 1 + assert result.ret == 2 def test_not_collectable_arguments(self, testdir): p1 = testdir.makepyfile("") @@ -665,11 +665,13 @@ def test_with_failing_collection(self, testdir): testdir.makepyfile(self.source) testdir.makepyfile(test_collecterror="""xyz""") result = testdir.runpytest("--durations=2", "-k test_1") - assert result.ret != 0 + assert result.ret == 2 result.stdout.fnmatch_lines([ - "*durations*", - "*call*test_1*", + "*Interrupted: 1 errors during collection*", ]) + # Collection errors abort test execution, therefore no duration is + # output + assert "duration" not in result.stdout.str() def test_with_not(self, testdir): testdir.makepyfile(self.source) diff --git a/testing/test_collection.py b/testing/test_collection.py index 9f46c71fba1..f42ec8f2578 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -642,3 +642,114 @@ def test___repr__(): """) reprec = testdir.inline_run("-k repr") reprec.assertoutcome(passed=1, failed=0) + + +COLLECTION_ERROR_PY_FILES = dict( + test_01_failure=""" + def test_1(): + assert False + """, + test_02_import_error=""" + import asdfasdfasdf + def test_2(): + assert True + """, + test_03_import_error=""" + import asdfasdfasdf + def test_3(): + assert True + """, + test_04_success=""" + def test_4(): + assert True + """, +) + +def test_exit_on_collection_error(testdir): + """Verify that all collection errors are collected and no tests executed""" + testdir.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = testdir.runpytest() + assert res.ret == 2 + + res.stdout.fnmatch_lines([ + "collected 2 items / 2 errors", + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*ERROR collecting test_03_import_error.py*", + "*No module named *asdfa*", + ]) + + +def test_exit_on_collection_with_maxfail_smaller_than_n_errors(testdir): + """ + Verify collection is aborted once maxfail errors are encountered ignoring + further modules which would cause more collection errors. + """ + testdir.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = testdir.runpytest("--maxfail=1") + assert res.ret == 2 + + res.stdout.fnmatch_lines([ + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*Interrupted: stopping after 1 failures*", + ]) + + assert 'test_03' not in res.stdout.str() + + +def test_exit_on_collection_with_maxfail_bigger_than_n_errors(testdir): + """ + Verify the test run aborts due to collection errors even if maxfail count of + errors was not reached. + """ + testdir.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = testdir.runpytest("--maxfail=4") + assert res.ret == 2 + + res.stdout.fnmatch_lines([ + "collected 2 items / 2 errors", + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*ERROR collecting test_03_import_error.py*", + "*No module named *asdfa*", + ]) + + +def test_continue_on_collection_errors(testdir): + """ + Verify tests are executed even when collection errors occur when the + --continue-on-collection-errors flag is set + """ + testdir.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = testdir.runpytest("--continue-on-collection-errors") + assert res.ret == 1 + + res.stdout.fnmatch_lines([ + "collected 2 items / 2 errors", + "*1 failed, 1 passed, 2 error*", + ]) + + +def test_continue_on_collection_errors_maxfail(testdir): + """ + Verify tests are executed even when collection errors occur and that maxfail + is honoured (including the collection error count). + 4 tests: 2 collection errors + 1 failure + 1 success + test_4 is never executed because the test run is with --maxfail=3 which + means it is interrupted after the 2 collection errors + 1 failure. + """ + testdir.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = testdir.runpytest("--continue-on-collection-errors", "--maxfail=3") + assert res.ret == 2 + + res.stdout.fnmatch_lines([ + "collected 2 items / 2 errors", + "*Interrupted: stopping after 3 failures*", + "*1 failed, 2 error*", + ]) diff --git a/testing/test_doctest.py b/testing/test_doctest.py index d104d98d343..4bda1bf1f11 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -199,8 +199,20 @@ def test(self): "*1 failed*", ]) + def test_doctest_unex_importerror_only_txt(self, testdir): + testdir.maketxtfile(""" + >>> import asdalsdkjaslkdjasd + >>> + """) + result = testdir.runpytest() + # doctest is never executed because of error during hello.py collection + result.stdout.fnmatch_lines([ + "*>>> import asdals*", + "*UNEXPECTED*ImportError*", + "ImportError: No module named *asdal*", + ]) - def test_doctest_unex_importerror(self, testdir): + def test_doctest_unex_importerror_with_module(self, testdir): testdir.tmpdir.join("hello.py").write(_pytest._code.Source(""" import asdalsdkjaslkdjasd """)) @@ -209,10 +221,11 @@ def test_doctest_unex_importerror(self, testdir): >>> """) result = testdir.runpytest("--doctest-modules") + # doctest is never executed because of error during hello.py collection result.stdout.fnmatch_lines([ - "*>>> import hello", - "*UNEXPECTED*ImportError*", - "*import asdals*", + "*ERROR collecting hello.py*", + "*ImportError: No module named *asdals*", + "*Interrupted: 1 errors during collection*", ]) def test_doctestmodule(self, testdir): diff --git a/testing/test_resultlog.py b/testing/test_resultlog.py index 74d13f64330..373d1921381 100644 --- a/testing/test_resultlog.py +++ b/testing/test_resultlog.py @@ -231,6 +231,6 @@ def test_func(): pass """) result = testdir.runpytest("--resultlog=log") - assert result.ret == 1 + assert result.ret == 2 diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 2173fa6fcb2..d5cc10aee45 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -273,7 +273,7 @@ def test_method(self): def test_collectonly_error(self, testdir): p = testdir.makepyfile("import Errlkjqweqwe") result = testdir.runpytest("--collect-only", p) - assert result.ret == 1 + assert result.ret == 2 result.stdout.fnmatch_lines(_pytest._code.Source(""" *ERROR* *ImportError*