From cda345e642128734149c0c09d8e6cee920ca2546 Mon Sep 17 00:00:00 2001 From: cjihrig Date: Wed, 2 Mar 2016 12:12:47 -0500 Subject: [PATCH] tools: support testing known issues This commit adds a known_issues directory to the test directory for scripts that reproduce known bugs. Since these scripts are expected to fail, it also adds a --expect-fail flag to test.py which reports tests as successful when they fail. Refs: https://github.com/nodejs/testing/issues/18 Backport-URL: https://github.com/nodejs/node/pull/5785 PR-URL: https://github.com/nodejs/node/pull/5528 Reviewed-By: James M Snell Reviewed-By: Wyatt Preul Reviewed-By: Rich Trott Conflicts: tools/test.py --- Makefile | 3 +++ test/known_issues/testcfg.py | 6 ++++++ tools/test.py | 9 +++++++-- vcbuild.bat | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 test/known_issues/testcfg.py diff --git a/Makefile b/Makefile index e526ad5242d519..5ad0000f7d6d39 100644 --- a/Makefile +++ b/Makefile @@ -168,6 +168,9 @@ test-internet: all test-debugger: all $(PYTHON) tools/test.py debugger +test-known-issues: all + $(PYTHON) tools/test.py known_issues --expect-fail + test-npm: $(NODE_EXE) NODE=$(NODE) tools/test-npm.sh diff --git a/test/known_issues/testcfg.py b/test/known_issues/testcfg.py new file mode 100644 index 00000000000000..af4f56574b365b --- /dev/null +++ b/test/known_issues/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.SimpleTestConfiguration(context, root, 'known_issues') diff --git a/tools/test.py b/tools/test.py index 217e72b870a6f8..d5caff92e1232c 100755 --- a/tools/test.py +++ b/tools/test.py @@ -430,7 +430,7 @@ def __init__(self, context, path, arch, mode): self.thread_id = 0 def IsNegative(self): - return False + return self.context.expect_fail def CompareTime(self, other): return cmp(other.duration, self.duration) @@ -774,11 +774,13 @@ def GetTestStatus(self, context, sections, defs): class Context(object): - def __init__(self, workspace, buildspace, verbose, vm, timeout, processor, suppress_dialogs, store_unexpected_output): + def __init__(self, workspace, buildspace, verbose, vm, expect_fail, + timeout, processor, suppress_dialogs, store_unexpected_output): self.workspace = workspace self.buildspace = buildspace self.verbose = verbose self.vm_root = vm + self.expect_fail = expect_fail self.timeout = timeout self.processor = processor self.suppress_dialogs = suppress_dialogs @@ -1281,6 +1283,8 @@ def BuildOptions(): result.add_option("--snapshot", help="Run the tests with snapshot turned on", default=False, action="store_true") result.add_option("--special-command", default=None) + result.add_option("--expect-fail", dest="expect_fail", + help="Expect test cases to fail", default=False, action="store_true") result.add_option("--valgrind", help="Run tests through valgrind", default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", @@ -1471,6 +1475,7 @@ def Main(): buildspace, VERBOSE, shell, + options.expect_fail, options.timeout, processor, options.suppress_dialogs, diff --git a/vcbuild.bat b/vcbuild.bat index e955860b8a464e..cd3f758a3933a1 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -61,6 +61,7 @@ if /i "%1"=="test-gc" set test_args=%test_args% gc&set buildnodeweak=1&got if /i "%1"=="test-internet" set test_args=%test_args% internet&goto arg-ok if /i "%1"=="test-pummel" set test_args=%test_args% pummel&goto arg-ok if /i "%1"=="test-all" set test_args=%test_args% sequential parallel message gc internet pummel&set buildnodeweak=1&set jslint=1&goto arg-ok +if /i "%1"=="test-known-issues" set test_args=%test_args% known_issues --expect-fail&goto arg-ok if /i "%1"=="jslint" set jslint=1&goto arg-ok if /i "%1"=="msi" set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok if /i "%1"=="build-release" set build_release=1&goto arg-ok