Implement check-result: test check failure affects test result by default

docs/releases.rst

@@ -4,6 +4,14 @@
     Releases
 ======================
 
+tmt-1.38.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new ``result`` key has been added to the :ref:`test check</spec/tests/check>`
+specification. This key allows users to configure how check results are
+interpreted.
+
+
 tmt-1.37.0
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

spec/tests/check.fmf

@@ -16,6 +16,15 @@ description: |
 
     See :ref:`/plugins/test-checks` for the list of available checks.
 
+    The `result` key can be used to specify how the check result should be
+    interpreted. It accepts the following options:
+    - 'respect' (default): The check result affects the overall test result.
+    - 'ignore': The check result does not affect the overall test result.
+    - 'xfail': The check is expected to fail. If it passes, the test fails,
+      and if it fails, the test passes.
+    - 'pass', 'fail', 'info', 'warn', 'error': The check result is interpreted
+      as the specified outcome, regardless of the actual check result.
+
 example:
   - |
     # Enable a single check, AVC denial detection.
@@ -37,6 +46,26 @@ example:
       - how: test-inspector
         enable: false
 
+  - |
+    # Enable a check with a specific result interpretation
+    check:
+      - how: dmesg
+        result: xfail
+
+  - |
+    # Enable multiple checks with different result interpretations
+    check:
+      - how: dmesg
+        result: ignore
+      - how: avc
+        result: xfail
+
+  - |
+    # Enable a check with a specific outcome interpretation
+    check:
+      - how: dmesg
+        result: warn
+
 link:
   - implemented-by: /tmt/checks
   - verified-by: /tests/test/check/avc

tests/execute/result/check_results.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "run=\$(mktemp -d)" 0 "Create run directory"
+        rlRun "pushd check_results"
+        rlRun "set -o pipefail"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Check results are respected"
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-pass execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-pass" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-respect execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-fail-respect" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "check failed" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-fail-ignore execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-fail-ignore" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertNotGrep "check failed" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-pass execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-xfail-fail execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "expected failure" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-multiple execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "fail /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "check failed" $rlRun_LOG
+        rlAssertGrep "unexpected pass" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-override execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "pass /test/check-override" $rlRun_LOG
+        rlAssertGrep "pass dmesg (before-test check)" $rlRun_LOG
+        rlAssertGrep "fail dmesg (after-test check)" $rlRun_LOG
+        rlAssertNotGrep "check failed" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-warn execute -vv report -vv 2>&1 >/dev/null" 1
+        rlAssertGrep "warn /test/check-warn" $rlRun_LOG
+        rlAssertGrep "warn dmesg (after-test check)" $rlRun_LOG
+
+        rlRun -s "tmt run --id \${run} --scratch provision --how local test --name /test/check-info execute -vv report -vv 2>&1 >/dev/null" 0
+        rlAssertGrep "info /test/check-info" $rlRun_LOG
+        rlAssertGrep "info dmesg (after-test check)" $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartTest "Verify results.yaml content"
+        results_file="${run}/default/plan/execute/results.yaml"
+        rlAssertExists "$results_file"
+
+        rlRun -s "yq e '.[0]' $results_file"
+        rlAssertGrep "name: /test/check-pass" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[1]' $results_file"
+        rlAssertGrep "name: /test/check-fail-respect" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: check failed" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[2]' $results_file"
+        rlAssertGrep "name: /test/check-fail-ignore" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertNotGrep "note: check failed" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[3]' $results_file"
+        rlAssertGrep "name: /test/check-xfail-pass" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[4]' $results_file"
+        rlAssertGrep "name: /test/check-xfail-fail" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertGrep "note: expected failure" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[5]' $results_file"
+        rlAssertGrep "name: /test/check-multiple" $rlRun_LOG
+        rlAssertGrep "result: fail" $rlRun_LOG
+        rlAssertGrep "note: check failed, unexpected pass" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+
+        rlRun -s "yq e '.[6]' $results_file"
+        rlAssertGrep "name: /test/check-override" $rlRun_LOG
+        rlAssertGrep "result: pass" $rlRun_LOG
+        rlAssertNotGrep "note: check failed" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: pass" $rlRun_LOG
+        rlAssertGrep "  event: before-test" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: fail" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[7]' $results_file"
+        rlAssertGrep "name: /test/check-warn" $rlRun_LOG
+        rlAssertGrep "result: warn" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: warn" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+
+        rlRun -s "yq e '.[8]' $results_file"
+        rlAssertGrep "name: /test/check-info" $rlRun_LOG
+        rlAssertGrep "result: info" $rlRun_LOG
+        rlAssertGrep "check:" $rlRun_LOG
+        rlAssertGrep "- how: dmesg" $rlRun_LOG
+        rlAssertGrep "  result: info" $rlRun_LOG
+        rlAssertGrep "  event: after-test" $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlRun "rm -r ${run}" 0 "Remove run directory"
+    rlPhaseEnd
+rlJournalEnd

tests/execute/result/check_results/main.fmf

@@ -0,0 +1,83 @@
+summary: Tests for check results behaviour
+description: Verify that check results, including after-test checks, are correctly handled
+
+/test/check-pass:
+    summary: Test with passing checks
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          result: respect
+    # Expected outcome: PASS (test passes, check passes)
+
+/test/check-fail-respect:
+    summary: Test with failing dmesg check (respect)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: respect
+    # Expected outcome: FAIL (test passes, but check fails and is respected)
+
+/test/check-fail-ignore:
+    summary: Test with failing dmesg check (ignore)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: ignore
+    # Expected outcome: PASS (test passes, check fails but is ignored)
+
+/test/check-xfail-pass:
+    summary: Test with passing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          result: xfail
+    # Expected outcome: FAIL (test passes, check passes but xfail expects it to fail)
+
+/test/check-xfail-fail:
+    summary: Test with failing dmesg check (xfail)
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: xfail
+    # Expected outcome: PASS (test passes, check fails but xfail expects it to fail)
+
+/test/check-multiple:
+    summary: Test with multiple checks with different result interpretations
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: respect
+        - how: dmesg
+          result: xfail
+        - how: dmesg
+          failure-pattern: '.*'
+          result: ignore
+    # Expected outcome: FAIL (first dmesg check fails and is respected, second dmesg check passes but xfail expects it to fail, third failing dmesg check is ignored)
+
+/test/check-override:
+    summary: Test with failing dmesg check but overridden by test result
+    test: echo "Test passed"
+    framework: shell
+    duration: 1m
+    result: pass
+    check:
+        - how: dmesg
+          failure-pattern: '.*'
+          result: respect
+    # Expected outcome: PASS (test passes, check fails but is overridden by 'result: pass')

tests/execute/result/main.fmf

@@ -10,3 +10,6 @@
 /special:
     summary: Test special characters generated to tmt-report-results.yaml
     test: ./special.sh
+/check_results:
+    summary: Test behaviour of check results, including after-test checks
+    test: ./check_results.sh

tmt/checks/__init__.py

@@ -1,5 +1,4 @@
 import dataclasses
-import enum
 import functools
 from typing import TYPE_CHECKING, Any, Callable, Generic, Optional, TypedDict, TypeVar, cast
 
@@ -20,6 +19,8 @@
     from tmt.steps.execute import TestInvocation
     from tmt.steps.provision import Guest
 
+from tmt.checks.events import CheckEvent
+from tmt.result import CheckResultInterpret
 
 #: A type variable representing a :py:class:`Check` instances.
 CheckT = TypeVar('CheckT', bound='Check')
@@ -67,20 +68,7 @@ def find_plugin(name: str) -> 'CheckPluginClass':
 class _RawCheck(TypedDict):
     how: str
     enabled: bool
-
-
-class CheckEvent(enum.Enum):
-    """ Events in test runtime when a check can be executed """
-
-    BEFORE_TEST = 'before-test'
-    AFTER_TEST = 'after-test'
-
-    @classmethod
-    def from_spec(cls, spec: str) -> 'CheckEvent':
-        try:
-            return CheckEvent(spec)
-        except ValueError:
-            raise tmt.utils.SpecificationError(f"Invalid test check event '{spec}'.")
+    result: str
 
 
 @dataclasses.dataclass
@@ -100,6 +88,11 @@ class Check(
         default=True,
         is_flag=True,
         help='Whether the check is enabled or not.')
+    result: CheckResultInterpret = field(
+        default=CheckResultInterpret.RESPECT,
+        serialize=lambda result: result.value,
+        unserialize=CheckResultInterpret.from_spec,
+        help='How to interpret the check result.')
 
     @functools.cached_property
     def plugin(self) -> 'CheckPluginClass':
@@ -228,14 +221,16 @@ def normalize_test_check(
     if isinstance(raw_test_check, str):
         try:
             return CheckPlugin.delegate(
-                raw_data={'how': raw_test_check, 'enabled': True},
+                raw_data={'how': raw_test_check, 'enabled': True, 'result': 'respect'},
                 logger=logger)
 
         except Exception as exc:
             raise tmt.utils.SpecificationError(
                 f"Cannot instantiate check from '{key_address}'.") from exc
 
     if isinstance(raw_test_check, dict):
+        if 'result' not in raw_test_check:
+            raw_test_check['result'] = 'respect'
         try:
             return CheckPlugin.delegate(
                 raw_data=cast(_RawCheck, raw_test_check),