diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc index 34aec9be9e00..535dd7bd657f 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.7.1.adoc @@ -27,6 +27,8 @@ GitHub. of this change may be witnessed by end users and test engine or extension authors. * Method `scanForClassesInPackage(String)` in `ClasspathScanner` now returns a valid list of class names when the package name is equal to the name of a module on the module path. +* The legacy XML report now always includes container-level failures (e.g. from + `@BeforeAll` Jupiter lifecycle methods). ==== Deprecations and Breaking Changes diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java index 53b30d51c6ad..a1940053c3e2 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoEngineExecutionContext.java @@ -13,5 +13,5 @@ /** * @since 1.0 */ -class DemoEngineExecutionContext implements EngineExecutionContext { +public class DemoEngineExecutionContext implements EngineExecutionContext { } diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java index 731357e4722d..ef827fe13a6a 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestDescriptor.java @@ -26,7 +26,7 @@ public class DemoHierarchicalTestDescriptor extends AbstractTestDescriptor imple private String skippedReason; private boolean skipped; - DemoHierarchicalTestDescriptor(UniqueId uniqueId, String displayName, Runnable executeBlock) { + public DemoHierarchicalTestDescriptor(UniqueId uniqueId, String displayName, Runnable executeBlock) { this(uniqueId, displayName, null, executeBlock); } diff --git a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java index a2db3afaeeab..a7efea183517 100644 --- a/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java +++ b/junit-platform-engine/src/testFixtures/java/org/junit/platform/engine/support/hierarchical/DemoHierarchicalTestEngine.java @@ -11,6 +11,7 @@ package org.junit.platform.engine.support.hierarchical; import java.lang.reflect.Method; +import java.util.function.Function; import org.junit.platform.engine.EngineDiscoveryRequest; import org.junit.platform.engine.ExecutionRequest; @@ -59,10 +60,8 @@ public DemoHierarchicalTestDescriptor addTest(Method testMethod, Runnable execut } public DemoHierarchicalTestDescriptor addTest(String uniqueName, String displayName, Runnable executeBlock) { - UniqueId uniqueId = engineDescriptor.getUniqueId().append("test", uniqueName); - DemoHierarchicalTestDescriptor child = new DemoHierarchicalTestDescriptor(uniqueId, displayName, executeBlock); - engineDescriptor.addChild(child); - return child; + return addChild(uniqueName, uniqueId -> new DemoHierarchicalTestDescriptor(uniqueId, displayName, executeBlock), + "test"); } public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, String displayName, TestSource source) { @@ -76,11 +75,17 @@ public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, Runna public DemoHierarchicalContainerDescriptor addContainer(String uniqueName, String displayName, TestSource source, Runnable beforeBlock) { - UniqueId uniqueId = engineDescriptor.getUniqueId().append("container", uniqueName); - DemoHierarchicalContainerDescriptor container = new DemoHierarchicalContainerDescriptor(uniqueId, displayName, - source, beforeBlock); - engineDescriptor.addChild(container); - return container; + return addChild(uniqueName, + uniqueId -> new DemoHierarchicalContainerDescriptor(uniqueId, displayName, source, beforeBlock), + "container"); + } + + public > T addChild(String uniqueName, + Function creator, String segmentType) { + var uniqueId = engineDescriptor.getUniqueId().append(segmentType, uniqueName); + var child = creator.apply(uniqueId); + engineDescriptor.addChild(child); + return child; } @Override diff --git a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java index 16cf43236705..674b5eab805b 100644 --- a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java +++ b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportData.java @@ -11,8 +11,8 @@ package org.junit.platform.reporting.legacy.xml; import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; import static org.junit.platform.engine.TestExecutionResult.Status.ABORTED; -import static org.junit.platform.engine.TestExecutionResult.Status.SUCCESSFUL; import java.time.Clock; import java.time.Duration; @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Predicate; @@ -103,19 +104,11 @@ String getSkipReason(TestIdentifier testIdentifier) { }).orElse(null); } - Optional getResult(TestIdentifier testIdentifier) { - if (this.finishedTests.containsKey(testIdentifier)) { - return Optional.of(this.finishedTests.get(testIdentifier)); - } - Optional parent = this.testPlan.getParent(testIdentifier); - Optional ancestor = findAncestor(parent, this.finishedTests::containsKey); - if (ancestor.isPresent()) { - TestExecutionResult result = this.finishedTests.get(ancestor.get()); - if (result.getStatus() != SUCCESSFUL) { - return Optional.of(result); - } - } - return Optional.empty(); + List getResults(TestIdentifier testIdentifier) { + return getAncestors(testIdentifier).stream() // + .map(this.finishedTests::get) // + .filter(Objects::nonNull) // + .collect(toList()); } List getReportEntries(TestIdentifier testIdentifier) { @@ -123,12 +116,11 @@ List getReportEntries(TestIdentifier testIdentifier) { } private Optional findSkippedAncestor(TestIdentifier testIdentifier) { - return findAncestor(Optional.of(testIdentifier), this.skippedTests::containsKey); + return findAncestor(testIdentifier, this.skippedTests::containsKey); } - private Optional findAncestor(Optional testIdentifier, - Predicate predicate) { - Optional current = testIdentifier; + private Optional findAncestor(TestIdentifier testIdentifier, Predicate predicate) { + Optional current = Optional.of(testIdentifier); while (current.isPresent()) { if (predicate.test(current.get())) { return current; @@ -138,4 +130,14 @@ private Optional findAncestor(Optional testIdent return Optional.empty(); } + private List getAncestors(TestIdentifier testIdentifier) { + TestIdentifier current = testIdentifier; + List ancestors = new ArrayList<>(); + while (current != null) { + ancestors.add(current); + current = this.testPlan.getParent(current).orElse(null); + } + return ancestors; + } + } diff --git a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java index 8348e179e6ca..2ab3c6a17856 100644 --- a/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java +++ b/junit-platform-reporting/src/main/java/org/junit/platform/reporting/legacy/xml/XmlReportWriter.java @@ -12,12 +12,23 @@ import static java.text.MessageFormat.format; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE_TIME; +import static java.util.Collections.emptyList; +import static java.util.Comparator.naturalOrder; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import static org.junit.platform.commons.util.ExceptionUtils.readStackTrace; import static org.junit.platform.commons.util.StringUtils.isNotBlank; import static org.junit.platform.engine.TestExecutionResult.Status.FAILED; import static org.junit.platform.launcher.LauncherConstants.STDERR_REPORT_ENTRY_KEY; import static org.junit.platform.launcher.LauncherConstants.STDOUT_REPORT_ENTRY_KEY; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.ERROR; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.FAILURE; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.SKIPPED; +import static org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type.SUCCESS; import java.io.Writer; import java.net.InetAddress; @@ -25,10 +36,13 @@ import java.text.NumberFormat; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Properties; import java.util.TreeSet; @@ -41,7 +55,9 @@ import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; import org.junit.platform.reporting.legacy.LegacyReportingUtils; +import org.junit.platform.reporting.legacy.xml.XmlReportWriter.AggregatedTestResult.Type; /** * {@code XmlReportWriter} writes an XML report whose format is compatible @@ -62,18 +78,28 @@ class XmlReportWriter { this.reportData = reportData; } - void writeXmlReport(TestIdentifier testIdentifier, Writer out) throws XMLStreamException { - // @formatter:off - List tests = this.reportData.getTestPlan().getDescendants(testIdentifier) - .stream() - .filter(TestIdentifier::isTest) - .collect(toList()); - // @formatter:on - writeXmlReport(testIdentifier, tests, out); + void writeXmlReport(TestIdentifier rootDescriptor, Writer out) throws XMLStreamException { + TestPlan testPlan = this.reportData.getTestPlan(); + Map tests = testPlan.getDescendants(rootDescriptor) // + .stream() // + .filter(testIdentifier -> shouldInclude(testPlan, testIdentifier)) // + .collect(toMap(identity(), this::toAggregatedResult)); // + writeXmlReport(rootDescriptor, tests, out); } - private void writeXmlReport(TestIdentifier testIdentifier, List tests, Writer out) - throws XMLStreamException { + private AggregatedTestResult toAggregatedResult(TestIdentifier testIdentifier) { + if (this.reportData.wasSkipped(testIdentifier)) { + return AggregatedTestResult.skipped(); + } + return AggregatedTestResult.nonSkipped(this.reportData.getResults(testIdentifier)); + } + + private boolean shouldInclude(TestPlan testPlan, TestIdentifier testIdentifier) { + return testIdentifier.isTest() || testPlan.getChildren(testIdentifier).isEmpty(); + } + + private void writeXmlReport(TestIdentifier testIdentifier, Map tests, + Writer out) throws XMLStreamException { XMLOutputFactory factory = XMLOutputFactory.newInstance(); XMLStreamWriter xmlWriter = factory.createXMLStreamWriter(out); @@ -85,8 +111,8 @@ private void writeXmlReport(TestIdentifier testIdentifier, List xmlWriter.close(); } - private void writeTestsuite(TestIdentifier testIdentifier, List tests, XMLStreamWriter writer) - throws XMLStreamException { + private void writeTestsuite(TestIdentifier testIdentifier, Map tests, + XMLStreamWriter writer) throws XMLStreamException { // NumberFormat is not thread-safe. Thus, we instantiate it here and pass it to // writeTestcase instead of using a constant @@ -94,13 +120,13 @@ private void writeTestsuite(TestIdentifier testIdentifier, List writer.writeStartElement("testsuite"); - writeSuiteAttributes(testIdentifier, tests, numberFormat, writer); + writeSuiteAttributes(testIdentifier, tests.values(), numberFormat, writer); newLine(writer); writeSystemProperties(writer); - for (TestIdentifier test : tests) { - writeTestcase(test, numberFormat, writer); + for (Entry entry : tests.entrySet()) { + writeTestcase(entry.getKey(), entry.getValue(), numberFormat, writer); } writeOutputElement("system-out", formatNonStandardAttributesAsString(testIdentifier), writer); @@ -109,22 +135,24 @@ private void writeTestsuite(TestIdentifier testIdentifier, List newLine(writer); } - private void writeSuiteAttributes(TestIdentifier testIdentifier, List tests, + private void writeSuiteAttributes(TestIdentifier testIdentifier, Collection testResults, NumberFormat numberFormat, XMLStreamWriter writer) throws XMLStreamException { writeAttributeSafely(writer, "name", testIdentifier.getDisplayName()); - writeTestCounts(tests, writer); + writeTestCounts(testResults, writer); writeAttributeSafely(writer, "time", getTime(testIdentifier, numberFormat)); writeAttributeSafely(writer, "hostname", getHostname().orElse("")); writeAttributeSafely(writer, "timestamp", ISO_LOCAL_DATE_TIME.format(getCurrentDateTime())); } - private void writeTestCounts(List tests, XMLStreamWriter writer) throws XMLStreamException { - TestCounts testCounts = TestCounts.from(this.reportData, tests); - writeAttributeSafely(writer, "tests", String.valueOf(testCounts.getTotal())); - writeAttributeSafely(writer, "skipped", String.valueOf(testCounts.getSkipped())); - writeAttributeSafely(writer, "failures", String.valueOf(testCounts.getFailures())); - writeAttributeSafely(writer, "errors", String.valueOf(testCounts.getErrors())); + private void writeTestCounts(Collection testResults, XMLStreamWriter writer) + throws XMLStreamException { + Map counts = testResults.stream().map(it -> it.type).collect(groupingBy(identity(), counting())); + long total = counts.values().stream().mapToLong(Long::longValue).sum(); + writeAttributeSafely(writer, "tests", String.valueOf(total)); + writeAttributeSafely(writer, "skipped", counts.getOrDefault(SKIPPED, 0L).toString()); + writeAttributeSafely(writer, "failures", counts.getOrDefault(FAILURE, 0L).toString()); + writeAttributeSafely(writer, "errors", counts.getOrDefault(ERROR, 0L).toString()); } private void writeSystemProperties(XMLStreamWriter writer) throws XMLStreamException { @@ -141,8 +169,8 @@ private void writeSystemProperties(XMLStreamWriter writer) throws XMLStreamExcep newLine(writer); } - private void writeTestcase(TestIdentifier testIdentifier, NumberFormat numberFormat, XMLStreamWriter writer) - throws XMLStreamException { + private void writeTestcase(TestIdentifier testIdentifier, AggregatedTestResult testResult, + NumberFormat numberFormat, XMLStreamWriter writer) throws XMLStreamException { writer.writeStartElement("testcase"); @@ -151,7 +179,7 @@ private void writeTestcase(TestIdentifier testIdentifier, NumberFormat numberFor writeAttributeSafely(writer, "time", getTime(testIdentifier, numberFormat)); newLine(writer); - writeSkippedOrErrorOrFailureElement(testIdentifier, writer); + writeSkippedOrErrorOrFailureElement(testIdentifier, testResult, writer); List systemOutElements = new ArrayList<>(); List systemErrElements = new ArrayList<>(); @@ -172,16 +200,18 @@ private String getClassName(TestIdentifier testIdentifier) { return LegacyReportingUtils.getClassName(this.reportData.getTestPlan(), testIdentifier); } - private void writeSkippedOrErrorOrFailureElement(TestIdentifier testIdentifier, XMLStreamWriter writer) - throws XMLStreamException { + private void writeSkippedOrErrorOrFailureElement(TestIdentifier testIdentifier, AggregatedTestResult testResult, + XMLStreamWriter writer) throws XMLStreamException { - if (this.reportData.wasSkipped(testIdentifier)) { + if (testResult.type == SKIPPED) { writeSkippedElement(this.reportData.getSkipReason(testIdentifier), writer); } else { - Optional result = this.reportData.getResult(testIdentifier); - if (result.isPresent() && result.get().getStatus() == FAILED) { - writeErrorOrFailureElement(result.get(), writer); + Map>> throwablesByType = testResult.getThrowablesByType(); + for (Type type : EnumSet.of(FAILURE, ERROR)) { + for (Optional throwable : throwablesByType.getOrDefault(type, emptyList())) { + writeErrorOrFailureElement(type, throwable.orElse(null), writer); + } } } } @@ -198,17 +228,17 @@ private void writeSkippedElement(String reason, XMLStreamWriter writer) throws X newLine(writer); } - private void writeErrorOrFailureElement(TestExecutionResult result, XMLStreamWriter writer) + private void writeErrorOrFailureElement(Type type, Throwable throwable, XMLStreamWriter writer) throws XMLStreamException { - Optional throwable = result.getThrowable(); - if (throwable.isPresent()) { - writer.writeStartElement(isFailure(result) ? "failure" : "error"); - writeFailureAttributesAndContent(throwable.get(), writer); + String elementName = type == FAILURE ? "failure" : "error"; + if (throwable != null) { + writer.writeStartElement(elementName); + writeFailureAttributesAndContent(throwable, writer); writer.writeEndElement(); } else { - writer.writeEmptyElement("error"); + writer.writeEmptyElement(elementName); } newLine(writer); } @@ -342,54 +372,46 @@ private static boolean isFailure(TestExecutionResult result) { return throwable.isPresent() && throwable.get() instanceof AssertionError; } - private static class TestCounts { - - static TestCounts from(XmlReportData reportData, List tests) { - TestCounts counts = new TestCounts(tests.size()); - for (TestIdentifier test : tests) { - if (reportData.wasSkipped(test)) { - counts.skipped++; - } - else { - Optional result = reportData.getResult(test); - if (result.isPresent() && result.get().getStatus() == FAILED) { - if (isFailure(result.get())) { - counts.failures++; - } - else { - counts.errors++; - } - } - } - } - return counts; - } + static class AggregatedTestResult { - private final long total; - private long skipped; - private long failures; - private long errors; + private static final AggregatedTestResult SKIPPED_RESULT = new AggregatedTestResult(SKIPPED, emptyList()); - TestCounts(long total) { - this.total = total; + public static AggregatedTestResult skipped() { + return SKIPPED_RESULT; } - public long getTotal() { - return total; + public static AggregatedTestResult nonSkipped(List executionResults) { + Type type = executionResults.stream() // + .map(Type::from) // + .max(naturalOrder()) // + .orElse(SUCCESS); + return new AggregatedTestResult(type, executionResults); } - public long getSkipped() { - return skipped; - } + private final Type type; + private final List executionResults; - public long getFailures() { - return failures; + private AggregatedTestResult(Type type, List executionResults) { + this.type = type; + this.executionResults = executionResults; } - public long getErrors() { - return errors; + public Map>> getThrowablesByType() { + return executionResults.stream() // + .collect(groupingBy(Type::from, mapping(TestExecutionResult::getThrowable, toList()))); } + enum Type { + + SUCCESS, SKIPPED, FAILURE, ERROR; + + private static Type from(TestExecutionResult executionResult) { + if (executionResult.getStatus() == FAILED) { + return isFailure(executionResult) ? FAILURE : ERROR; + } + return SUCCESS; + } + } } } diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java index 76a1276649d6..40fb937b5141 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/LegacyXmlReportGeneratingListenerTests.java @@ -45,6 +45,9 @@ import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.junit.platform.engine.support.hierarchical.DemoEngineExecutionContext; +import org.junit.platform.engine.support.hierarchical.DemoHierarchicalContainerDescriptor; +import org.junit.platform.engine.support.hierarchical.DemoHierarchicalTestDescriptor; import org.junit.platform.engine.support.hierarchical.DemoHierarchicalTestEngine; import org.junit.platform.fakes.TestDescriptorStub; import org.junit.platform.launcher.TestIdentifier; @@ -58,8 +61,11 @@ */ class LegacyXmlReportGeneratingListenerTests { + @TempDir + Path tempDirectory; + @Test - void writesFileForSingleSucceedingTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleSucceedingTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("succeedingTest", "display<-->Name 😎", () -> { }); @@ -89,7 +95,7 @@ void writesFileForSingleSucceedingTest(@TempDir Path tempDirectory) throws Excep } @Test - void writesFileForSingleFailingTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleFailingTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("failingTest", () -> fail("expected to fail")); @@ -115,7 +121,7 @@ void writesFileForSingleFailingTest(@TempDir Path tempDirectory) throws Exceptio } @Test - void writesFileForSingleErroneousTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleErroneousTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("failingTest", () -> { throw new RuntimeException("error occurred"); @@ -143,7 +149,7 @@ void writesFileForSingleErroneousTest(@TempDir Path tempDirectory) throws Except } @Test - void writesFileForSingleSkippedTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleSkippedTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); var testDescriptor = engine.addTest("skippedTest", () -> fail("never called")); testDescriptor.markSkipped("should be skipped"); @@ -167,7 +173,7 @@ void writesFileForSingleSkippedTest(@TempDir Path tempDirectory) throws Exceptio @SuppressWarnings("ConstantConditions") @Test - void writesFileForSingleAbortedTest(@TempDir Path tempDirectory) throws Exception { + void writesFileForSingleAbortedTest() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("abortedTest", () -> assumeFalse(true, "deliberately aborted")); @@ -190,7 +196,7 @@ void writesFileForSingleAbortedTest(@TempDir Path tempDirectory) throws Exceptio } @Test - void measuresTimesInSeconds(@TempDir Path tempDirectory) throws Exception { + void measuresTimesInSeconds() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("firstTest", () -> { }); @@ -216,7 +222,7 @@ void measuresTimesInSeconds(@TempDir Path tempDirectory) throws Exception { } @Test - void testWithImmeasurableTimeIsOutputCorrectly(@TempDir Path tempDirectory) throws Exception { + void testWithImmeasurableTimeIsOutputCorrectly() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -229,7 +235,7 @@ void testWithImmeasurableTimeIsOutputCorrectly(@TempDir Path tempDirectory) thro } @Test - void writesFileForSkippedContainer(@TempDir Path tempDirectory) throws Exception { + void writesFileForSkippedContainer() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> fail("never called")); engine.getEngineDescriptor().markSkipped("should be skipped"); @@ -247,7 +253,7 @@ void writesFileForSkippedContainer(@TempDir Path tempDirectory) throws Exception } @Test - void writesFileForFailingContainer(@TempDir Path tempDirectory) throws Exception { + void writesFileForFailingContainer() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> fail("never called")); engine.getEngineDescriptor().setBeforeAllBehavior(() -> fail("failure before all tests")); @@ -269,7 +275,63 @@ void writesFileForFailingContainer(@TempDir Path tempDirectory) throws Exception } @Test - void writesSystemProperties(@TempDir Path tempDirectory) throws Exception { + void writesFileForFailingContainerWithoutTest() throws Exception { + var engine = new DemoHierarchicalTestEngine("dummy"); + engine.addContainer("failingContainer", () -> { + throw new RuntimeException("boom"); + }); + + executeTests(engine, tempDirectory); + + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("errors", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("failingContainer"); + assertThat(testcase.attr("classname")).isEqualTo("dummy"); + + var error = testcase.child("error"); + assertThat(error.attr("message")).isEqualTo("boom"); + assertThat(error.attr("type")).isEqualTo(RuntimeException.class.getName()); + assertThat(error.text()).containsSubsequence("RuntimeException: boom", "\tat"); + } + + @Test + void writesFileForContainerFailingAfterTest() throws Exception { + var engine = new DemoHierarchicalTestEngine("dummy"); + + var container = engine.addChild("failingContainer", + uniqueId -> new DemoHierarchicalContainerDescriptor(uniqueId, "failingContainer", null, null) { + @Override + public void after(DemoEngineExecutionContext context) { + throw new RuntimeException("boom"); + } + }, "child"); + container.addChild( + new DemoHierarchicalTestDescriptor(container.getUniqueId().append("test", "someTest"), "someTest", () -> { + })); + + executeTests(engine, tempDirectory); + + var testsuite = readValidXmlFile(tempDirectory.resolve("TEST-dummy.xml")); + + assertThat(testsuite.attr("tests", int.class)).isEqualTo(1); + assertThat(testsuite.attr("errors", int.class)).isEqualTo(1); + + var testcase = testsuite.child("testcase"); + assertThat(testcase.attr("name")).isEqualTo("someTest"); + assertThat(testcase.attr("classname")).isEqualTo("failingContainer"); + + var error = testcase.child("error"); + assertThat(error.attr("message")).isEqualTo("boom"); + assertThat(error.attr("type")).isEqualTo(RuntimeException.class.getName()); + assertThat(error.text()).containsSubsequence("RuntimeException: boom", "\tat"); + } + + @Test + void writesSystemProperties() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -283,7 +345,7 @@ void writesSystemProperties(@TempDir Path tempDirectory) throws Exception { } @Test - void writesHostNameAndTimestamp(@TempDir Path tempDirectory) throws Exception { + void writesHostNameAndTimestamp() throws Exception { var engine = new DemoHierarchicalTestEngine("dummy"); engine.addTest("test", () -> { }); @@ -299,7 +361,7 @@ void writesHostNameAndTimestamp(@TempDir Path tempDirectory) throws Exception { } @Test - void printsExceptionWhenReportsDirCannotBeCreated(@TempDir Path tempDirectory) throws Exception { + void printsExceptionWhenReportsDirCannotBeCreated() throws Exception { var reportsDir = tempDirectory.resolve("dummy.txt"); Files.write(reportsDir, Set.of("content")); @@ -313,7 +375,7 @@ void printsExceptionWhenReportsDirCannotBeCreated(@TempDir Path tempDirectory) t } @Test - void printsExceptionWhenReportCouldNotBeWritten(@TempDir Path tempDirectory) throws Exception { + void printsExceptionWhenReportCouldNotBeWritten() throws Exception { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); var xmlFile = tempDirectory.resolve("TEST-engine.xml"); @@ -329,7 +391,7 @@ void printsExceptionWhenReportCouldNotBeWritten(@TempDir Path tempDirectory) thr } @Test - void writesReportEntriesToSystemOutElement(@TempDir Path tempDirectory) throws Exception { + void writesReportEntriesToSystemOutElement() throws Exception { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); diff --git a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java index ca13a6782cb9..617280e1df0b 100644 --- a/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java +++ b/platform-tests/src/test/java/org/junit/platform/reporting/legacy/xml/XmlReportDataTests.java @@ -29,19 +29,19 @@ class XmlReportDataTests { @Test - void resultOfTestIdentifierWithoutAnyReportedEventsIsEmpty() { + void resultsOfTestIdentifierWithoutAnyReportedEventsAreEmpty() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).isEmpty(); + assertThat(results).isEmpty(); } @Test - void resultOfTestIdentifierWithoutReportedEventsIsFailureOfAncestor() { + void resultsOfTestIdentifierWithoutReportedEventsContainsOnlyFailureOfAncestor() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -50,13 +50,13 @@ void resultOfTestIdentifierWithoutReportedEventsIsFailureOfAncestor() { var failureOfAncestor = failed(new RuntimeException("failed!")); reportData.markFinished(testPlan.getTestIdentifier("[engine:engine]"), failureOfAncestor); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).contains(failureOfAncestor); + assertThat(results).containsExactly(failureOfAncestor); } @Test - void resultOfTestIdentifierWithoutReportedEventsIsEmptyWhenAncestorWasSuccessful() { + void resultsOfTestIdentifierWithoutReportedEventsContainsOnlySuccessOfAncestor() { var engineDescriptor = new EngineDescriptor(UniqueId.forEngine("engine"), "Engine"); engineDescriptor.addChild(new TestDescriptorStub(UniqueId.root("child", "test"), "test")); var testPlan = TestPlan.from(Set.of(engineDescriptor)); @@ -64,8 +64,8 @@ void resultOfTestIdentifierWithoutReportedEventsIsEmptyWhenAncestorWasSuccessful var reportData = new XmlReportData(testPlan, Clock.systemDefaultZone()); reportData.markFinished(testPlan.getTestIdentifier("[engine:engine]"), successful()); - var result = reportData.getResult(testPlan.getTestIdentifier("[child:test]")); + var results = reportData.getResults(testPlan.getTestIdentifier("[child:test]")); - assertThat(result).isEmpty(); + assertThat(results).containsExactly(successful()); } }