Skip to content

Commit

Permalink
Googletest export
Browse files Browse the repository at this point in the history 
gtest: Output a canned test suite for environment failures in XML/JSON

This surfaces useful information about the environment failure in a structured form.

As we can see from the updated test, previously unsurfaced information is now present.

PiperOrigin-RevId: 362292322
  • Loading branch information
Abseil Team authored and Andy Soffer committed Mar 16, 2021
1 parent 3bd41ab commit ac1d60c
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 19 deletions.
104 changes: 104 additions & 0 deletions googletest/src/gtest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3928,6 +3928,12 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener {
// Streams an XML CDATA section, escaping invalid CDATA sequences as needed.
static void OutputXmlCDataSection(::std::ostream* stream, const char* data);

// Streams a test suite XML stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputXmlTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams an XML representation of a TestResult object.
static void OutputXmlTestResult(::std::ostream* stream,
const TestResult& result);
Expand Down Expand Up @@ -4147,6 +4153,43 @@ void XmlUnitTestResultPrinter::OutputXmlAttribute(
*stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\"";
}

// Streams a test suite XML stanza containing the given test result.
void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a minimal test suite with one test.
*stream << " <testsuite";
OutputXmlAttribute(stream, "testsuite", "name", "NonTestSuiteFailure");
OutputXmlAttribute(stream, "testsuite", "tests", "1");
OutputXmlAttribute(stream, "testsuite", "failures", "1");
OutputXmlAttribute(stream, "testsuite", "disabled", "0");
OutputXmlAttribute(stream, "testsuite", "skipped", "0");
OutputXmlAttribute(stream, "testsuite", "errors", "0");
OutputXmlAttribute(stream, "testsuite", "time",
FormatTimeInMillisAsSeconds(result.elapsed_time()));
OutputXmlAttribute(
stream, "testsuite", "timestamp",
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));
*stream << ">";

// Output the boilerplate for a minimal test case with a single test.
*stream << " <testcase";
OutputXmlAttribute(stream, "testcase", "name", "");
OutputXmlAttribute(stream, "testcase", "status", "run");
OutputXmlAttribute(stream, "testcase", "result", "completed");
OutputXmlAttribute(stream, "testcase", "classname", "");
OutputXmlAttribute(stream, "testcase", "time",
FormatTimeInMillisAsSeconds(result.elapsed_time()));
OutputXmlAttribute(
stream, "testcase", "timestamp",
FormatEpochTimeInMillisAsIso8601(result.start_timestamp()));

// Output the actual test result.
OutputXmlTestResult(stream, result);

// Complete the test suite.
*stream << " </testsuite>\n";
}

// Prints an XML representation of a TestInfo object.
void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream,
const char* test_suite_name,
Expand Down Expand Up @@ -4309,6 +4352,13 @@ void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream,
if (unit_test.GetTestSuite(i)->reportable_test_count() > 0)
PrintXmlTestSuite(stream, *unit_test.GetTestSuite(i));
}

// If there was a test failure outside of one of the test suites (like in a
// test environment) include that in the output.
if (unit_test.ad_hoc_test_result().Failed()) {
OutputXmlTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result());
}

*stream << "</" << kTestsuites << ">\n";
}

Expand Down Expand Up @@ -4399,6 +4449,12 @@ class JsonUnitTestResultPrinter : public EmptyTestEventListener {
const std::string& indent,
bool comma = true);

// Streams a test suite JSON stanza containing the given test result.
//
// Requires: result.Failed()
static void OutputJsonTestSuiteForTestResult(::std::ostream* stream,
const TestResult& result);

// Streams a JSON representation of a TestResult object.
static void OutputJsonTestResult(::std::ostream* stream,
const TestResult& result);
Expand Down Expand Up @@ -4553,6 +4609,48 @@ void JsonUnitTestResultPrinter::OutputJsonKey(
*stream << ",\n";
}

// Streams a test suite JSON stanza containing the given test result.
void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult(
::std::ostream* stream, const TestResult& result) {
// Output the boilerplate for a new test suite.
*stream << Indent(4) << "{\n";
OutputJsonKey(stream, "testsuite", "name", "NonTestSuiteFailure", Indent(6));
OutputJsonKey(stream, "testsuite", "tests", 1, Indent(6));
if (!GTEST_FLAG(list_tests)) {
OutputJsonKey(stream, "testsuite", "failures", 1, Indent(6));
OutputJsonKey(stream, "testsuite", "disabled", 0, Indent(6));
OutputJsonKey(stream, "testsuite", "skipped", 0, Indent(6));
OutputJsonKey(stream, "testsuite", "errors", 0, Indent(6));
OutputJsonKey(stream, "testsuite", "time",
FormatTimeInMillisAsDuration(result.elapsed_time()),
Indent(6));
OutputJsonKey(stream, "testsuite", "timestamp",
FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()),
Indent(6));
}
*stream << Indent(6) << "\"testsuite\": [\n";

// Output the boilerplate for a new test case.
*stream << Indent(8) << "{\n";
OutputJsonKey(stream, "testcase", "name", "", Indent(10));
OutputJsonKey(stream, "testcase", "status", "RUN", Indent(10));
OutputJsonKey(stream, "testcase", "result", "COMPLETED", Indent(10));
OutputJsonKey(stream, "testcase", "timestamp",
FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()),
Indent(10));
OutputJsonKey(stream, "testcase", "time",
FormatTimeInMillisAsDuration(result.elapsed_time()),
Indent(10));
OutputJsonKey(stream, "testcase", "classname", "", Indent(10), false);
*stream << TestPropertiesAsJson(result, Indent(10));

// Output the actual test result.
OutputJsonTestResult(stream, result);

// Finish the test suite.
*stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}";
}

// Prints a JSON representation of a TestInfo object.
void JsonUnitTestResultPrinter::OutputJsonTestInfo(::std::ostream* stream,
const char* test_suite_name,
Expand Down Expand Up @@ -4712,6 +4810,12 @@ void JsonUnitTestResultPrinter::PrintJsonUnitTest(std::ostream* stream,
}
}

// If there was a test failure outside of one of the test suites (like in a
// test environment) include that in the output.
if (unit_test.ad_hoc_test_result().Failed()) {
OutputJsonTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result());
}

*stream << "\n" << kIndent << "]\n" << "}\n";
}

Expand Down
70 changes: 57 additions & 13 deletions googletest/test/googletest-json-output-unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,15 +612,59 @@
}],
}

EXPECTED_EMPTY = {
u'tests': 0,
u'failures': 0,
u'disabled': 0,
u'errors': 0,
u'time': u'*',
u'timestamp': u'*',
u'name': u'AllTests',
u'testsuites': [],
EXPECTED_NO_TEST = {
u'tests':
0,
u'failures':
0,
u'disabled':
0,
u'errors':
0,
u'time':
u'*',
u'timestamp':
u'*',
u'name':
u'AllTests',
u'testsuites': [{
u'name':
u'NonTestSuiteFailure',
u'tests':
1,
u'failures':
1,
u'disabled':
0,
u'skipped':
0,
u'errors':
0,
u'time':
u'*',
u'timestamp':
u'*',
u'testsuite': [{
u'name':
u'',
u'status':
u'RUN',
u'result':
u'COMPLETED',
u'time':
u'*',
u'timestamp':
u'*',
u'classname':
u'',
u'failures': [{
u'failure': u'gtest_no_test_unittest.cc:*\n'
u'Expected equality of these values:\n'
u' 1\n 2' + STACK_TRACE_TEMPLATE,
u'type': u'',
}]
}]
}],
}

GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
Expand All @@ -645,14 +689,14 @@ def testNonEmptyJsonOutput(self):
"""
self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)

def testEmptyJsonOutput(self):
def testNoTestJsonOutput(self):
"""Verifies JSON output for a Google Test binary without actual tests.
Runs a test program that generates an empty JSON output, and
tests that the JSON output is expected.
Runs a test program that generates an JSON output for a binary with no
tests, and tests that the JSON output is expected.
"""

self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0)
self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_NO_TEST, 0)

def testTimestampValue(self):
"""Checks whether the timestamp attribute in the JSON output is valid.
Expand Down
22 changes: 16 additions & 6 deletions googletest/test/gtest_xml_output_unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,10 +216,20 @@
</testsuite>
</testsuites>"""

EXPECTED_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
EXPECTED_NO_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="0" failures="0" disabled="0" errors="0" time="*"
timestamp="*" name="AllTests">
</testsuites>"""
<testsuite name="NonTestSuiteFailure" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="" status="run" result="completed" time="*" timestamp="*" classname="">
<failure message="gtest_no_test_unittest.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2" type=""><![CDATA[gtest_no_test_unittest.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
</testsuites>""" % {
'stack': STACK_TRACE_TEMPLATE
}

GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)

Expand All @@ -242,14 +252,14 @@ def testNonEmptyXmlOutput(self):
"""
self._TestXmlOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY_XML, 1)

def testEmptyXmlOutput(self):
def testNoTestXmlOutput(self):
"""Verifies XML output for a Google Test binary without actual tests.
Runs a test program that generates an empty XML output, and
tests that the XML output is expected.
Runs a test program that generates an XML output for a binary without tests,
and tests that the XML output is expected.
"""

self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_EMPTY_XML, 0)
self._TestXmlOutput('gtest_no_test_unittest', EXPECTED_NO_TEST_XML, 0)

def testTimestampValue(self):
"""Checks whether the timestamp attribute in the XML output is valid.
Expand Down

0 comments on commit ac1d60c

Please sign in to comment.