Merge pull request #23 from Xray-App/report_only_annotated

report only annotated tests; fixes #21

README.md

@@ -260,6 +260,7 @@ If you choose to use a properties file, it should be named `src/test/resources/x
 - `test_environments`: (optional) Test Environment(s) to assign to the Test Execution
 
 - `report_filename`: the name of the report. Default is "xray-report.json"
+- `report_only_annotated`: (optional) indicates if only annotated test methods with @XrayTest or @Requirement should be included in the report. Defaults to false.
 - `use_manual_tests_for_regular_tests`: Use "Manual" tests as an abstraction of regular, non-datadriven, tests. Default is false (i.e., "Generic" tests will be created).
 - `use_manual_tests_for_datadriven_tests`: Use "Manual" tests as an abstraction of DD tests during autoprovisioning. Default is true (for the time being, only these are supported)
 

src/main/java/app/getxray/xray/testng/listeners/XrayJsonReporter.java

@@ -123,6 +123,7 @@ private void loadConfigPropertiesFile() {
                 this.config.setXrayCloud(!"false".equals(properties.getProperty("xray_cloud")));  // true, if not specified
                 this.config.setUseManualTestsForDatadrivenTests(!"false".equals(properties.getProperty("use_manual_tests_for_datadriven_tests"))); // true, if not specified
                 this.config.setUseManualTestsForRegularTests("true".equals(properties.getProperty("use_manual_tests_for_regular_tests"))); // false, if not specified
+                this.config.setReportOnlyAnnotatedTests("true".equals(properties.getProperty("report_only_annotated_tests"))); // false, if not specified
 			}
         } catch (Exception e) {
             LOGGER.error("error loading listener configuration from properties files", e);
@@ -209,7 +210,10 @@ public void generateReport(List<XmlSuite> xmlSuites, List<ISuite> suites, String
         }
 
         for (String testMethod : results.keySet()) {
-            addTestResults(tests, results.get(testMethod));
+            Method method = results.get(testMethod).get(0).getMethod().getConstructorOrMethod().getMethod();
+            if (!this.config.isReportOnlyAnnotatedTests() || (this.config.isReportOnlyAnnotatedTests() && (method.isAnnotationPresent(XrayTest.class) || method.isAnnotationPresent(Requirement.class)))) {
+                addTestResults(tests, results.get(testMethod));
+            }
         }
         report.put("tests", tests);
         saveReport(outputDirectory, report);

src/main/java/app/getxray/xray/testng/listeners/XrayJsonReporterConfig.java

@@ -82,6 +82,11 @@ public class XrayJsonReporterConfig implements IReporterConfig {
     private String reportFilename = "xray-report.json";
 
 
+    /**
+     * Indicates if only annotated test methods with @XrayTest or @Requirement should be included in the report. Defaults to false.
+     */
+    private boolean reportOnlyAnnotatedTests = false;
+
     public void setXrayCloud(boolean xrayCloud) {
         this.xrayCloud = xrayCloud;
     }
@@ -185,4 +190,12 @@ public void setUseManualTestsForRegularTests(boolean useManualTestsForRegularTes
     public boolean isUseManualTestsForRegularTests() {
         return useManualTestsForRegularTests;
     }
+
+    public void setReportOnlyAnnotatedTests(boolean reportOnlyAnnotatedTests) {
+        this.reportOnlyAnnotatedTests = reportOnlyAnnotatedTests;
+    }
+
+    public boolean isReportOnlyAnnotatedTests() {
+        return reportOnlyAnnotatedTests;
+    }
 }

src/test/java/app/getxray/xray/testng/tests/XrayJsonReportTests.java

@@ -62,7 +62,7 @@ public void tearDown() throws Exception {
     }
 
     @Test
-    public void legacyTestsShouldBeMappedToATest() throws Exception {
+    public void legacyTestsShouldBeMappedToATestByDefault() throws Exception {
         String testMethodName = "legacyTest";
         executeTestMethod(BASIC_EXAMPLES_CLASS, testMethodName);
 
@@ -80,6 +80,21 @@ public void legacyTestsShouldBeMappedToATest() throws Exception {
         JSONAssert.assertEquals(expectedTestInfo, actualTestInfo, JSONCompareMode.LENIENT);
     }
 
+    @Test
+    public void legacyTestsShouldNotBeMappedToATestIfReportOnlyAnnotatedTests() throws Exception {
+        String customProperties = "report_only_annotated_tests=true\n";
+        Path customPropertiesFile = Files.createTempFile("xray", ".properties");
+        Files.write(customPropertiesFile, customProperties.getBytes());
+
+        String testMethodName = "legacyTest";
+        executeTestMethodWithCustomProperties(BASIC_EXAMPLES_CLASS, testMethodName, customPropertiesFile.toAbsolutePath().toString());
+
+        JSONObject report = readJsonFile(tempDirectory.resolve(REPORT_NAME));
+
+        JSONArray actualTests = (JSONArray)report.getJSONArray("tests");
+        Assert.assertEquals(actualTests.length(), 0);
+    }
+
     @Test
     public void shouldMapXrayTestSummaryToTestIssueSummary() throws Exception {
         String testMethodName = "annotatedWithXrayTestSummary";
@@ -106,7 +121,6 @@ public void shouldMapXrayTestDescriptionToTestIssueSummary() throws Exception {
 
         JSONObject report = readJsonFile(tempDirectory.resolve(REPORT_NAME));
 
-
         JSONArray actualTests = (JSONArray)report.getJSONArray("tests");
         Assert.assertEquals(actualTests.length(), 1);
         JSONObject actualTest = (JSONObject)(report.getJSONArray("tests")).get(0);
@@ -115,7 +129,6 @@ public void shouldMapXrayTestDescriptionToTestIssueSummary() throws Exception {
 
         JSONObject expectedTestInfo = new JSONObject();
         expectedTestInfo.put("summary", "custom description");
-        // expectedTestInfo.put("type", "Manual");
 
         JSONAssert.assertEquals(expectedTestInfo, actualTestInfo, JSONCompareMode.LENIENT);
     }
@@ -161,6 +174,30 @@ public void shouldMapXrayTestKeyToTestIssueKey() throws Exception {
         JSONAssert.assertEquals(expectedTest, actualTest, JSONCompareMode.LENIENT);
     }
 
+    @Test
+    public void shouldMapXrayTestKeyToTestIssueKeyIfReportOnlyAnnotatedTests() throws Exception {
+        String customProperties = "report_only_annotated_tests=true\n";
+        Path customPropertiesFile = Files.createTempFile("xray", ".properties");
+        Files.write(customPropertiesFile, customProperties.getBytes());
+
+        String testMethodName = "annotatedWithXrayTestKey";
+        executeTestMethodWithCustomProperties(BASIC_EXAMPLES_CLASS, testMethodName, customPropertiesFile.toAbsolutePath().toString());
+
+        JSONObject report = readJsonFile(tempDirectory.resolve(REPORT_NAME));
+
+
+        JSONArray actualTests = (JSONArray)report.getJSONArray("tests");
+        Assert.assertEquals(actualTests.length(), 1);
+        JSONObject actualTest = (JSONObject)(report.getJSONArray("tests")).get(0);
+        Assert.assertFalse(actualTest.has("testInfo"));
+
+        JSONObject expectedTest = new JSONObject();
+        expectedTest.put("testKey", "CALC-2000");
+        expectedTest.put("status", "PASSED");
+
+        JSONAssert.assertEquals(expectedTest, actualTest, JSONCompareMode.LENIENT);
+    }
+
     @Test
     public void shouldProcessTestWithParametersAnnotationAsManual() throws Exception {
         String testMethodName = "givenNumberFromXML_ifEvenCheckOK_thenCorrect";
@@ -271,6 +308,34 @@ public void shouldMapXrayRequirementKeyToTestcaseAttribute() throws Exception {
         JSONAssert.assertEquals(expectedTest, actualTest, JSONCompareMode.LENIENT);
     }
 
+    @Test
+    public void shouldMapXrayRequirementKeyToTestcaseAttributeIfReportOnlyAnnotatedTests() throws Exception {
+        String customProperties = "report_only_annotated_tests=true\n";
+        Path customPropertiesFile = Files.createTempFile("xray", ".properties");
+        Files.write(customPropertiesFile, customProperties.getBytes());
+
+        String testMethodName = "annotatedWithRequirementKey";
+        executeTestMethodWithCustomProperties(BASIC_EXAMPLES_CLASS, testMethodName, customPropertiesFile.toAbsolutePath().toString());
+
+        JSONObject report = readJsonFile(tempDirectory.resolve(REPORT_NAME));
+
+        JSONArray actualTests = (JSONArray)report.getJSONArray("tests");
+        Assert.assertEquals(actualTests.length(), 1);
+        JSONObject actualTest = (JSONObject)(report.getJSONArray("tests")).get(0);
+        Assert.assertTrue(actualTest.has("testInfo"));
+        JSONObject actualTestInfo = actualTest.getJSONObject("testInfo");
+        JSONObject expectedTestInfo = new JSONObject();
+        expectedTestInfo.put("summary", testMethodName);
+        expectedTestInfo.put("type", "Generic");
+        String[] requirementKeys = { "CALC-1234" };
+        expectedTestInfo.put("requirementKeys", new JSONArray(requirementKeys));  
+        JSONAssert.assertEquals(expectedTestInfo, actualTestInfo, JSONCompareMode.LENIENT);   
+
+        JSONObject expectedTest = new JSONObject();
+        expectedTest.put("status", "PASSED");
+        JSONAssert.assertEquals(expectedTest, actualTest, JSONCompareMode.LENIENT);
+    }
+
     @Test
     public void shouldMapXraySpaceDelimitedLabelsToTestcaseAttribute() throws Exception {
         String testMethodName = "annotatedWithXrayTestLabels";