diff --git a/CHANGELOG.md b/CHANGELOG.md index dc7b9b7..4ed6807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## [Unreleased] ### Removed - Shutdown hook register on supplied Launch, by @HardNorth +### Added +- Implemented new feature: display last error log in scenario description, by @vrymar +- Implemented unit tests for the new feature, by @vrymar +### Changed +- Improved dependencies references in build.gradle, by @vrymar ## [5.0.3] ### Fixed diff --git a/build.gradle b/build.gradle index 72db23c..3a505f2 100644 --- a/build.gradle +++ b/build.gradle @@ -42,20 +42,21 @@ repositories { } dependencies { - api 'com.epam.reportportal:client-java:5.2.5' + api "com.epam.reportportal:client-java:${project.client_java_version}" + compileOnly "com.intuit.karate:karate-core:${project.karate_version}" - implementation 'org.slf4j:slf4j-api:2.0.7' + implementation "org.slf4j:slf4j-api:${project.slf4j_api_version}" testImplementation "com.intuit.karate:karate-core:${project.karate_version}" - testImplementation 'com.epam.reportportal:logger-java-logback:5.2.1' - testImplementation 'com.epam.reportportal:agent-java-test-utils:0.0.3' + testImplementation "com.epam.reportportal:logger-java-logback:${project.logger_java_logback_version}" + testImplementation "com.epam.reportportal:agent-java-test-utils:${project.agent_java_test_utils_version}" testImplementation "org.junit.jupiter:junit-jupiter-api:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-engine:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-params:${project.junit_version}" testImplementation "org.mockito:mockito-core:${project.mockito_version}" testImplementation "org.mockito:mockito-junit-jupiter:${project.mockito_version}" - testImplementation 'org.hamcrest:hamcrest-core:2.2' - testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' + testImplementation "org.hamcrest:hamcrest-core:${project.hamcrest_core_version}" + testImplementation "com.squareup.okhttp3:okhttp:${project.okhttp_version}" } test { diff --git a/gradle.properties b/gradle.properties index c433a06..d6e4060 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,6 +5,12 @@ gradle_version=8.2 karate_version=1.4.1 junit_version=5.10.1 mockito_version=5.4.0 +agent_java_test_utils_version=0.0.3 +client_java_version=5.2.5 +slf4j_api_version=2.0.7 +logger_java_logback_version=5.2.1 +hamcrest_core_version=2.2 +okhttp_version=4.12.0 scripts_url=https://github.com/raw/reportportal/gradle-scripts scripts_branch=develop excludeTests= diff --git a/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java b/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java index 19f97cc..3a6f7a3 100644 --- a/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java +++ b/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java @@ -20,25 +20,55 @@ import com.intuit.karate.Results; import com.intuit.karate.Runner; +/** + * Karate runner with ReportPortal integration + */ public class KarateReportPortalRunner { + /** + * Create a new builder for the Karate runner with ReportPortal integration + * + * @param paths paths to feature files + * @param type of the builder + * @return a new builder + */ public static > Builder path(String... paths) { Builder builder = new Builder<>(); return builder.path(paths); } + /** + * Builder for the Karate runner with ReportPortal integration + * + * @param type of the builder + */ public static class Builder> extends Runner.Builder { private ReportPortal rp; + /** + * Create a new builder + */ public Builder() { super(); } + /** + * Set the ReportPortal instance to use + * + * @param reportPortal the ReportPortal instance + * @return the builder + */ public Builder withReportPortal(ReportPortal reportPortal) { rp = reportPortal; return this; } + /** + * Run the tests in parallel + * + * @param threadCount number of threads to use + * @return the results of the tests + */ @Override public Results parallel(int threadCount) { if (rp == null) { diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java index b3d49dc..6254647 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java @@ -65,6 +65,11 @@ public class ReportPortalHook implements RuntimeHook { private final Map, Date> stepStartTimeMap = new ConcurrentHashMap<>(); private volatile Thread shutDownHook; + /** + * Create a new instance of the ReportPortalHook with the specified ReportPortal instance. + * + * @param reportPortal the ReportPortal instance + */ public ReportPortalHook(ReportPortal reportPortal) { launch = new MemoizingSupplier<>(() -> { ListenerParameters params = reportPortal.getParameters(); @@ -77,10 +82,15 @@ public ReportPortalHook(ReportPortal reportPortal) { }); } + /** + * Default constructor. Create a new instance of the ReportPortalHook with default ReportPortal instance. + */ + @SuppressWarnings("unused") public ReportPortalHook() { this(ReportPortal.builder().build()); } + @SuppressWarnings("unused") public ReportPortalHook(Supplier launchSupplier) { launch = new MemoizingSupplier<>(launchSupplier); } @@ -186,7 +196,7 @@ public void afterFeature(FeatureRuntime fr) { */ @Nonnull protected StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioRuntime sr) { - return ReportPortalUtils.buildStartScenarioRq(sr.scenario); + return ReportPortalUtils.buildStartScenarioRq(sr.result); } @Override @@ -206,9 +216,7 @@ public boolean beforeScenario(ScenarioRuntime sr) { */ @Nonnull protected FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioRuntime sr) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), - sr.result.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED - ); + return ReportPortalUtils.buildFinishScenarioRq(sr.result); } /** diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java index 7c0701b..d78382f 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java @@ -188,7 +188,7 @@ public void finishFeature(FeatureResult featureResult) { */ @Nonnull protected StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioResult scenarioResult) { - return ReportPortalUtils.buildStartScenarioRq(scenarioResult.getScenario()); + return ReportPortalUtils.buildStartScenarioRq(scenarioResult); } /** @@ -212,10 +212,7 @@ public void startScenario(ScenarioResult scenarioResult, FeatureResult featureRe */ @Nonnull protected FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioResult scenarioResult) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), - scenarioResult.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED - ); - + return ReportPortalUtils.buildFinishScenarioRq(scenarioResult); } /** diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java index ed5da64..a97231b 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java @@ -60,7 +60,8 @@ public class ReportPortalUtils { public static final String SKIPPED_ISSUE_KEY = "skippedIssue"; public static final String SCENARIO_CODE_REFERENCE_PATTERN = "%s/[SCENARIO:%s]"; public static final String EXAMPLE_CODE_REFERENCE_PATTERN = "%s/[EXAMPLE:%s%s]"; - public static final String MARKDOWN_DELIMITER_PATTERN = "%s\n\n---\n\n%s"; + public static final String MARKDOWN_DELIMITER = "\n\n---\n\n"; + public static final String MARKDOWN_DELIMITER_PATTERN = "%s" + MARKDOWN_DELIMITER + "%s"; private static final Logger LOGGER = LoggerFactory.getLogger(ReportPortalUtils.class); private static final String PARAMETER_ITEMS_START = "["; private static final String PARAMETER_ITEMS_END = "]"; @@ -294,35 +295,57 @@ public static TestCaseIdEntry getTestCaseId(@Nonnull Scenario scenario) { /** * Build ReportPortal request for start Scenario event * - * @param scenario Karate's Scenario object instance + * @param result Karate's ScenarioResult object instance * @return request to ReportPortal */ @Nonnull - public static StartTestItemRQ buildStartScenarioRq(@Nonnull Scenario scenario) { + public static StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioResult result) { + Scenario scenario = result.getScenario(); StartTestItemRQ rq = buildStartTestItemRq(scenario.getName(), Calendar.getInstance().getTime(), ItemType.STEP); rq.setCodeRef(getCodeRef(scenario)); rq.setTestCaseId(ofNullable(getTestCaseId(scenario)).map(TestCaseIdEntry::getId).orElse(null)); rq.setAttributes(toAttributes(scenario.getTags())); - List parameters = getParameters(scenario); - boolean hasParameters = ofNullable(parameters).filter(p -> !p.isEmpty()).isPresent(); - if (hasParameters) { - rq.setParameters(parameters); + rq.setParameters(getParameters(scenario)); + rq.setDescription(buildDescription(scenario, result.getErrorMessage(), getParameters(scenario))); + return rq; + } + + /** + * Build ReportPortal request for finish Scenario event + * + * @param result Karate's ScenarioResult object instance + * @return request to ReportPortal + */ + @Nonnull + public static FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioResult result) { + Scenario scenario = result.getScenario(); + FinishTestItemRQ rq = buildFinishTestItemRq(Calendar.getInstance().getTime(), result.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED); + rq.setDescription(buildDescription(scenario, result.getErrorMessage(), getParameters(scenario))); + return rq; + } + + @Nonnull + private static String buildDescription(@Nonnull Scenario scenario, @Nullable String errorMessage, @Nullable List parameters) { + StringBuilder descriptionBuilder = new StringBuilder(); + + if (parameters != null && !parameters.isEmpty()) { + descriptionBuilder.append(String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters))); + } + if (isNotBlank(scenario.getDescription())) { + appendWithDelimiter(descriptionBuilder, scenario.getDescription()); + } + if (isNotBlank(errorMessage)) { + appendWithDelimiter(descriptionBuilder, String.format("Error:\n%s", errorMessage)); } - String description = scenario.getDescription(); - if (isNotBlank(description)) { - if (hasParameters) { - rq.setDescription(String.format(MARKDOWN_DELIMITER_PATTERN, - String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters)), - description - )); - } else { - rq.setDescription(description); - } - } else if (hasParameters) { - rq.setDescription(String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters))); + return descriptionBuilder.toString(); + } + + private static void appendWithDelimiter(StringBuilder builder, String text) { + if (builder.length() > 0) { + builder.append(MARKDOWN_DELIMITER); } - return rq; + builder.append(text); } /** diff --git a/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java b/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java index 8128ee4..b35ccd8 100644 --- a/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java +++ b/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java @@ -75,7 +75,7 @@ public void test_description_for_all_possible_items(boolean report) { assertThat(featureStart.getDescription(), endsWith("feature/simple.feature")); StartTestItemRQ scenarioStart = scenarioCaptor.getValue(); - assertThat(scenarioStart.getDescription(), nullValue()); + assertThat(scenarioStart.getDescription(), emptyString()); stepCaptor.getAllValues().forEach(step -> assertThat(step.getDescription(), nullValue())); } diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java new file mode 100644 index 0000000..909c68c --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogTest { + + public static final String ERROR = "did not evaluate to 'true': actualFour != four\nclasspath:feature/simple_failed.feature:6"; + public static final String ERROR_MESSAGE = "Then assert actualFour != four\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + private static final String TEST_FEATURE = "classpath:feature/simple_failed.feature"; + private final String launchUuid = CommonUtils.namedId("launch_"); + private final String featureId = CommonUtils.namedId("feature_"); + private final String scenarioId = CommonUtils.namedId("scenario_"); + private final List stepIds = Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList()); + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, launchUuid, featureId, scenarioId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_in_description(boolean report) { + Results results; + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getItemUuid(), oneOf(stepIds.toArray(new String[0]))); + assertThat(log.getLaunchUuid(), equalTo(launchUuid)); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptorFinish = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client, times(1)).finishTestItem(same(scenarioId), scenarioCaptorFinish.capture()); + + List scenarios = scenarioCaptorFinish.getAllValues(); + FinishTestItemRQ scenario = scenarios.get(0); + assertThat(scenario.getDescription(), allOf(notNullValue(), equalTo(DESCRIPTION_ERROR_LOG))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java new file mode 100644 index 0000000..a74d244 --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest { + + public static final String MARKDOWN_DELIMITER_PATTERN_THREE_ARGS = "%s\n\n---\n\n%s\n\n---\n\n%s"; + public static final String ERROR = "did not evaluate to 'true': mathResult == 5\nclasspath:feature/simple_failed_description_examples.feature:8"; + public static final String ERROR_MESSAGE = "Then assert mathResult == 5\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + public static final String DESCRIPTION = "This is my Scenario description."; + private static final String EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN = + "Parameters:\n\n" + MarkdownUtils.TABLE_INDENT + "| vara | varb | result |\n" + MarkdownUtils.TABLE_INDENT + + "|------|------|--------|\n" + MarkdownUtils.TABLE_INDENT; + public static final String FIRST_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  2   |  2   |   4    |"; + public static final String SECOND_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  1   |  2   |   5    |"; + + public static final String FIRST_EXAMPLE_DESCRIPTION_WITHOUT_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN, + FIRST_EXAMPLE_DESCRIPTION, + DESCRIPTION + ); + public static final String SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN_THREE_ARGS, + SECOND_EXAMPLE_DESCRIPTION, + DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + + private static final String TEST_FEATURE = "classpath:feature/simple_failed_description_examples.feature"; + private final String featureId = CommonUtils.namedId("feature_"); + private final List exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList()); + private final List>> stepIds = exampleIds.stream() + .map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(2).collect(Collectors.toList()))) + .collect(Collectors.toList()); + + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, null, featureId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_examples_in_description(boolean report) { + Results results; + + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(0)), scenarioCaptor.capture()); + verify(client).finishTestItem(same(exampleIds.get(1)), scenarioCaptor.capture()); + + List> stepCaptors = new ArrayList<>(Collections.nCopies(stepIds.size(), ArgumentCaptor.forClass(FinishTestItemRQ.class))); + stepIds.forEach(pair -> pair.getValue().forEach(id -> verify(client).finishTestItem(same(id), stepCaptors.get(0).capture()))); + + FinishTestItemRQ firstScenarioRq = scenarioCaptor.getAllValues().get(0); + assertThat(firstScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.PASSED.name()))); + assertThat(firstScenarioRq.getDescription(), allOf(notNullValue(), equalTo(FIRST_EXAMPLE_DESCRIPTION_WITHOUT_ERROR_LOG))); + + FinishTestItemRQ secondScenarioRq = scenarioCaptor.getAllValues().get(1); + assertThat(secondScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name()))); + assertThat(secondScenarioRq.getDescription(), allOf(notNullValue(), equalTo(SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java new file mode 100644 index 0000000..454066c --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithDescriptionTest { + + public static final String ERROR = "did not evaluate to 'true': actualFour != four\nclasspath:feature/simple_failed_description.feature:9"; + public static final String ERROR_MESSAGE = "Then assert actualFour != four\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + public static final String DESCRIPTION = "This is my Scenario description."; + public static final String DESCRIPTION_ERROR_LOG_WITH_DESCRIPTION = String.format( + MARKDOWN_DELIMITER_PATTERN, + DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + private static final String TEST_FEATURE = "classpath:feature/simple_failed_description.feature"; + private final String launchUuid = CommonUtils.namedId("launch_"); + private final String featureId = CommonUtils.namedId("feature_"); + private final String scenarioId = CommonUtils.namedId("scenario_"); + private final List stepIds = Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList()); + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, launchUuid, featureId, scenarioId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_description_in_description(boolean report) { + Results results; + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getItemUuid(), oneOf(stepIds.toArray(new String[0]))); + assertThat(log.getLaunchUuid(), equalTo(launchUuid)); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE) + ); + + ArgumentCaptor scenarioCaptorFinish = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client, times(1)).finishTestItem(same(scenarioId), scenarioCaptorFinish.capture()); + + List scenarios = scenarioCaptorFinish.getAllValues(); + FinishTestItemRQ scenario = scenarios.get(0); + + assertThat(scenario.getDescription(), allOf(notNullValue(), equalTo(DESCRIPTION_ERROR_LOG_WITH_DESCRIPTION))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java new file mode 100644 index 0000000..9b98368 --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java @@ -0,0 +1,123 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithExamplesTest { + + public static final String ERROR = "did not evaluate to 'true': mathResult == 5\nclasspath:feature/simple_failed_examples.feature:5"; + public static final String ERROR_MESSAGE = "Then assert mathResult == 5\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + private static final String EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN = + "Parameters:\n\n" + MarkdownUtils.TABLE_INDENT + "| vara | varb | result |\n" + MarkdownUtils.TABLE_INDENT + + "|------|------|--------|\n" + MarkdownUtils.TABLE_INDENT; + public static final String FIRST_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  2   |  2   |   4    |"; + public static final String SECOND_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  1   |  2   |   5    |"; + + public static final String SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN, + SECOND_EXAMPLE_DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + + private static final String TEST_FEATURE = "classpath:feature/simple_failed_examples.feature"; + private final String featureId = CommonUtils.namedId("feature_"); + private final List exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList()); + private final List>> stepIds = exampleIds.stream() + .map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(2).collect(Collectors.toList()))) + .collect(Collectors.toList()); + + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, null, featureId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_examples_in_description(boolean report) { + Results results; + + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(0)), scenarioCaptor.capture()); + verify(client).finishTestItem(same(exampleIds.get(1)), scenarioCaptor.capture()); + + List> stepCaptors = new ArrayList<>(Collections.nCopies(stepIds.size(), ArgumentCaptor.forClass(FinishTestItemRQ.class))); + stepIds.forEach(pair -> pair.getValue().forEach(id -> verify(client).finishTestItem(same(id), stepCaptors.get(0).capture()))); + + FinishTestItemRQ firstScenarioRq = scenarioCaptor.getAllValues().get(0); + assertThat(firstScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.PASSED.name()))); + assertThat(firstScenarioRq.getDescription(), allOf(notNullValue(), equalTo(FIRST_EXAMPLE_DESCRIPTION))); + + FinishTestItemRQ secondScenarioRq = scenarioCaptor.getAllValues().get(1); + assertThat(secondScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name()))); + assertThat(secondScenarioRq.getDescription(), allOf(notNullValue(), equalTo(SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG))); + } +} diff --git a/src/test/resources/feature/simple_failed_description.feature b/src/test/resources/feature/simple_failed_description.feature new file mode 100644 index 0000000..821f408 --- /dev/null +++ b/src/test/resources/feature/simple_failed_description.feature @@ -0,0 +1,9 @@ +Feature: the test to show last error log in scenario description and with actual description + + Scenario: Verify math + + This is my Scenario description. + + Given def four = 4 + When def actualFour = 2 * 2 + Then assert actualFour != four diff --git a/src/test/resources/feature/simple_failed_description_examples.feature b/src/test/resources/feature/simple_failed_description_examples.feature new file mode 100644 index 0000000..41743fd --- /dev/null +++ b/src/test/resources/feature/simple_failed_description_examples.feature @@ -0,0 +1,13 @@ +Feature: the test to show last error log in scenario description and with actual description and examples + + Scenario Outline: Verify math + + This is my Scenario description. + + Given def mathResult = + + Then assert mathResult == + + Examples: + | vara | varb | result | + | 2 | 2 | 4 | + | 1 | 2 | 5 | diff --git a/src/test/resources/feature/simple_failed_examples.feature b/src/test/resources/feature/simple_failed_examples.feature new file mode 100644 index 0000000..0cb4003 --- /dev/null +++ b/src/test/resources/feature/simple_failed_examples.feature @@ -0,0 +1,10 @@ +Feature: the test to show last error log in scenario description and with examples + + Scenario Outline: Verify different maths + Given def mathResult = + + Then assert mathResult == + + Examples: + | vara | varb | result | + | 2 | 2 | 4 | + | 1 | 2 | 5 | \ No newline at end of file