From 129ef24b18ba49a91dc8b6f21eea9248640ecdf1 Mon Sep 17 00:00:00 2001 From: Vitalii Rymar Date: Wed, 28 Feb 2024 16:22:33 +0200 Subject: [PATCH 1/5] Implemented new feature: display last error log in scenario description Implemented unit tests for the new feature Improved dependencies references in build.gradle --- CHANGELOG.md | 7 + build.gradle | 12 +- gradle.properties | 6 + .../karate/KarateReportPortalRunner.java | 30 ++++ .../reportportal/karate/ReportPortalHook.java | 17 ++- .../karate/ReportPortalPublisher.java | 7 +- .../karate/ReportPortalUtils.java | 62 ++++++--- .../karate/description/NoDescriptionTest.java | 2 +- .../ScenarioDescriptionErrorLogTest.java | 95 +++++++++++++ ...rrorLogWithDescriptionAndExamplesTest.java | 131 ++++++++++++++++++ ...escriptionErrorLogWithDescriptionTest.java | 104 ++++++++++++++ ...ioDescriptionErrorLogWithExamplesTest.java | 123 ++++++++++++++++ .../feature/simple_failed_description.feature | 9 ++ ...simple_failed_description_examples.feature | 13 ++ .../feature/simple_failed_examples.feature | 10 ++ 15 files changed, 592 insertions(+), 36 deletions(-) create mode 100644 src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java create mode 100644 src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java create mode 100644 src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java create mode 100644 src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java create mode 100644 src/test/resources/feature/simple_failed_description.feature create mode 100644 src/test/resources/feature/simple_failed_description_examples.feature create mode 100644 src/test/resources/feature/simple_failed_examples.feature diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e07e7..574f793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ - Karate dependency marked as `compileOnly` to force users specify their own version of Karate, by @HardNorth - Client version updated on [5.2.5](https://github.com/reportportal/client-java/releases/tag/5.2.5), by @HardNorth +## [5.1.0] +### Added +- Implemented new feature: display last error log in scenario description, by @vrymar +- Implemented unit tests for the new feature, by @vrymar +### Changed +- Improved dependencies references in build.gradle, by @vrymar + ## [5.0.0] ### Added - Basic Agent functionality, by @vrymar diff --git a/build.gradle b/build.gradle index 3d18514..0fcb32d 100644 --- a/build.gradle +++ b/build.gradle @@ -42,21 +42,21 @@ repositories { } dependencies { - api 'com.epam.reportportal:client-java:5.2.5' + api "com.epam.reportportal:client-java:${project.client_java_version}" compileOnly "com.intuit.karate:karate-core:${project.karate_version}" - implementation 'org.slf4j:slf4j-api:2.0.7' + implementation "org.slf4j:slf4j-api:${project.slf4j_api_version}" testImplementation "com.intuit.karate:karate-core:${project.karate_version}" - testImplementation 'com.epam.reportportal:logger-java-logback:5.2.1' - testImplementation 'com.epam.reportportal:agent-java-test-utils:0.0.3' + testImplementation "com.epam.reportportal:logger-java-logback:${project.logger_java_logback_version}" + testImplementation "com.epam.reportportal:agent-java-test-utils:${project.agent_java_test_utils_version}" testImplementation "org.junit.jupiter:junit-jupiter-api:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-engine:${project.junit_version}" testImplementation "org.junit.jupiter:junit-jupiter-params:${project.junit_version}" testImplementation "org.mockito:mockito-core:${project.mockito_version}" testImplementation "org.mockito:mockito-junit-jupiter:${project.mockito_version}" - testImplementation 'org.hamcrest:hamcrest-core:2.2' - testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' + testImplementation "org.hamcrest:hamcrest-core:${project.hamcrest_core_version}" + testImplementation "com.squareup.okhttp3:okhttp:${project.okhttp_version}" } test { diff --git a/gradle.properties b/gradle.properties index 206f8c7..1085675 100644 --- a/gradle.properties +++ b/gradle.properties @@ -5,6 +5,12 @@ gradle_version=8.2 karate_version=1.4.1 junit_version=5.10.1 mockito_version=5.4.0 +agent_java_test_utils_version=0.0.3 +client_java_version=5.2.5 +slf4j_api_version=2.0.7 +logger_java_logback_version=5.2.1 +hamcrest_core_version=2.2 +okhttp_version=4.12.0 scripts_url=https://github.com/raw/reportportal/gradle-scripts scripts_branch=develop excludeTests= diff --git a/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java b/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java index 19f97cc..3a6f7a3 100644 --- a/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java +++ b/src/main/java/com/epam/reportportal/karate/KarateReportPortalRunner.java @@ -20,25 +20,55 @@ import com.intuit.karate.Results; import com.intuit.karate.Runner; +/** + * Karate runner with ReportPortal integration + */ public class KarateReportPortalRunner { + /** + * Create a new builder for the Karate runner with ReportPortal integration + * + * @param paths paths to feature files + * @param type of the builder + * @return a new builder + */ public static > Builder path(String... paths) { Builder builder = new Builder<>(); return builder.path(paths); } + /** + * Builder for the Karate runner with ReportPortal integration + * + * @param type of the builder + */ public static class Builder> extends Runner.Builder { private ReportPortal rp; + /** + * Create a new builder + */ public Builder() { super(); } + /** + * Set the ReportPortal instance to use + * + * @param reportPortal the ReportPortal instance + * @return the builder + */ public Builder withReportPortal(ReportPortal reportPortal) { rp = reportPortal; return this; } + /** + * Run the tests in parallel + * + * @param threadCount number of threads to use + * @return the results of the tests + */ @Override public Results parallel(int threadCount) { if (rp == null) { diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java index 1cc9530..3def51a 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalHook.java @@ -62,6 +62,11 @@ public class ReportPortalHook implements RuntimeHook { private final Map, Date> stepStartTimeMap = new ConcurrentHashMap<>(); private volatile Thread shutDownHook; + /** + * Create a new instance of the ReportPortalHook with the specified ReportPortal instance. + * + * @param reportPortal the ReportPortal instance + */ public ReportPortalHook(ReportPortal reportPortal) { launch = new MemoizingSupplier<>(() -> { ListenerParameters params = reportPortal.getParameters(); @@ -74,10 +79,15 @@ public ReportPortalHook(ReportPortal reportPortal) { }); } + /** + * Default constructor. Create a new instance of the ReportPortalHook with default ReportPortal instance. + */ + @SuppressWarnings("unused") public ReportPortalHook() { this(ReportPortal.builder().build()); } + @SuppressWarnings("unused") public ReportPortalHook(Supplier launchSupplier) { launch = new MemoizingSupplier<>(launchSupplier); shutDownHook = registerShutdownHook(this::finishLaunch); @@ -187,7 +197,7 @@ public void afterFeature(FeatureRuntime fr) { */ @Nonnull protected StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioRuntime sr) { - return ReportPortalUtils.buildStartScenarioRq(sr.scenario); + return ReportPortalUtils.buildStartScenarioRq(sr.result); } @Override @@ -208,9 +218,7 @@ public boolean beforeScenario(ScenarioRuntime sr) { */ @Nonnull protected FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioRuntime sr) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), - sr.result.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED - ); + return ReportPortalUtils.buildFinishScenarioRq(sr.result); } /** @@ -231,6 +239,7 @@ protected StartTestItemRQ buildStartBackgroundRq(@Nonnull Step step, @Nonnull Sc * * @param step Karate's Step object instance * @param sr Karate's ScenarioRuntime object instance + * @return item ID future */ public Maybe startBackground(@Nonnull Step step, @Nonnull ScenarioRuntime sr) { return backgroundIdMap.computeIfAbsent(sr.scenario.getUniqueId(), k -> { diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java index 90d68e8..868ea9c 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalPublisher.java @@ -187,7 +187,7 @@ public void finishFeature(FeatureResult featureResult) { */ @Nonnull protected StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioResult scenarioResult) { - return ReportPortalUtils.buildStartScenarioRq(scenarioResult.getScenario()); + return ReportPortalUtils.buildStartScenarioRq(scenarioResult); } /** @@ -211,10 +211,7 @@ public void startScenario(ScenarioResult scenarioResult, FeatureResult featureRe */ @Nonnull protected FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioResult scenarioResult) { - return buildFinishTestItemRq(Calendar.getInstance().getTime(), - scenarioResult.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED - ); - + return ReportPortalUtils.buildFinishScenarioRq(scenarioResult); } /** diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java index ed5da64..2735cc7 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java @@ -60,7 +60,8 @@ public class ReportPortalUtils { public static final String SKIPPED_ISSUE_KEY = "skippedIssue"; public static final String SCENARIO_CODE_REFERENCE_PATTERN = "%s/[SCENARIO:%s]"; public static final String EXAMPLE_CODE_REFERENCE_PATTERN = "%s/[EXAMPLE:%s%s]"; - public static final String MARKDOWN_DELIMITER_PATTERN = "%s\n\n---\n\n%s"; + public static final String MARKDOWN_DELIMITER = "\n\n---\n\n"; + public static final String MARKDOWN_DELIMITER_PATTERN = "%s" + MARKDOWN_DELIMITER + "%s"; private static final Logger LOGGER = LoggerFactory.getLogger(ReportPortalUtils.class); private static final String PARAMETER_ITEMS_START = "["; private static final String PARAMETER_ITEMS_END = "]"; @@ -294,35 +295,56 @@ public static TestCaseIdEntry getTestCaseId(@Nonnull Scenario scenario) { /** * Build ReportPortal request for start Scenario event * - * @param scenario Karate's Scenario object instance + * @param result Karate's ScenarioResult object instance * @return request to ReportPortal */ @Nonnull - public static StartTestItemRQ buildStartScenarioRq(@Nonnull Scenario scenario) { + public static StartTestItemRQ buildStartScenarioRq(@Nonnull ScenarioResult result) { + Scenario scenario = result.getScenario(); StartTestItemRQ rq = buildStartTestItemRq(scenario.getName(), Calendar.getInstance().getTime(), ItemType.STEP); rq.setCodeRef(getCodeRef(scenario)); rq.setTestCaseId(ofNullable(getTestCaseId(scenario)).map(TestCaseIdEntry::getId).orElse(null)); rq.setAttributes(toAttributes(scenario.getTags())); - List parameters = getParameters(scenario); - boolean hasParameters = ofNullable(parameters).filter(p -> !p.isEmpty()).isPresent(); - if (hasParameters) { - rq.setParameters(parameters); + rq.setParameters(getParameters(scenario)); + rq.setDescription(buildDescription(scenario, result.getErrorMessage(), getParameters(scenario))); + return rq; + } + + /** + * Build ReportPortal request for finish Scenario event + * + * @param result Karate's ScenarioResult object instance + * @return request to ReportPortal + */ + @Nonnull + public static FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioResult result) { + Scenario scenario = result.getScenario(); + FinishTestItemRQ rq = buildFinishTestItemRq(Calendar.getInstance().getTime(), result.getFailureMessageForDisplay() == null ? ItemStatus.PASSED : ItemStatus.FAILED); + rq.setDescription(buildDescription(scenario, result.getErrorMessage(), getParameters(scenario))); + return rq; + } + + private static String buildDescription(Scenario scenario, String errorMessage, List parameters) { + StringBuilder descriptionBuilder = new StringBuilder(); + + if (parameters != null && !parameters.isEmpty()) { + descriptionBuilder.append(String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters))); + } + if (isNotBlank(scenario.getDescription())) { + appendWithDelimiter(descriptionBuilder, scenario.getDescription()); + } + if (isNotBlank(errorMessage)) { + appendWithDelimiter(descriptionBuilder, String.format("Error:\n%s", errorMessage)); } - String description = scenario.getDescription(); - if (isNotBlank(description)) { - if (hasParameters) { - rq.setDescription(String.format(MARKDOWN_DELIMITER_PATTERN, - String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters)), - description - )); - } else { - rq.setDescription(description); - } - } else if (hasParameters) { - rq.setDescription(String.format(PARAMETERS_PATTERN, ParameterUtils.formatParametersAsTable(parameters))); + return descriptionBuilder.toString(); + } + + private static void appendWithDelimiter(StringBuilder builder, String text) { + if (builder.length() > 0) { + builder.append(MARKDOWN_DELIMITER); } - return rq; + builder.append(text); } /** diff --git a/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java b/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java index 8128ee4..b35ccd8 100644 --- a/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java +++ b/src/test/java/com/epam/reportportal/karate/description/NoDescriptionTest.java @@ -75,7 +75,7 @@ public void test_description_for_all_possible_items(boolean report) { assertThat(featureStart.getDescription(), endsWith("feature/simple.feature")); StartTestItemRQ scenarioStart = scenarioCaptor.getValue(); - assertThat(scenarioStart.getDescription(), nullValue()); + assertThat(scenarioStart.getDescription(), emptyString()); stepCaptor.getAllValues().forEach(step -> assertThat(step.getDescription(), nullValue())); } diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java new file mode 100644 index 0000000..909c68c --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogTest { + + public static final String ERROR = "did not evaluate to 'true': actualFour != four\nclasspath:feature/simple_failed.feature:6"; + public static final String ERROR_MESSAGE = "Then assert actualFour != four\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + private static final String TEST_FEATURE = "classpath:feature/simple_failed.feature"; + private final String launchUuid = CommonUtils.namedId("launch_"); + private final String featureId = CommonUtils.namedId("feature_"); + private final String scenarioId = CommonUtils.namedId("scenario_"); + private final List stepIds = Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList()); + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, launchUuid, featureId, scenarioId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_in_description(boolean report) { + Results results; + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getItemUuid(), oneOf(stepIds.toArray(new String[0]))); + assertThat(log.getLaunchUuid(), equalTo(launchUuid)); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptorFinish = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client, times(1)).finishTestItem(same(scenarioId), scenarioCaptorFinish.capture()); + + List scenarios = scenarioCaptorFinish.getAllValues(); + FinishTestItemRQ scenario = scenarios.get(0); + assertThat(scenario.getDescription(), allOf(notNullValue(), equalTo(DESCRIPTION_ERROR_LOG))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java new file mode 100644 index 0000000..a74d244 --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithDescriptionAndExamplesTest { + + public static final String MARKDOWN_DELIMITER_PATTERN_THREE_ARGS = "%s\n\n---\n\n%s\n\n---\n\n%s"; + public static final String ERROR = "did not evaluate to 'true': mathResult == 5\nclasspath:feature/simple_failed_description_examples.feature:8"; + public static final String ERROR_MESSAGE = "Then assert mathResult == 5\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + public static final String DESCRIPTION = "This is my Scenario description."; + private static final String EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN = + "Parameters:\n\n" + MarkdownUtils.TABLE_INDENT + "| vara | varb | result |\n" + MarkdownUtils.TABLE_INDENT + + "|------|------|--------|\n" + MarkdownUtils.TABLE_INDENT; + public static final String FIRST_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  2   |  2   |   4    |"; + public static final String SECOND_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  1   |  2   |   5    |"; + + public static final String FIRST_EXAMPLE_DESCRIPTION_WITHOUT_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN, + FIRST_EXAMPLE_DESCRIPTION, + DESCRIPTION + ); + public static final String SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN_THREE_ARGS, + SECOND_EXAMPLE_DESCRIPTION, + DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + + private static final String TEST_FEATURE = "classpath:feature/simple_failed_description_examples.feature"; + private final String featureId = CommonUtils.namedId("feature_"); + private final List exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList()); + private final List>> stepIds = exampleIds.stream() + .map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(2).collect(Collectors.toList()))) + .collect(Collectors.toList()); + + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, null, featureId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_examples_in_description(boolean report) { + Results results; + + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(0)), scenarioCaptor.capture()); + verify(client).finishTestItem(same(exampleIds.get(1)), scenarioCaptor.capture()); + + List> stepCaptors = new ArrayList<>(Collections.nCopies(stepIds.size(), ArgumentCaptor.forClass(FinishTestItemRQ.class))); + stepIds.forEach(pair -> pair.getValue().forEach(id -> verify(client).finishTestItem(same(id), stepCaptors.get(0).capture()))); + + FinishTestItemRQ firstScenarioRq = scenarioCaptor.getAllValues().get(0); + assertThat(firstScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.PASSED.name()))); + assertThat(firstScenarioRq.getDescription(), allOf(notNullValue(), equalTo(FIRST_EXAMPLE_DESCRIPTION_WITHOUT_ERROR_LOG))); + + FinishTestItemRQ secondScenarioRq = scenarioCaptor.getAllValues().get(1); + assertThat(secondScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name()))); + assertThat(secondScenarioRq.getDescription(), allOf(notNullValue(), equalTo(SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java new file mode 100644 index 0000000..454066c --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithDescriptionTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithDescriptionTest { + + public static final String ERROR = "did not evaluate to 'true': actualFour != four\nclasspath:feature/simple_failed_description.feature:9"; + public static final String ERROR_MESSAGE = "Then assert actualFour != four\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + public static final String DESCRIPTION = "This is my Scenario description."; + public static final String DESCRIPTION_ERROR_LOG_WITH_DESCRIPTION = String.format( + MARKDOWN_DELIMITER_PATTERN, + DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + private static final String TEST_FEATURE = "classpath:feature/simple_failed_description.feature"; + private final String launchUuid = CommonUtils.namedId("launch_"); + private final String featureId = CommonUtils.namedId("feature_"); + private final String scenarioId = CommonUtils.namedId("scenario_"); + private final List stepIds = Stream.generate(() -> CommonUtils.namedId("step_")).limit(3).collect(Collectors.toList()); + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, launchUuid, featureId, scenarioId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_description_in_description(boolean report) { + Results results; + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getItemUuid(), oneOf(stepIds.toArray(new String[0]))); + assertThat(log.getLaunchUuid(), equalTo(launchUuid)); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE) + ); + + ArgumentCaptor scenarioCaptorFinish = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client, times(1)).finishTestItem(same(scenarioId), scenarioCaptorFinish.capture()); + + List scenarios = scenarioCaptorFinish.getAllValues(); + FinishTestItemRQ scenario = scenarios.get(0); + + assertThat(scenario.getDescription(), allOf(notNullValue(), equalTo(DESCRIPTION_ERROR_LOG_WITH_DESCRIPTION))); + } +} diff --git a/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java new file mode 100644 index 0000000..9b98368 --- /dev/null +++ b/src/test/java/com/epam/reportportal/karate/description/ScenarioDescriptionErrorLogWithExamplesTest.java @@ -0,0 +1,123 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.karate.description; + +import com.epam.reportportal.karate.utils.TestUtils; +import com.epam.reportportal.listeners.ItemStatus; +import com.epam.reportportal.listeners.LogLevel; +import com.epam.reportportal.service.ReportPortal; +import com.epam.reportportal.service.ReportPortalClient; +import com.epam.reportportal.util.test.CommonUtils; +import com.epam.reportportal.utils.markdown.MarkdownUtils; +import com.epam.ta.reportportal.ws.model.FinishTestItemRQ; +import com.epam.ta.reportportal.ws.model.log.SaveLogRQ; +import com.intuit.karate.Results; +import okhttp3.MultipartBody; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.epam.reportportal.karate.ReportPortalUtils.MARKDOWN_DELIMITER_PATTERN; +import static com.epam.reportportal.karate.utils.TestUtils.*; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +public class ScenarioDescriptionErrorLogWithExamplesTest { + + public static final String ERROR = "did not evaluate to 'true': mathResult == 5\nclasspath:feature/simple_failed_examples.feature:5"; + public static final String ERROR_MESSAGE = "Then assert mathResult == 5\n" + ERROR; + public static final String DESCRIPTION_ERROR_LOG = "Error:\n" + ERROR; + private static final String EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN = + "Parameters:\n\n" + MarkdownUtils.TABLE_INDENT + "| vara | varb | result |\n" + MarkdownUtils.TABLE_INDENT + + "|------|------|--------|\n" + MarkdownUtils.TABLE_INDENT; + public static final String FIRST_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  2   |  2   |   4    |"; + public static final String SECOND_EXAMPLE_DESCRIPTION = EXAMPLE_PARAMETERS_DESCRIPTION_PATTERN + "|  1   |  2   |   5    |"; + + public static final String SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG = String.format( + MARKDOWN_DELIMITER_PATTERN, + SECOND_EXAMPLE_DESCRIPTION, + DESCRIPTION_ERROR_LOG + ); + + private static final String TEST_FEATURE = "classpath:feature/simple_failed_examples.feature"; + private final String featureId = CommonUtils.namedId("feature_"); + private final List exampleIds = Stream.generate(() -> CommonUtils.namedId("example_")).limit(2).collect(Collectors.toList()); + private final List>> stepIds = exampleIds.stream() + .map(e -> Pair.of(e, Stream.generate(() -> CommonUtils.namedId("step_")).limit(2).collect(Collectors.toList()))) + .collect(Collectors.toList()); + + private final ReportPortalClient client = mock(ReportPortalClient.class); + private final ReportPortal rp = ReportPortal.create(client, standardParameters(), testExecutor()); + + @BeforeEach + public void setupMock() { + mockLaunch(client, null, featureId, stepIds); + mockBatchLogging(client); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void test_error_log_and_examples_in_description(boolean report) { + Results results; + + if (report) { + results = TestUtils.runAsReport(rp, TEST_FEATURE); + } else { + results = TestUtils.runAsHook(rp, TEST_FEATURE); + } + + assertThat(results.getFailCount(), equalTo(1)); + + @SuppressWarnings("unchecked") + ArgumentCaptor> logCaptor = ArgumentCaptor.forClass(List.class); + verify(client, atLeastOnce()).log(logCaptor.capture()); + List logs = logCaptor.getAllValues() + .stream() + .flatMap(rq -> extractJsonParts(rq).stream()) + .filter(rq -> LogLevel.ERROR.name().equals(rq.getLevel())) + .collect(Collectors.toList()); + + assertThat(logs, hasSize(greaterThan(0))); + SaveLogRQ log = logs.get(logs.size() - 1); + assertThat(log.getMessage(), equalTo(ERROR_MESSAGE)); + + ArgumentCaptor scenarioCaptor = ArgumentCaptor.forClass(FinishTestItemRQ.class); + verify(client).finishTestItem(same(exampleIds.get(0)), scenarioCaptor.capture()); + verify(client).finishTestItem(same(exampleIds.get(1)), scenarioCaptor.capture()); + + List> stepCaptors = new ArrayList<>(Collections.nCopies(stepIds.size(), ArgumentCaptor.forClass(FinishTestItemRQ.class))); + stepIds.forEach(pair -> pair.getValue().forEach(id -> verify(client).finishTestItem(same(id), stepCaptors.get(0).capture()))); + + FinishTestItemRQ firstScenarioRq = scenarioCaptor.getAllValues().get(0); + assertThat(firstScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.PASSED.name()))); + assertThat(firstScenarioRq.getDescription(), allOf(notNullValue(), equalTo(FIRST_EXAMPLE_DESCRIPTION))); + + FinishTestItemRQ secondScenarioRq = scenarioCaptor.getAllValues().get(1); + assertThat(secondScenarioRq.getStatus(), allOf(notNullValue(), equalTo(ItemStatus.FAILED.name()))); + assertThat(secondScenarioRq.getDescription(), allOf(notNullValue(), equalTo(SECOND_EXAMPLE_DESCRIPTION_WITH_ERROR_LOG))); + } +} diff --git a/src/test/resources/feature/simple_failed_description.feature b/src/test/resources/feature/simple_failed_description.feature new file mode 100644 index 0000000..821f408 --- /dev/null +++ b/src/test/resources/feature/simple_failed_description.feature @@ -0,0 +1,9 @@ +Feature: the test to show last error log in scenario description and with actual description + + Scenario: Verify math + + This is my Scenario description. + + Given def four = 4 + When def actualFour = 2 * 2 + Then assert actualFour != four diff --git a/src/test/resources/feature/simple_failed_description_examples.feature b/src/test/resources/feature/simple_failed_description_examples.feature new file mode 100644 index 0000000..41743fd --- /dev/null +++ b/src/test/resources/feature/simple_failed_description_examples.feature @@ -0,0 +1,13 @@ +Feature: the test to show last error log in scenario description and with actual description and examples + + Scenario Outline: Verify math + + This is my Scenario description. + + Given def mathResult = + + Then assert mathResult == + + Examples: + | vara | varb | result | + | 2 | 2 | 4 | + | 1 | 2 | 5 | diff --git a/src/test/resources/feature/simple_failed_examples.feature b/src/test/resources/feature/simple_failed_examples.feature new file mode 100644 index 0000000..0cb4003 --- /dev/null +++ b/src/test/resources/feature/simple_failed_examples.feature @@ -0,0 +1,10 @@ +Feature: the test to show last error log in scenario description and with examples + + Scenario Outline: Verify different maths + Given def mathResult = + + Then assert mathResult == + + Examples: + | vara | varb | result | + | 2 | 2 | 4 | + | 1 | 2 | 5 | \ No newline at end of file From 62e57be2610f53dc3e28ecd0553b4d9afa0f53b1 Mon Sep 17 00:00:00 2001 From: Vitalii Rymar Date: Mon, 4 Mar 2024 18:31:47 +0200 Subject: [PATCH 2/5] Edited CHANGELOG.md --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d36de0..222b7d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,6 @@ ### Fixed - Issue [#23](https://github.com/reportportal/agent-java-karate/issues/23) scenarios outside features in parallel execution, by @HardNorth -## [5.0.1] -### Changed -- Karate dependency marked as `compileOnly` to force users specify their own version of Karate, by @HardNorth -- Client version updated on [5.2.5](https://github.com/reportportal/client-java/releases/tag/5.2.5), by @HardNorth -### Fixed -- Issue [#23](https://github.com/reportportal/agent-java-karate/issues/23) scenarios outside features in parallel execution, by @HardNorth - ## [5.1.0] ### Added - Implemented new feature: display last error log in scenario description, by @vrymar @@ -21,6 +14,13 @@ ### Changed - Improved dependencies references in build.gradle, by @vrymar +## [5.0.1] +### Changed +- Karate dependency marked as `compileOnly` to force users specify their own version of Karate, by @HardNorth +- Client version updated on [5.2.5](https://github.com/reportportal/client-java/releases/tag/5.2.5), by @HardNorth +### Fixed +- Issue [#23](https://github.com/reportportal/agent-java-karate/issues/23) scenarios outside features in parallel execution, by @HardNorth + ## [5.0.0] ### Added - Basic Agent functionality, by @vrymar From 7710c7fe5fbd37272ab7b1061543a27f9ce40581 Mon Sep 17 00:00:00 2001 From: Vitalii Rymar Date: Tue, 5 Mar 2024 18:04:30 +0200 Subject: [PATCH 3/5] Merged with develop branch --- CHANGELOG.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 915e35d..4f51492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ ### Removed - Shutdown hook register on supplied Launch, by @HardNorth +## [5.0.4] +### Added +- Implemented new feature: display last error log in scenario description, by @vrymar +- Implemented unit tests for the new feature, by @vrymar +### Changed +- Improved dependencies references in build.gradle, by @vrymar + ## [5.0.3] ### Fixed - Backgrounds finish with `FAILED` status, by @HardNorth @@ -15,13 +22,6 @@ ### Fixed - Issue [#23](https://github.com/reportportal/agent-java-karate/issues/23) scenarios outside features in parallel execution, by @HardNorth -## [5.1.0] -### Added -- Implemented new feature: display last error log in scenario description, by @vrymar -- Implemented unit tests for the new feature, by @vrymar -### Changed -- Improved dependencies references in build.gradle, by @vrymar - ## [5.0.1] ### Changed - Karate dependency marked as `compileOnly` to force users specify their own version of Karate, by @HardNorth From b7974bfdb821b35ad5ff6f609226617f672f3baf Mon Sep 17 00:00:00 2001 From: Vitalii Rymar Date: Wed, 6 Mar 2024 14:46:59 +0200 Subject: [PATCH 4/5] Solved issues after code review --- CHANGELOG.md | 4 ---- .../java/com/epam/reportportal/karate/ReportPortalUtils.java | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f51492..6b90028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,6 @@ # Changelog ## [Unreleased] -### Removed -- Shutdown hook register on supplied Launch, by @HardNorth - -## [5.0.4] ### Added - Implemented new feature: display last error log in scenario description, by @vrymar - Implemented unit tests for the new feature, by @vrymar diff --git a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java index 2735cc7..a97231b 100644 --- a/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java +++ b/src/main/java/com/epam/reportportal/karate/ReportPortalUtils.java @@ -324,7 +324,8 @@ public static FinishTestItemRQ buildFinishScenarioRq(@Nonnull ScenarioResult res return rq; } - private static String buildDescription(Scenario scenario, String errorMessage, List parameters) { + @Nonnull + private static String buildDescription(@Nonnull Scenario scenario, @Nullable String errorMessage, @Nullable List parameters) { StringBuilder descriptionBuilder = new StringBuilder(); if (parameters != null && !parameters.isEmpty()) { From 7c84d051bfb192ae2a8ec72cff8878d6e0929acf Mon Sep 17 00:00:00 2001 From: Vitalii Rymar Date: Wed, 6 Mar 2024 17:54:45 +0200 Subject: [PATCH 5/5] Solved issues after code review --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b90028..4ed6807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Removed +- Shutdown hook register on supplied Launch, by @HardNorth ### Added - Implemented new feature: display last error log in scenario description, by @vrymar - Implemented unit tests for the new feature, by @vrymar