diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy index 52f225f11794a..88060ebc0837c 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy @@ -24,6 +24,7 @@ import org.elasticsearch.gradle.NoticeTask import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.test.rest.RestResourcesPlugin import org.elasticsearch.gradle.test.RestIntegTestTask import org.elasticsearch.gradle.testclusters.RunTask import org.elasticsearch.gradle.testclusters.TestClustersPlugin @@ -51,6 +52,7 @@ class PluginBuildPlugin implements Plugin { void apply(Project project) { project.pluginManager.apply(BuildPlugin) project.pluginManager.apply(TestClustersPlugin) + project.pluginManager.apply(RestResourcesPlugin) PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project) configureDependencies(project) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy index 58e72e2dd7be4..8d9517fe42629 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy @@ -18,18 +18,12 @@ */ package org.elasticsearch.gradle.test -import org.elasticsearch.gradle.VersionProperties -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.ElasticsearchCluster import org.elasticsearch.gradle.testclusters.RestTestRunnerTask -import org.elasticsearch.gradle.tool.Boilerplate import org.gradle.api.DefaultTask import org.gradle.api.Task -import org.gradle.api.file.FileCopyDetails -import org.gradle.api.tasks.Copy -import org.gradle.api.tasks.Input import org.gradle.api.tasks.testing.Test -import org.gradle.plugins.ide.idea.IdeaPlugin + /** * A wrapper task around setting up a cluster and running rest tests. */ @@ -37,10 +31,6 @@ class RestIntegTestTask extends DefaultTask { protected Test runner - /** Flag indicating whether the rest tests in the rest spec should be run. */ - @Input - Boolean includePackaged = false - RestIntegTestTask() { runner = project.tasks.create("${name}Runner", RestTestRunnerTask.class) super.dependsOn(runner) @@ -69,10 +59,6 @@ class RestIntegTestTask extends DefaultTask { runner.systemProperty('test.clustername', System.getProperty("tests.clustername")) } - // copy the rest spec/tests onto the test classpath - Copy copyRestSpec = createCopyRestSpecTask() - project.sourceSets.test.output.builtBy(copyRestSpec) - // this must run after all projects have been configured, so we know any project // references can be accessed as a fully configured project.gradle.projectsEvaluated { @@ -83,12 +69,6 @@ class RestIntegTestTask extends DefaultTask { } } - /** Sets the includePackaged property */ - public void includePackaged(boolean include) { - includePackaged = include - } - - @Override public Task dependsOn(Object... dependencies) { runner.dependsOn(dependencies) @@ -114,37 +94,4 @@ class RestIntegTestTask extends DefaultTask { project.tasks.getByName("${name}Runner").configure(configure) } - Copy createCopyRestSpecTask() { - Boilerplate.maybeCreate(project.configurations, 'restSpec') { - project.dependencies.add( - 'restSpec', - BuildParams.internal ? project.project(':rest-api-spec') : - "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}" - ) - } - - return Boilerplate.maybeCreate(project.tasks, 'copyRestSpec', Copy) { Copy copy -> - copy.dependsOn project.configurations.restSpec - copy.into(project.sourceSets.test.output.resourcesDir) - copy.from({ project.zipTree(project.configurations.restSpec.singleFile) }) { - includeEmptyDirs = false - include 'rest-api-spec/**' - filesMatching('rest-api-spec/test/**') { FileCopyDetails details -> - if (includePackaged == false) { - details.exclude() - } - } - } - - if (project.plugins.hasPlugin(IdeaPlugin)) { - project.idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) - } - } - } - } - } - } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index b3d323bf3942f..d668aa4b6b7fa 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -26,9 +26,9 @@ import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.elasticsearch.gradle.test.rest.RestResourcesPlugin import org.elasticsearch.gradle.testclusters.TestClustersPlugin import org.gradle.api.InvalidUserDataException -import org.gradle.api.JavaVersion import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.artifacts.Configuration @@ -42,6 +42,7 @@ import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.testing.Test import org.gradle.plugins.ide.eclipse.model.EclipseModel import org.gradle.plugins.ide.idea.model.IdeaModel + /** * Configures the build to compile tests against Elasticsearch's test framework * and run REST tests. Use BuildPlugin if you want to build main code as well @@ -74,6 +75,8 @@ class StandaloneRestTestPlugin implements Plugin { // only setup tests to build SourceSetContainer sourceSets = project.extensions.getByType(SourceSetContainer) SourceSet testSourceSet = sourceSets.create('test') + // need to apply plugin after test source sets are created + project.pluginManager.apply(RestResourcesPlugin) project.tasks.withType(Test) { Test test -> test.testClassesDirs = testSourceSet.output.classesDirs diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java index 5b16ca7d40f1e..a662ff4dbcaeb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java @@ -250,7 +250,7 @@ public void doCheck() throws IOException { Files.write(getSuccessMarker().toPath(), new byte[] {}, StandardOpenOption.CREATE); } else { getLogger().error(problems); - throw new IllegalStateException("Testing conventions are not honored"); + throw new IllegalStateException(String.format("Testing conventions [%s] are not honored", problems)); } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java new file mode 100644 index 0000000000000..6e66df54dc48d --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.gradle.api.DefaultTask; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.util.PatternFilterable; +import org.gradle.api.tasks.util.PatternSet; +import org.gradle.internal.Factory; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Copies the files needed for the Rest YAML specs to the current projects test resources output directory. + * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed + * configurations and custom extensions. + * @see RestResourcesPlugin + */ +public class CopyRestApiTask extends DefaultTask { + private static final String COPY_TO = "rest-api-spec/api"; + final ListProperty includeCore = getProject().getObjects().listProperty(String.class); + final ListProperty includeXpack = getProject().getObjects().listProperty(String.class); + + Configuration coreConfig; + Configuration xpackConfig; + + private final PatternFilterable corePatternSet; + private final PatternFilterable xpackPatternSet; + + public CopyRestApiTask() { + corePatternSet = getPatternSetFactory().create(); + xpackPatternSet = getPatternSetFactory().create(); + } + + @Inject + protected Factory getPatternSetFactory() { + throw new UnsupportedOperationException(); + } + + @Input + public ListProperty getIncludeCore() { + return includeCore; + } + + @Input + public ListProperty getIncludeXpack() { + return includeXpack; + } + + @SkipWhenEmpty + @InputFiles + public FileTree getInputDir() { + xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + ConfigurableFileCollection fileCollection = getProject().files(xpackConfig.getAsFileTree().matching(xpackPatternSet)); + if (BuildParams.isInternal()) { + corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + fileCollection.plus(coreConfig.getAsFileTree().matching(corePatternSet)); + } else { + fileCollection.plus(coreConfig); + } + // if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input + return projectHasYamlRestTests() || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false + ? fileCollection.getAsFileTree() + : null; + } + + @OutputDirectory + public File getOutputDir() { + return new File(getTestSourceSet().getOutput().getResourcesDir(), COPY_TO); + } + + @TaskAction + void copy() { + Project project = getProject(); + // always copy the core specs if the task executes + if (BuildParams.isInternal()) { + getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", project.getPath()); + project.copy(c -> { + c.from(coreConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(corePatternSet.getIncludes()); + }); + } else { + getLogger().debug( + "Rest specs for project [{}] will be copied to the test resources from the published jar (version: [{}]).", + project.getPath(), + VersionProperties.getElasticsearch() + ); + project.copy(c -> { + c.from(project.zipTree(coreConfig.getSingleFile())); + c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir + c.include(includeCore.get().stream().map(prefix -> COPY_TO + "/" + prefix + "*/**").collect(Collectors.toList())); + }); + } + // only copy x-pack specs if explicitly instructed + if (includeXpack.get().isEmpty() == false) { + getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", project.getPath()); + project.copy(c -> { + c.from(xpackConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(xpackPatternSet.getIncludes()); + }); + } + } + + /** + * Returns true if any files with a .yml extension exist the test resources rest-api-spec/test directory (from source or output dir) + */ + private boolean projectHasYamlRestTests() { + File testSourceResourceDir = getTestSourceResourceDir(); + File testOutputResourceDir = getTestOutputResourceDir(); // check output for cases where tests are copied programmatically + + if (testSourceResourceDir == null && testOutputResourceDir == null) { + return false; + } + try { + if (testSourceResourceDir != null) { + return new File(testSourceResourceDir, "rest-api-spec/test").exists() == false + || Files.walk(testSourceResourceDir.toPath().resolve("rest-api-spec/test")) + .anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + if (testOutputResourceDir != null) { + return new File(testOutputResourceDir, "rest-api-spec/test").exists() == false + || Files.walk(testOutputResourceDir.toPath().resolve("rest-api-spec/test")) + .anyMatch(p -> p.getFileName().toString().endsWith("yml")); + } + } catch (IOException e) { + throw new IllegalStateException(String.format("Error determining if this project [%s] has rest tests.", getProject()), e); + } + return false; + } + + private File getTestSourceResourceDir() { + SourceSet testSources = getTestSourceSet(); + if (testSources == null) { + return null; + } + Set resourceDir = testSources.getResources() + .getSrcDirs() + .stream() + .filter(f -> f.isDirectory() && f.getParentFile().getName().equals("test") && f.getName().equals("resources")) + .collect(Collectors.toSet()); + assert resourceDir.size() <= 1; + if (resourceDir.size() == 0) { + return null; + } + return resourceDir.iterator().next(); + } + + private File getTestOutputResourceDir() { + SourceSet testSources = getTestSourceSet(); + if (testSources == null) { + return null; + } + return testSources.getOutput().getResourcesDir(); + } + + private SourceSet getTestSourceSet() { + return Boilerplate.getJavaSourceSets(getProject()).findByName("test"); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java new file mode 100644 index 0000000000000..2fd5a207482a4 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestTestsTask.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.tool.Boilerplate; +import org.gradle.api.DefaultTask; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.SkipWhenEmpty; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.util.PatternFilterable; +import org.gradle.api.tasks.util.PatternSet; +import org.gradle.internal.Factory; + +import javax.inject.Inject; +import java.io.File; +import java.util.stream.Collectors; + +/** + * Copies the Rest YAML test to the current projects test resources output directory. + * This is intended to be be used from {@link RestResourcesPlugin} since the plugin wires up the needed + * configurations and custom extensions. + * @see RestResourcesPlugin + */ +public class CopyRestTestsTask extends DefaultTask { + private static final String COPY_TO = "rest-api-spec/test"; + final ListProperty includeCore = getProject().getObjects().listProperty(String.class); + final ListProperty includeXpack = getProject().getObjects().listProperty(String.class); + + Configuration coreConfig; + Configuration xpackConfig; + + private final PatternFilterable corePatternSet; + private final PatternFilterable xpackPatternSet; + + public CopyRestTestsTask() { + corePatternSet = getPatternSetFactory().create(); + xpackPatternSet = getPatternSetFactory().create(); + } + + @Inject + protected Factory getPatternSetFactory() { + throw new UnsupportedOperationException(); + } + + @Input + public ListProperty getIncludeCore() { + return includeCore; + } + + @Input + public ListProperty getIncludeXpack() { + return includeXpack; + } + + @SkipWhenEmpty + @InputFiles + public FileTree getInputDir() { + xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + ConfigurableFileCollection fileCollection = getProject().files(xpackConfig.getAsFileTree().matching(xpackPatternSet)); + if (BuildParams.isInternal()) { + corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList())); + fileCollection.plus(coreConfig.getAsFileTree().matching(corePatternSet)); + } else { + fileCollection.plus(coreConfig); + } + // copy tests only if explicitly requested + return includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false ? fileCollection.getAsFileTree() : null; + } + + @OutputDirectory + public File getOutputDir() { + return new File(getTestSourceSet().getOutput().getResourcesDir(), COPY_TO); + } + + @TaskAction + void copy() { + Project project = getProject(); + // only copy core tests if explicitly instructed + if (includeCore.get().isEmpty() == false) { + if (BuildParams.isInternal()) { + getLogger().debug("Rest tests for project [{}] will be copied to the test resources.", project.getPath()); + project.copy(c -> { + c.from(coreConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(corePatternSet.getIncludes()); + }); + + } else { + getLogger().debug( + "Rest tests for project [{}] will be copied to the test resources from the published jar (version: [{}]).", + project.getPath(), + VersionProperties.getElasticsearch() + ); + project.copy(c -> { + c.from(project.zipTree(coreConfig.getSingleFile())); + c.into(getTestSourceSet().getOutput().getResourcesDir()); // this ends up as the same dir as outputDir + c.include(includeCore.get().stream().map(prefix -> COPY_TO + "/" + prefix + "*/**").collect(Collectors.toList())); + }); + } + } + // only copy x-pack tests if explicitly instructed + if (includeXpack.get().isEmpty() == false) { + getLogger().debug("X-pack rest tests for project [{}] will be copied to the test resources.", project.getPath()); + project.copy(c -> { + c.from(xpackConfig.getSingleFile()); + c.into(getOutputDir()); + c.include(xpackPatternSet.getIncludes()); + }); + } + } + + private SourceSet getTestSourceSet() { + return Boilerplate.getJavaSourceSets(getProject()).findByName("test"); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java new file mode 100644 index 0000000000000..2865963bdb3bd --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.info.BuildParams; +import org.gradle.api.Action; +import org.gradle.api.model.ObjectFactory; +import org.gradle.api.provider.ListProperty; + +import javax.inject.Inject; + +/** + * Custom extension to configure the {@link CopyRestApiTask} + */ +public class RestResourcesExtension { + + final RestResourcesSpec restApi; + final RestResourcesSpec restTests; + + @Inject + public RestResourcesExtension(ObjectFactory objects) { + restApi = new RestResourcesSpec(objects); + restTests = new RestResourcesSpec(objects); + } + + void restApi(Action spec) { + spec.execute(restApi); + } + + void restTests(Action spec) { + spec.execute(restTests); + } + + static class RestResourcesSpec { + + private final ListProperty includeCore; + private final ListProperty includeXpack; + + RestResourcesSpec(ObjectFactory objects) { + includeCore = objects.listProperty(String.class); + includeXpack = objects.listProperty(String.class); + } + + public void includeCore(String... include) { + this.includeCore.addAll(include); + } + + public void includeXpack(String... include) { + if (BuildParams.isInternal() == false) { + throw new IllegalStateException("Can not include x-pack rest resources from an external build."); + } + this.includeXpack.addAll(include); + } + + public ListProperty getIncludeCore() { + return includeCore; + } + + public ListProperty getIncludeXpack() { + return includeXpack; + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java new file mode 100644 index 0000000000000..a512b9b1fc025 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesPlugin.java @@ -0,0 +1,138 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.gradle.test.rest; + +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.info.BuildParams; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.provider.Provider; + +import java.util.Map; + +/** + *

+ * Gradle plugin to help configure {@link CopyRestApiTask}'s and {@link CopyRestTestsTask} that copies the artifacts needed for the Rest API + * spec and YAML based rest tests. + *

+ * Rest API specification:
+ * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestApiTask} will automatically copy the core Rest API specification + * if there are any Rest YAML tests present in source, or copied from {@link CopyRestTestsTask} output. X-pack specs must be explicitly + * declared to be copied. + *
+ * For example: + * 
+ * restResources {
+ *   restApi {
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ *
+ * Will copy the entire core Rest API specifications (assuming the project has tests) and any of the the X-pack specs starting with enrich*. + * It is recommended (but not required) to also explicitly declare which core specs your project depends on to help optimize the caching + * behavior. + * For example: + * 
+ * restResources {
+ *   restApi {
+ *     includeCore 'index', 'cat'
+ *     includeXpack 'enrich'
+ *   }
+ * }
+ *
+ *
+ * Rest YAML tests :
+ * When the {@link RestResourcesPlugin} has been applied the {@link CopyRestTestsTask} will copy the Rest YAML tests if explicitly + * configured with `includeCore` or `includeXpack` through the `restResources.restTests` extension. + * For example: + * 
+ * restResources {
+ *  restApi {
+ *      includeXpack 'graph'
+ *   }
+ *   restTests {
+ *     includeXpack 'graph'
+ *   }
+ * }
+ *
+ * Will copy any of the the x-pack tests that start with graph, and will copy the X-pack graph specification, as well as the full core + * Rest API specification. + * + * @see CopyRestApiTask + * @see CopyRestTestsTask + */ +public class RestResourcesPlugin implements Plugin { + + private static final String EXTENSION_NAME = "restResources"; + + @Override + public void apply(Project project) { + RestResourcesExtension extension = project.getExtensions().create(EXTENSION_NAME, RestResourcesExtension.class); + + Provider copyRestYamlTestTask = project.getTasks() + .register("copyYamlTestsTask", CopyRestTestsTask.class, task -> { + task.includeCore.set(extension.restTests.getIncludeCore()); + task.includeXpack.set(extension.restTests.getIncludeXpack()); + task.coreConfig = project.getConfigurations().create("restTest"); + if (BuildParams.isInternal()) { + Dependency restTestdependency = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restTests")); + project.getDependencies().add(task.coreConfig.getName(), restTestdependency); + + task.xpackConfig = project.getConfigurations().create("restXpackTest"); + Dependency restXPackTestdependency = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackTests")); + project.getDependencies().add(task.xpackConfig.getName(), restXPackTestdependency); + task.dependsOn(task.xpackConfig); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } + task.dependsOn(task.coreConfig); + }); + + Provider copyRestYamlSpecTask = project.getTasks() + .register("copyRestApiSpecsTask", CopyRestApiTask.class, task -> { + task.includeCore.set(extension.restApi.getIncludeCore()); + task.includeXpack.set(extension.restApi.getIncludeXpack()); + task.dependsOn(copyRestYamlTestTask); + task.coreConfig = project.getConfigurations().create("restSpec"); + if (BuildParams.isInternal()) { + Dependency restSpecDependency = project.getDependencies() + .project(Map.of("path", ":rest-api-spec", "configuration", "restSpecs")); + project.getDependencies().add(task.coreConfig.getName(), restSpecDependency); + + task.xpackConfig = project.getConfigurations().create("restXpackSpec"); + Dependency restXpackSpecDependency = project.getDependencies() + .project(Map.of("path", ":x-pack:plugin", "configuration", "restXpackSpecs")); + project.getDependencies().add(task.xpackConfig.getName(), restXpackSpecDependency); + task.dependsOn(task.xpackConfig); + } else { + Dependency dependency = project.getDependencies() + .create("org.elasticsearch:rest-api-spec:" + VersionProperties.getElasticsearch()); + project.getDependencies().add(task.coreConfig.getName(), dependency); + } + task.dependsOn(task.coreConfig); + }); + + project.getTasks().named("processTestResources").configure(t -> t.dependsOn(copyRestYamlSpecTask)); + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties new file mode 100644 index 0000000000000..af2d3e866ea05 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.rest-resources.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.test.rest.RestResourcesPlugin diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index e707dff3998e0..fabf6e70eacf3 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -25,6 +25,7 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' apply plugin: 'com.github.johnrengelman.shadow' +apply plugin: 'elasticsearch.rest-resources' group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-high-level-client' @@ -37,15 +38,10 @@ publishing { } } -configurations { - restSpec -} - -idea { - module { - if (scopes.TEST != null) { - scopes.TEST.plus.add(project.configurations.restSpec) - } +restResources { + //we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) + restApi { + includeCore '*' } } @@ -72,16 +68,9 @@ dependencies { exclude group: 'org.elasticsearch', module: 'elasticsearch-rest-high-level-client' } testCompile(project(':x-pack:plugin:eql')) - - restSpec project(':rest-api-spec') } -//we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions) processTestResources { - dependsOn configurations.restSpec // so that configurations resolve - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from(project(':client:rest-high-level').file('src/test/resources')) } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle index c72d09adde302..029eb0733023f 100644 --- a/distribution/archives/build.gradle +++ b/distribution/archives/build.gradle @@ -317,9 +317,14 @@ configure(subprojects.findAll { it.name == 'integ-test-zip' }) { group = "org.elasticsearch.distribution.integ-test-zip" + restResources { + restTests { + includeCore '*' + } + } + integTest { dependsOn assemble - includePackaged = true } processTestResources { diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index e0573155b7af8..a6b2295c5318e 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -14,13 +14,11 @@ configurations { dockerPlugins dockerSource ossDockerSource - restSpec } dependencies { dockerSource project(path: ":distribution:archives:linux-tar") ossDockerSource project(path: ":distribution:archives:oss-linux-tar") - restSpec project(':rest-api-spec') } ext.expansions = { oss, local -> @@ -138,12 +136,8 @@ preProcessFixture { } processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') - dependsOn configurations.restSpec } task integTest(type: Test) { diff --git a/docs/build.gradle b/docs/build.gradle index 299e48d9872ed..0deeb0ea9cdd2 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -41,6 +41,12 @@ buildRestTests.expectedUnconvertedCandidates = [ 'reference/ml/anomaly-detection/apis/update-job.asciidoc' ] +restResources { + restApi { + includeCore '*' + } +} + testClusters.integTest { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 0f02cbd52d4e9..ef9d05bf65744 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -30,17 +30,10 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) - dependsOn configurations.restSpec +restResources { + restTests { + includeCore '*' + } } for (Version bwcVersion : bwcVersions.wireCompatible) { diff --git a/qa/remote-clusters/build.gradle b/qa/remote-clusters/build.gradle index 2d0acca20a50d..cc3d3851e4df4 100644 --- a/qa/remote-clusters/build.gradle +++ b/qa/remote-clusters/build.gradle @@ -25,12 +25,7 @@ apply plugin: 'elasticsearch.distribution-download' testFixtures.useFixture() -configurations { - restSpec -} - dependencies { - restSpec project(':rest-api-spec') testCompile project(':client:rest-high-level') } @@ -90,12 +85,8 @@ def createAndSetWritable(Object... locations) { } processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } from project(':x-pack:plugin:core') .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') - dependsOn configurations.restSpec } task integTest(type: Test) { diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index bf1bc2d5b1073..003fc75910523 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -31,21 +31,6 @@ task bwcTest { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - dependsOn configurations.restSpec -} - for (Version bwcVersion : bwcVersions.wireCompatible) { /* * The goal here is to: diff --git a/qa/smoke-test-multinode/build.gradle b/qa/smoke-test-multinode/build.gradle index b76a865c6f898..244b5d1e8af6e 100644 --- a/qa/smoke-test-multinode/build.gradle +++ b/qa/smoke-test-multinode/build.gradle @@ -21,8 +21,10 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' -integTest { - includePackaged = true +restResources { + restTests { + includeCore '*' + } } File repo = file("$buildDir/testclusters/repo") diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index d95ad476682b1..fa29345e0ff6b 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -4,3 +4,13 @@ apply plugin: 'nebula.maven-scm' test.enabled = false jarHell.enabled = false + +configurations { + restSpecs + restTests +} + +artifacts { + restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) + restTests(new File(projectDir, "src/main/resources/rest-api-spec/test")) +} diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 57759b9427bee..c7a82ba62ec3e 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -1,5 +1,3 @@ -import java.nio.charset.StandardCharsets - apply plugin: 'elasticsearch.docs-test' /* List of files that have snippets that probably should be converted to @@ -21,10 +19,11 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } -// copy xpack rest api -File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') -project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' +restResources { + restApi { + includeCore '*' + includeXpack '*' + } } testClusters.integTest { diff --git a/x-pack/plugin/autoscaling/qa/build.gradle b/x-pack/plugin/autoscaling/qa/build.gradle index 46609933699ca..e1c6fb4f95bea 100644 --- a/x-pack/plugin/autoscaling/qa/build.gradle +++ b/x-pack/plugin/autoscaling/qa/build.gradle @@ -1,18 +1,6 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - apply plugin: 'elasticsearch.build' test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } - -} diff --git a/x-pack/plugin/autoscaling/qa/rest/build.gradle b/x-pack/plugin/autoscaling/qa/rest/build.gradle index 9e297a87ea18d..631c6e950ee1a 100644 --- a/x-pack/plugin/autoscaling/qa/rest/build.gradle +++ b/x-pack/plugin/autoscaling/qa/rest/build.gradle @@ -9,6 +9,12 @@ dependencies { testCompile project(path: xpackModule('autoscaling'), configuration: 'runtime') } +restResources { + restApi { + includeXpack 'autoscaling' + } +} + task restTest(type: RestIntegTestTask) { mustRunAfter(precommit) } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 85d28fcd65d40..f2ca68bd81416 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -13,6 +13,13 @@ dependencies { // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime + restXpackSpecs + restXpackTests +} + +artifacts { + restXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api")) + restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test")) } task testJar(type: Jar) { diff --git a/x-pack/plugin/ccr/qa/build.gradle b/x-pack/plugin/ccr/qa/build.gradle index 46609933699ca..e1c6fb4f95bea 100644 --- a/x-pack/plugin/ccr/qa/build.gradle +++ b/x-pack/plugin/ccr/qa/build.gradle @@ -1,18 +1,6 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - apply plugin: 'elasticsearch.build' test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } - -} diff --git a/x-pack/plugin/ccr/qa/rest/build.gradle b/x-pack/plugin/ccr/qa/rest/build.gradle index 4383db34f3e11..d95ac271df519 100644 --- a/x-pack/plugin/ccr/qa/rest/build.gradle +++ b/x-pack/plugin/ccr/qa/rest/build.gradle @@ -3,6 +3,12 @@ import org.elasticsearch.gradle.test.RestIntegTestTask apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' +restResources { + restApi { + includeXpack 'ccr' + } +} + dependencies { testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('ccr'), configuration: 'runtime') diff --git a/x-pack/plugin/enrich/qa/build.gradle b/x-pack/plugin/enrich/qa/build.gradle index d3e95d997c3fb..79ff4091f6d2d 100644 --- a/x-pack/plugin/enrich/qa/build.gradle +++ b/x-pack/plugin/enrich/qa/build.gradle @@ -6,12 +6,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/enrich/qa/rest/build.gradle b/x-pack/plugin/enrich/qa/rest/build.gradle index 30ad9ff335e98..209154dd448b0 100644 --- a/x-pack/plugin/enrich/qa/rest/build.gradle +++ b/x-pack/plugin/enrich/qa/rest/build.gradle @@ -2,6 +2,12 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' +restResources { + restApi { + includeXpack 'enrich' + } +} + dependencies { testCompile project(path: xpackModule('enrich'), configuration: 'runtime') testCompile project(path: xpackModule('enrich:qa:common'), configuration: 'runtime') diff --git a/x-pack/plugin/eql/qa/build.gradle b/x-pack/plugin/eql/qa/build.gradle index d3e95d997c3fb..79ff4091f6d2d 100644 --- a/x-pack/plugin/eql/qa/build.gradle +++ b/x-pack/plugin/eql/qa/build.gradle @@ -6,12 +6,3 @@ test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/eql/qa/rest/build.gradle b/x-pack/plugin/eql/qa/rest/build.gradle index 8e4f9dc97e1fa..a7e94fcba6d6d 100644 --- a/x-pack/plugin/eql/qa/rest/build.gradle +++ b/x-pack/plugin/eql/qa/rest/build.gradle @@ -4,6 +4,12 @@ apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' +restResources { + restApi { + includeXpack 'eql' + } +} + dependencies { testCompile project(path: xpackModule('eql'), configuration: 'runtime') testCompile project(path: xpackModule('eql:qa:common'), configuration: 'runtime') diff --git a/x-pack/plugin/graph/qa/build.gradle b/x-pack/plugin/graph/qa/build.gradle index 012e25f5d4f9d..e69de29bb2d1d 100644 --- a/x-pack/plugin/graph/qa/build.gradle +++ b/x-pack/plugin/graph/qa/build.gradle @@ -1,17 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/graph/qa/with-security/build.gradle b/x-pack/plugin/graph/qa/with-security/build.gradle index 578f910146748..869e926102dcb 100644 --- a/x-pack/plugin/graph/qa/with-security/build.gradle +++ b/x-pack/plugin/graph/qa/with-security/build.gradle @@ -7,14 +7,15 @@ dependencies { } // bring in graph rest test suite -task copyGraphRestTests(type: Copy) { - into project.sourceSets.test.output.resourcesDir - from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs - include 'rest-api-spec/test/graph/**' +restResources { + restApi { + includeXpack 'graph' + } + restTests { + includeXpack 'graph' + } } -integTest.dependsOn copyGraphRestTests - testClusters.integTest { testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'true' diff --git a/x-pack/plugin/ilm/qa/build.gradle b/x-pack/plugin/ilm/qa/build.gradle index 46908e1d849b9..e69de29bb2d1d 100644 --- a/x-pack/plugin/ilm/qa/build.gradle +++ b/x-pack/plugin/ilm/qa/build.gradle @@ -1,11 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} - diff --git a/x-pack/plugin/ilm/qa/rest/build.gradle b/x-pack/plugin/ilm/qa/rest/build.gradle index 3f67169914928..6c5ea98a0b46d 100644 --- a/x-pack/plugin/ilm/qa/rest/build.gradle +++ b/x-pack/plugin/ilm/qa/rest/build.gradle @@ -8,6 +8,12 @@ dependencies { testCompile project(path: xpackModule('ilm'), configuration: 'runtime') } +restResources { + restApi { + includeXpack 'ilm', 'slm' + } +} + def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'), password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')] diff --git a/x-pack/plugin/ml/qa/build.gradle b/x-pack/plugin/ml/qa/build.gradle index 012e25f5d4f9d..e69de29bb2d1d 100644 --- a/x-pack/plugin/ml/qa/build.gradle +++ b/x-pack/plugin/ml/qa/build.gradle @@ -1,17 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index 8b4f118756d95..2ac378e472a7b 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -9,15 +9,15 @@ dependencies { } // bring in machine learning rest test suite -task copyMlRestTests(type: Copy) { - into project.sourceSets.test.output.resourcesDir - from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs - include 'rest-api-spec/test/ml/**' +restResources { + restApi { + includeXpack 'ml', 'cat' + } + restTests { + includeXpack 'ml' + } } -integTest.runner { - dependsOn copyMlRestTests -} integTest.runner { systemProperty 'tests.rest.blacklist', [ // Remove this test because it doesn't call an ML endpoint and we don't want diff --git a/x-pack/plugin/security/qa/build.gradle b/x-pack/plugin/security/qa/build.gradle index 46908e1d849b9..e69de29bb2d1d 100644 --- a/x-pack/plugin/security/qa/build.gradle +++ b/x-pack/plugin/security/qa/build.gradle @@ -1,11 +0,0 @@ -import org.elasticsearch.gradle.test.RestIntegTestTask - -subprojects { - project.tasks.withType(RestIntegTestTask) { - final File xPackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xPackResources) { - include 'rest-api-spec/api/**' - } - } -} - diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle index 2555b0ef729dc..f933cac8d31cc 100644 --- a/x-pack/qa/build.gradle +++ b/x-pack/qa/build.gradle @@ -1,27 +1,9 @@ // this file must exist so that qa projects are found // by the elasticsearch x-plugins include mechanism -import org.elasticsearch.gradle.test.RestIntegTestTask - apply plugin: 'elasticsearch.build' test.enabled = false dependencies { compile project(':test:framework') } - -subprojects { - // HACK: please fix this - // we want to add the rest api specs for xpack to qa tests, but we - // need to wait until after the project is evaluated to only apply - // to those that rest tests. this used to be done automatically - // when xpack was a plugin, but now there is no place with xpack as a module. - // instead, we should package these and make them easy to use for rest tests, - // but currently, they must be copied into the resources of the test runner. - project.tasks.withType(RestIntegTestTask) { - File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') - project.copyRestSpec.from(xpackResources) { - include 'rest-api-spec/api/**' - } - } -} diff --git a/x-pack/qa/core-rest-tests-with-security/build.gradle b/x-pack/qa/core-rest-tests-with-security/build.gradle index f273efaf5bcd2..4fc4428c8ed6f 100644 --- a/x-pack/qa/core-rest-tests-with-security/build.gradle +++ b/x-pack/qa/core-rest-tests-with-security/build.gradle @@ -6,8 +6,13 @@ dependencies { testCompile project(':x-pack:qa') } +restResources { + restTests { + includeCore '*' + } +} + integTest { - includePackaged = true runner { systemProperty 'tests.rest.blacklist', [ diff --git a/x-pack/qa/full-cluster-restart/build.gradle b/x-pack/qa/full-cluster-restart/build.gradle index 757d8cd298154..c0527f98766a6 100644 --- a/x-pack/qa/full-cluster-restart/build.gradle +++ b/x-pack/qa/full-cluster-restart/build.gradle @@ -1,8 +1,6 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask - apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' @@ -39,24 +37,6 @@ tasks.register("copyTestNodeKeyMaterial", Copy) { into outputDir } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackModule('core')).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - for (Version bwcVersion : bwcVersions.indexCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/multi-cluster-search-security/build.gradle b/x-pack/qa/multi-cluster-search-security/build.gradle index 3f3d03a7e685e..715d8857bac1b 100644 --- a/x-pack/qa/multi-cluster-search-security/build.gradle +++ b/x-pack/qa/multi-cluster-search-security/build.gradle @@ -7,6 +7,12 @@ dependencies { testCompile project(':x-pack:qa') } +restResources { + restApi { + includeXpack 'security' + } +} + task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) runner { diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle index 74b15cd87a51f..b6a3088f250aa 100644 --- a/x-pack/qa/multi-cluster-tests-with-security/build.gradle +++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle @@ -8,6 +8,12 @@ dependencies { testCompile project(':client:rest-high-level') } +restResources { + restApi { + includeXpack 'security', 'transform' + } +} + task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) runner { diff --git a/x-pack/qa/rolling-upgrade-basic/build.gradle b/x-pack/qa/rolling-upgrade-basic/build.gradle index 9367b74aae379..fd4c78f6ff8b8 100644 --- a/x-pack/qa/rolling-upgrade-basic/build.gradle +++ b/x-pack/qa/rolling-upgrade-basic/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -14,25 +13,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - - for (Version bwcVersion : bwcVersions.wireCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle index bba1d1fd96e24..5f7c9b6af5662 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle +++ b/x-pack/qa/rolling-upgrade-multi-cluster/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -14,24 +13,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - for (Version bwcVersion : bwcVersions.wireCompatible) { String baseName = "v${bwcVersion}" diff --git a/x-pack/qa/rolling-upgrade/build.gradle b/x-pack/qa/rolling-upgrade/build.gradle index 6bda921a54596..4f69e0fb8a4a7 100644 --- a/x-pack/qa/rolling-upgrade/build.gradle +++ b/x-pack/qa/rolling-upgrade/build.gradle @@ -1,5 +1,4 @@ import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testclusters.RestTestRunnerTask apply plugin: 'elasticsearch.testclusters' @@ -10,6 +9,13 @@ dependencies { testCompile project(':client:rest-high-level') } +restResources { + restApi { + includeCore '*' + includeXpack '*' + } +} + forbiddenPatterns { exclude '**/system_key' } @@ -21,24 +27,6 @@ tasks.register("bwcTest") { group = 'verification' } -configurations { - restSpec -} - -dependencies { - restSpec project(':rest-api-spec') -} - -processTestResources { - dependsOn configurations.restSpec - from({ zipTree(configurations.restSpec.singleFile) }) { - include 'rest-api-spec/api/**' - } - from(project(xpackProject('plugin').path).sourceSets.test.resources) { - include 'rest-api-spec/api/**' - } -} - task copyTestNodeKeyMaterial(type: Copy) { from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt') diff --git a/x-pack/qa/smoke-test-security-with-mustache/build.gradle b/x-pack/qa/smoke-test-security-with-mustache/build.gradle index 748252044c36c..3c38b9ae8f30f 100644 --- a/x-pack/qa/smoke-test-security-with-mustache/build.gradle +++ b/x-pack/qa/smoke-test-security-with-mustache/build.gradle @@ -6,6 +6,12 @@ dependencies { testCompile project(':x-pack:qa') } +restResources { + restApi { + includeXpack 'security' + } +} + testClusters.integTest { testDistribution = 'DEFAULT' setting 'xpack.watcher.enabled', 'false' diff --git a/x-pack/qa/smoke-test-watcher-with-security/build.gradle b/x-pack/qa/smoke-test-watcher-with-security/build.gradle index 236ec27cfa4e0..6fbbfa87ed557 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/build.gradle +++ b/x-pack/qa/smoke-test-watcher-with-security/build.gradle @@ -6,14 +6,15 @@ dependencies { testCompile project(':x-pack:qa') } -// bring in watcher rest test suite -task copyWatcherRestTests(type: Copy) { - into project.sourceSets.test.output.resourcesDir - from project(xpackProject('plugin').path).sourceSets.test.resources.srcDirs - include 'rest-api-spec/test/watcher/**' +restResources { + restApi { + includeXpack 'watcher', 'security', 'xpack' + } + restTests { + includeXpack 'watcher' + } } -integTest.runner.dependsOn copyWatcherRestTests testClusters.integTest { testDistribution = 'DEFAULT' setting 'xpack.ilm.enabled', 'false' diff --git a/x-pack/qa/smoke-test-watcher/build.gradle b/x-pack/qa/smoke-test-watcher/build.gradle index b3b638e938343..51d56669b837e 100644 --- a/x-pack/qa/smoke-test-watcher/build.gradle +++ b/x-pack/qa/smoke-test-watcher/build.gradle @@ -6,6 +6,12 @@ dependencies { testCompile project(':x-pack:qa') } +restResources { + restApi { + includeXpack 'watcher' + } +} + testClusters.integTest { testDistribution = 'DEFAULT' setting 'xpack.slm.enabled', 'false' diff --git a/x-pack/qa/third-party/jira/build.gradle b/x-pack/qa/third-party/jira/build.gradle index f724d5f1f3674..f2642d7150049 100644 --- a/x-pack/qa/third-party/jira/build.gradle +++ b/x-pack/qa/third-party/jira/build.gradle @@ -12,22 +12,17 @@ dependencies { testCompile project(path: xpackModule('watcher'), configuration: 'runtime') } +restResources { + restApi { + includeXpack 'watcher' + } +} String jiraUrl = System.getenv('jira_url') String jiraUser = System.getenv('jira_user') String jiraPassword = System.getenv('jira_password') String jiraProject = System.getenv('jira_project') -testClusters.integTest { - setting 'xpack.security.enabled', 'false' - setting 'xpack.monitoring.enabled', 'false' - setting 'xpack.ml.enabled', 'false' - setting 'xpack.license.self_generated.type', 'trial' - setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' - setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug' - setting 'xpack.notification.jira.account.test.issue_defaults.labels.0', 'integration-tests' -} - task cleanJira(type: DefaultTask) { doLast { List issues = jiraIssues(jiraProject) @@ -46,6 +41,14 @@ if (!jiraUrl && !jiraUser && !jiraPassword && !jiraProject) { testingConventions.enabled = false } else { testClusters.integTest { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + setting 'logger.org.elasticsearch.xpack.watcher', 'DEBUG' + setting 'xpack.notification.jira.account.test.issue_defaults.issuetype.name', 'Bug' + setting 'xpack.notification.jira.account.test.issue_defaults.labels.0', 'integration-tests' setting 'xpack.notification.jira.account.test.issue_defaults.project.key', jiraProject keystore 'xpack.notification.jira.account.test.secure_url', jiraUrl keystore 'xpack.notification.jira.account.test.secure_user', jiraUser diff --git a/x-pack/qa/third-party/pagerduty/build.gradle b/x-pack/qa/third-party/pagerduty/build.gradle index ebee73f9eaaed..c65f04118e0d3 100644 --- a/x-pack/qa/third-party/pagerduty/build.gradle +++ b/x-pack/qa/third-party/pagerduty/build.gradle @@ -9,11 +9,18 @@ dependencies { String pagerDutyServiceKey = System.getenv('pagerduty_service_api_key') +restResources { + restApi { + includeXpack 'watcher' + } +} + if (!pagerDutyServiceKey) { integTest.enabled = false testingConventions.enabled = false } else { testClusters.integTest { + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false' diff --git a/x-pack/qa/third-party/slack/build.gradle b/x-pack/qa/third-party/slack/build.gradle index 815d1cb3bbfff..a1b92050482bb 100644 --- a/x-pack/qa/third-party/slack/build.gradle +++ b/x-pack/qa/third-party/slack/build.gradle @@ -7,6 +7,12 @@ dependencies { testCompile project(path: xpackModule('watcher'), configuration: 'runtime') } +restResources { + restApi { + includeXpack 'watcher' + } +} + String slackUrl = System.getenv('slack_url') if (!slackUrl) { @@ -14,6 +20,7 @@ if (!slackUrl) { testingConventions.enabled = false } else { testClusters.integTest { + testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' setting 'xpack.monitoring.enabled', 'false' setting 'xpack.ml.enabled', 'false'