Skip to content

Commit

Permalink
Merge remote-tracking branch 'es/ccr' into ccr_more_validation
Browse files Browse the repository at this point in the history
* es/ccr: (37 commits)
  Default to one shard (elastic#30539)
  Unmute IndexUpgradeIT tests
  Forbid expensive query parts in ranking evaluation (elastic#30151)
  Docs: Update HighLevelRestClient migration docs (elastic#30544)
  Clients: Switch to new performRequest (elastic#30543)
  [TEST] Fix typo in MovAvgIT test
  Add missing dependencies on testClasses (elastic#30527)
  [TEST] Mute ML test that needs updating to following ml-cpp changes
  Document woes between auto-expand-replicas and allocation filtering (elastic#30531)
  Moved tokenizers to analysis common module (elastic#30538)
  Adjust copy settings versions
  Mute ShrinkIndexIT suite
  SQL: SYS TABLES ordered according to *DBC specs (elastic#30530)
  Deprecate not copy settings and explicitly disallow (elastic#30404)
  [ML] Improve state persistence log message
  Build: Add mavenPlugin cluster configuration method (elastic#30541)
  Re-enable FlushIT tests
  Bump Gradle heap to 2 GB (elastic#30535)
  SQL: Use request flavored methods in tests (elastic#30345)
  Suppress hdfsFixture if there are spaces in the path (elastic#30302)
  ...
  • Loading branch information
martijnvg committed May 15, 2018
2 parents 80f5ccb + b971a81 commit 56e584a
Show file tree
Hide file tree
Showing 182 changed files with 2,106 additions and 1,391 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,7 @@ class BuildPlugin implements Plugin<Project> {
additionalTest.testClassesDir = test.testClassesDir
additionalTest.configure(commonTestConfig(project))
additionalTest.configure(config)
additionalTest.dependsOn(project.tasks.testClasses)
test.dependsOn(additionalTest)
});
return test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
* warning every time. */
current.println(" - skip:")
current.println(" features: ")
current.println(" - default_shards")
current.println(" - stash_in_key")
current.println(" - stash_in_path")
current.println(" - stash_path_replace")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ class ClusterConfiguration {
// map from destination path, to source file
Map<String, Object> extraConfigFiles = new HashMap<>()

LinkedHashMap<String, Project> plugins = new LinkedHashMap<>()
LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()

List<Project> modules = new ArrayList<>()

Expand Down Expand Up @@ -185,6 +185,11 @@ class ClusterConfiguration {
plugins.put(pluginProject.name, pluginProject)
}

@Input
void mavenPlugin(String name, String mavenCoords) {
plugins.put(name, mavenCoords)
}

/** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
@Input
void module(Project moduleProject) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ class ClusterFormationTasks {
// from mirrors using gradles built-in mechanism etc.

configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
for (Map.Entry<String, Object> entry : config.plugins.entrySet()) {
configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion)
}
bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
Expand Down Expand Up @@ -150,10 +150,15 @@ class ClusterFormationTasks {
}

/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) {
verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
static void configureBwcPluginDependency(Project project, Object plugin, Configuration configuration, Version elasticsearchVersion) {
if (plugin instanceof Project) {
Project pluginProject = (Project)plugin
verifyProjectHasBuildPlugin(configuration.name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
} else {
project.dependencies.add(configuration.name, "${plugin}@zip")
}
}

/**
Expand Down Expand Up @@ -210,9 +215,9 @@ class ClusterFormationTasks {
}

// install plugins
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
for (String pluginName : node.config.plugins.keySet()) {
String actionName = pluginTaskName('install', pluginName, 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, pluginName, prefix)
}

// sets up any extra config files that need to be copied over to the ES instance;
Expand Down Expand Up @@ -444,31 +449,40 @@ class ClusterFormationTasks {
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)

List<FileCollection> pluginFiles = []
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {

Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginConfigurationName(prefix, pluginProject)
String configurationName = pluginConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}
project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)

if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}



pluginFiles.add(configuration)
}

Expand All @@ -477,32 +491,37 @@ class ClusterFormationTasks {
return copyPlugins
}

private static String pluginConfigurationName(final String prefix, final Project project) {
return "_plugin_${prefix}_${project.path}".replace(':', '_')
private static String pluginConfigurationName(final String prefix, final String name) {
return "_plugin_${prefix}_${name}".replace(':', '_')
}

private static String pluginBwcConfigurationName(final String prefix, final Project project) {
return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
private static String pluginBwcConfigurationName(final String prefix, final String name) {
return "_plugin_bwc_${prefix}_${name}".replace(':', '_')
}

/** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
String configurationName = pluginBwcConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}

final String depName = findPluginName(pluginProject)
if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
final String depName = findPluginName(pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
}
configuration.dependencies.add(dep)
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}
configuration.dependencies.add(dep)
}

Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
Expand All @@ -527,12 +546,12 @@ class ClusterFormationTasks {
return installModule
}

static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
final FileCollection pluginZip;
if (node.nodeVersion != VersionProperties.elasticsearch) {
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
} else {
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName))
}
// delay reading the file location until execution time by wrapping in a closure within a GString
final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,19 @@
*/
package org.elasticsearch.client.benchmark.rest;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpStatus;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.benchmark.AbstractBenchmark;
import org.elasticsearch.client.benchmark.ops.bulk.BulkRequestExecutor;
import org.elasticsearch.client.benchmark.ops.search.SearchRequestExecutor;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
Expand Down Expand Up @@ -86,9 +78,10 @@ public boolean bulkIndex(List<String> bulkData) {
bulkRequestBody.append(bulkItem);
bulkRequestBody.append("\n");
}
HttpEntity entity = new NStringEntity(bulkRequestBody.toString(), ContentType.APPLICATION_JSON);
Request request = new Request("POST", "/geonames/type/_noop_bulk");
request.setJsonEntity(bulkRequestBody.toString());
try {
Response response = client.performRequest("POST", "/geonames/type/_noop_bulk", Collections.emptyMap(), entity);
Response response = client.performRequest(request);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (Exception e) {
throw new ElasticsearchException(e);
Expand All @@ -107,9 +100,10 @@ private RestSearchRequestExecutor(RestClient client, String indexName) {

@Override
public boolean search(String source) {
HttpEntity searchBody = new NStringEntity(source, StandardCharsets.UTF_8);
Request request = new Request("GET", endpoint);
request.setJsonEntity(source);
try {
Response response = client.performRequest("GET", endpoint, Collections.emptyMap(), searchBody);
Response response = client.performRequest(request);
return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
} catch (IOException e) {
throw new ElasticsearchException(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,18 +194,16 @@ public void testBulkProcessorWaitOnClose() throws Exception {
}

public void testBulkProcessorConcurrentRequestsReadOnlyIndex() throws Exception {

String createIndexBody = "{\n" +
Request request = new Request("PUT", "/test-ro");
request.setJsonEntity("{\n" +
" \"settings\" : {\n" +
" \"index\" : {\n" +
" \"blocks.write\" : true\n" +
" }\n" +
" }\n" +
" \n" +
"}";

NStringEntity entity = new NStringEntity(createIndexBody, ContentType.APPLICATION_JSON);
Response response = client().performRequest("PUT", "/test-ro", Collections.emptyMap(), entity);
"}");
Response response = client().performRequest(request);
assertThat(response.getStatusLine().getStatusCode(), equalTo(200));

int bulkActions = randomIntBetween(10, 100);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@

package org.elasticsearch.client;

import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.DocWriteRequest;
Expand All @@ -39,6 +36,7 @@
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -147,11 +145,10 @@ public void testExists() throws IOException {
GetRequest getRequest = new GetRequest("index", "type", "id");
assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
}
String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
stringEntity);
assertEquals(201, response.getStatusLine().getStatusCode());
IndexRequest index = new IndexRequest("index", "type", "id");
index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON);
index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
highLevelClient().index(index);
{
GetRequest getRequest = new GetRequest("index", "type", "id");
assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync));
Expand All @@ -175,12 +172,11 @@ public void testGet() throws IOException {
assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage());
assertEquals("index", exception.getMetadata("es.index").get(0));
}

IndexRequest index = new IndexRequest("index", "type", "id");
String document = "{\"field1\":\"value1\",\"field2\":\"value2\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response response = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id", Collections.singletonMap("refresh", "wait_for"),
stringEntity);
assertEquals(201, response.getStatusLine().getStatusCode());
index.source(document, XContentType.JSON);
index.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
highLevelClient().index(index);
{
GetRequest getRequest = new GetRequest("index", "type", "id").version(2);
ElasticsearchException exception = expectThrows(ElasticsearchException.class,
Expand Down Expand Up @@ -271,18 +267,15 @@ public void testMultiGet() throws IOException {
assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]",
response.getResponses()[1].getFailure().getFailure().getMessage());
}

String document = "{\"field\":\"value1\"}";
StringEntity stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
Response r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id1", Collections.singletonMap("refresh", "true"),
stringEntity);
assertEquals(201, r.getStatusLine().getStatusCode());

document = "{\"field\":\"value2\"}";
stringEntity = new StringEntity(document, ContentType.APPLICATION_JSON);
r = client().performRequest(HttpPut.METHOD_NAME, "/index/type/id2", Collections.singletonMap("refresh", "true"), stringEntity);
assertEquals(201, r.getStatusLine().getStatusCode());

BulkRequest bulk = new BulkRequest();
bulk.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
IndexRequest index = new IndexRequest("index", "type", "id1");
index.source("{\"field\":\"value1\"}", XContentType.JSON);
bulk.add(index);
index = new IndexRequest("index", "type", "id2");
index.source("{\"field\":\"value2\"}", XContentType.JSON);
bulk.add(index);
highLevelClient().bulk(bulk);
{
MultiGetRequest multiGetRequest = new MultiGetRequest();
multiGetRequest.add("index", "type", "id1");
Expand Down
Loading

0 comments on commit 56e584a

Please sign in to comment.