From 3d47d5f024b6b07d7585e5ea4fe060c02f5fef67 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 23 Jan 2019 16:48:23 -0700 Subject: [PATCH 01/13] Deprecate _type in simulate pipeline requests As mapping types are being removed throughout Elasticsearch, the use of `_type` in pipeline simulation requests is deprecated. --- .../ingest/SimulatePipelineRequest.java | 10 ++++ .../SimulatePipelineRequestParsingTests.java | 49 ++++++++++++++----- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 4c2736e3d86de..2d83e65a26311 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -19,11 +19,14 @@ package org.elasticsearch.action.ingest; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -43,6 +46,9 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject { + private static final Logger logger = LogManager.getLogger(SimulatePipelineRequest.class); + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger); + private String id; private boolean verbose; private BytesReference source; @@ -178,6 +184,10 @@ private static List parseDocs(Map config) { dataMap, Fields.SOURCE); String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"); + if (dataMap.containsKey(MetaData.TYPE.getFieldName())) { + deprecationLogger.deprecatedAndMaybeLog("type_in_pipeline_simulation", + "Specifying _type in pipeline simulation requests is deprecated."); + } String type = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.TYPE.getFieldName(), "_type"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 1711d16891083..0a1aa8fb4a551 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -19,15 +19,6 @@ package org.elasticsearch.action.ingest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.CompoundProcessor; import org.elasticsearch.ingest.IngestDocument; @@ -38,6 +29,15 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields; import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID; import static org.elasticsearch.ingest.IngestDocument.MetaData.ID; @@ -69,6 +69,7 @@ public void init() throws IOException { public void testParseUsingPipelineStore() throws Exception { int numDocs = randomIntBetween(1, 10); + boolean useExplicitType = randomBoolean(); Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); @@ -80,7 +81,9 @@ public void testParseUsingPipelineStore() throws Exception { String type = randomAlphaOfLengthBetween(1, 10); String id = randomAlphaOfLengthBetween(1, 10); doc.put(INDEX.getFieldName(), index); - doc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + doc.put(TYPE.getFieldName(), type); + } doc.put(ID.getFieldName(), id); String fieldName = randomAlphaOfLengthBetween(1, 10); String fieldValue = randomAlphaOfLengthBetween(1, 10); @@ -88,7 +91,11 @@ public void testParseUsingPipelineStore() throws Exception { docs.add(doc); Map expectedDoc = new HashMap<>(); expectedDoc.put(INDEX.getFieldName(), index); - expectedDoc.put(TYPE.getFieldName(), type); + if (useExplicitType) { + expectedDoc.put(TYPE.getFieldName(), type); + } else { + expectedDoc.put(TYPE.getFieldName(), "_type"); + } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); expectedDocs.add(expectedDoc); @@ -111,10 +118,14 @@ public void testParseUsingPipelineStore() throws Exception { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1)); + if (useExplicitType) { + assertWarnings("Specifying _type in pipeline simulation requests is deprecated."); + } } public void testParseWithProvidedPipeline() throws Exception { int numDocs = randomIntBetween(1, 10); + boolean useExplicitType = randomBoolean(); Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); @@ -135,6 +146,14 @@ public void testParseWithProvidedPipeline() throws Exception { ); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); + } else if (field == TYPE) { + if (useExplicitType) { + String value = randomAlphaOfLengthBetween(1, 10); + doc.put(field.getFieldName(), value); + expectedDoc.put(field.getFieldName(), value); + } else { + expectedDoc.put(field.getFieldName(), "_type"); + } } else { if (randomBoolean()) { String value = randomAlphaOfLengthBetween(1, 10); @@ -191,7 +210,6 @@ public void testParseWithProvidedPipeline() throws Exception { Map expectedDocument = expectedDocsIterator.next(); Map metadataMap = ingestDocument.extractMetadata(); assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName()))); - assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName()))); assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName()))); assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName()))); assertThat(metadataMap.get(VERSION), equalTo(expectedDocument.get(VERSION.getFieldName()))); @@ -202,6 +220,9 @@ public void testParseWithProvidedPipeline() throws Exception { assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID)); assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors)); + if (useExplicitType) { + assertWarnings("Specifying _type in pipeline simulation requests is deprecated."); + } } public void testNullPipelineId() { @@ -222,4 +243,8 @@ public void testNonExistentPipelineId() { () -> SimulatePipelineRequest.parseWithPipelineId(pipelineId, requestContent, false, ingestService)); assertThat(e.getMessage(), equalTo("pipeline [" + pipelineId + "] does not exist")); } + + public void testTypeDeprecation() { + + } } From 1a6c9b035dbaf677c73f6bd8fa29bce2c24e2c47 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Wed, 23 Jan 2019 16:54:25 -0700 Subject: [PATCH 02/13] Update docs --- .../ingest/apis/simulate-pipeline.asciidoc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index d4f043e802159..c5ac6ab2f29e5 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -65,7 +65,6 @@ POST _ingest/pipeline/_simulate "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -73,7 +72,6 @@ POST _ingest/pipeline/_simulate }, { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "rab" @@ -94,7 +92,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", "_source": { "field2": "_value", "foo": "bar" @@ -108,7 +105,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", "_source": { "field2": "_value", "foo": "rab" @@ -158,7 +154,6 @@ POST _ingest/pipeline/_simulate?verbose "docs": [ { "_index": "index", - "_type": "_doc", "_id": "id", "_source": { "foo": "bar" @@ -166,7 +161,7 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", - "_type": "_doc", + "_type": "_type", "_id": "id", "_source": { "foo": "rab" @@ -189,7 +184,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", "_source": { "field2": "_value2", "foo": "bar" @@ -203,7 +197,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", "_source": { "field3": "_value3", "field2": "_value2", @@ -222,7 +215,7 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", + "_type": "_type", "_source": { "field2": "_value2", "foo": "rab" @@ -236,7 +229,7 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_doc", + "_type": "_type", "_source": { "field3": "_value3", "field2": "_value2", From 8a85a2f2781234c2b1bb5505bca8d3db29d3fe09 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 28 Jan 2019 18:07:09 -0700 Subject: [PATCH 03/13] Cleanup --- .../action/ingest/SimulatePipelineRequestParsingTests.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 0a1aa8fb4a551..ad8a442eca651 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -243,8 +243,4 @@ public void testNonExistentPipelineId() { () -> SimulatePipelineRequest.parseWithPipelineId(pipelineId, requestContent, false, ingestService)); assertThat(e.getMessage(), equalTo("pipeline [" + pipelineId + "] does not exist")); } - - public void testTypeDeprecation() { - - } } From 5d8cacd21d26214c5381b6012d80ab37a9a05d7b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Mon, 28 Jan 2019 18:14:52 -0700 Subject: [PATCH 04/13] Tweak test setup --- .../ingest/SimulatePipelineRequestParsingTests.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index ad8a442eca651..ac0a1c6d10404 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.ingest.TestProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; @@ -55,6 +56,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { private IngestService ingestService; + // Ensure one of two tests will use the deprecated explicit type, and the other will not + private boolean explicitTypeToggle = randomBoolean(); + @Before public void init() throws IOException { TestProcessor processor = new TestProcessor(ingestDocument -> {}); @@ -69,7 +73,7 @@ public void init() throws IOException { public void testParseUsingPipelineStore() throws Exception { int numDocs = randomIntBetween(1, 10); - boolean useExplicitType = randomBoolean(); + boolean useExplicitType = explicitTypeToggle; Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); @@ -125,7 +129,7 @@ public void testParseUsingPipelineStore() throws Exception { public void testParseWithProvidedPipeline() throws Exception { int numDocs = randomIntBetween(1, 10); - boolean useExplicitType = randomBoolean(); + boolean useExplicitType = explicitTypeToggle == false; Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); From 7a3a08c6ba89bcf8195ac8c5c8e3a8ad7cf58609 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 29 Jan 2019 11:28:13 -0700 Subject: [PATCH 05/13] Omit _type from response if not present in request --- .../elasticsearch/action/ingest/SimulatePipelineRequest.java | 3 +-- .../action/ingest/SimulatePipelineRequestParsingTests.java | 5 ----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 2d83e65a26311..064c76adf3f25 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -188,8 +188,7 @@ private static List parseDocs(Map config) { deprecationLogger.deprecatedAndMaybeLog("type_in_pipeline_simulation", "Specifying _type in pipeline simulation requests is deprecated."); } - String type = ConfigurationUtils.readStringOrIntProperty(null, null, - dataMap, MetaData.TYPE.getFieldName(), "_type"); + String type = (String) dataMap.get(MetaData.TYPE.getFieldName()); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index ac0a1c6d10404..c1865160465dc 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.ingest.TestProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; @@ -97,8 +96,6 @@ public void testParseUsingPipelineStore() throws Exception { expectedDoc.put(INDEX.getFieldName(), index); if (useExplicitType) { expectedDoc.put(TYPE.getFieldName(), type); - } else { - expectedDoc.put(TYPE.getFieldName(), "_type"); } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); @@ -155,8 +152,6 @@ public void testParseWithProvidedPipeline() throws Exception { String value = randomAlphaOfLengthBetween(1, 10); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); - } else { - expectedDoc.put(field.getFieldName(), "_type"); } } else { if (randomBoolean()) { From 6fcccd9962b8c0cb88c87067851b11cf01c0f0cc Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 29 Jan 2019 11:31:39 -0700 Subject: [PATCH 06/13] Always test with and without types in both tests --- .../SimulatePipelineRequestParsingTests.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index c1865160465dc..4188a59431182 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -55,9 +55,6 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase { private IngestService ingestService; - // Ensure one of two tests will use the deprecated explicit type, and the other will not - private boolean explicitTypeToggle = randomBoolean(); - @Before public void init() throws IOException { TestProcessor processor = new TestProcessor(ingestDocument -> {}); @@ -70,9 +67,16 @@ public void init() throws IOException { when(ingestService.getProcessorFactories()).thenReturn(registry); } - public void testParseUsingPipelineStore() throws Exception { + public void testParseUsingPipelineStoreNoType() throws Exception { + innerTestParseUsingPipelineStore(false); + } + + public void testParseUsingPipelineStoreWithType() throws Exception { + innerTestParseUsingPipelineStore(true); + } + + private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); - boolean useExplicitType = explicitTypeToggle; Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); @@ -124,9 +128,16 @@ public void testParseUsingPipelineStore() throws Exception { } } - public void testParseWithProvidedPipeline() throws Exception { + public void testParseWithProvidedPipelineNoType() throws Exception { + innerTestParseWithProvidedPipeline(false); + } + + public void testParseWithProvidedPipelineWithType() throws Exception { + innerTestParseWithProvidedPipeline(true); + } + + private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws Exception { int numDocs = randomIntBetween(1, 10); - boolean useExplicitType = explicitTypeToggle == false; Map requestContent = new HashMap<>(); List> docs = new ArrayList<>(); From acc857fae55d93516f7514cd4f6911b491965ba7 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 29 Jan 2019 11:32:07 -0700 Subject: [PATCH 07/13] Remove _type from docs entirely --- docs/reference/ingest/apis/simulate-pipeline.asciidoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index c5ac6ab2f29e5..37a25f9cf64fe 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -161,7 +161,6 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", - "_type": "_type", "_id": "id", "_source": { "foo": "rab" @@ -215,7 +214,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_type", "_source": { "field2": "_value2", "foo": "rab" @@ -229,7 +227,6 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_type", "_source": { "field3": "_value3", "field2": "_value2", From 8d3ee4d26826205da640491636f7bb58ec0b002b Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 29 Jan 2019 12:51:12 -0700 Subject: [PATCH 08/13] Fix docs to remove _type as needed --- docs/reference/ingest/processors/date-index-name.asciidoc | 1 - docs/reference/ingest/processors/grok.asciidoc | 2 -- 2 files changed, 3 deletions(-) diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index fcece261bd440..bfd84d564ef7f 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -112,7 +112,6 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", - "_type" : "_type", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index b266879e40b16..39c811be03cea 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -193,7 +193,6 @@ response: "docs": [ { "doc": { - "_type": "_type", "_index": "_index", "_id": "_id", "_source": { @@ -254,7 +253,6 @@ POST _ingest/pipeline/_simulate "docs": [ { "doc": { - "_type": "_type", "_index": "_index", "_id": "_id", "_source": { From b9fb1278daf67e4607df5deea51cc0763ce9167d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 15:00:05 -0700 Subject: [PATCH 09/13] Revert "Omit _type from response if not present in request" This reverts commit 7a3a08c6ba89bcf8195ac8c5c8e3a8ad7cf58609. --- .../elasticsearch/action/ingest/SimulatePipelineRequest.java | 3 ++- .../action/ingest/SimulatePipelineRequestParsingTests.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 064c76adf3f25..2d83e65a26311 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -188,7 +188,8 @@ private static List parseDocs(Map config) { deprecationLogger.deprecatedAndMaybeLog("type_in_pipeline_simulation", "Specifying _type in pipeline simulation requests is deprecated."); } - String type = (String) dataMap.get(MetaData.TYPE.getFieldName()); + String type = ConfigurationUtils.readStringOrIntProperty(null, null, + dataMap, MetaData.TYPE.getFieldName(), "_type"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 4188a59431182..03076e760a9c9 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.ingest.TestProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; +import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; @@ -100,6 +101,8 @@ private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Ex expectedDoc.put(INDEX.getFieldName(), index); if (useExplicitType) { expectedDoc.put(TYPE.getFieldName(), type); + } else { + expectedDoc.put(TYPE.getFieldName(), "_type"); } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); @@ -163,6 +166,8 @@ private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws String value = randomAlphaOfLengthBetween(1, 10); doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); + } else { + expectedDoc.put(field.getFieldName(), "_type"); } } else { if (randomBoolean()) { From 1a5949f3211c3529580b0db206e950039e0ab25f Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 15:00:09 -0700 Subject: [PATCH 10/13] Revert "Remove _type from docs entirely" This reverts commit acc857fae55d93516f7514cd4f6911b491965ba7. --- docs/reference/ingest/apis/simulate-pipeline.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index 37a25f9cf64fe..c5ac6ab2f29e5 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -161,6 +161,7 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", + "_type": "_type", "_id": "id", "_source": { "foo": "rab" @@ -214,6 +215,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_type", "_source": { "field2": "_value2", "foo": "rab" @@ -227,6 +229,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_type", "_source": { "field3": "_value3", "field2": "_value2", From 16ae2c56f5681e0571937b9149a548f2f29d62a6 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 15:00:10 -0700 Subject: [PATCH 11/13] Revert "Fix docs to remove _type as needed" This reverts commit 8d3ee4d26826205da640491636f7bb58ec0b002b. --- docs/reference/ingest/processors/date-index-name.asciidoc | 1 + docs/reference/ingest/processors/grok.asciidoc | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index bfd84d564ef7f..fcece261bd440 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -112,6 +112,7 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", + "_type" : "_type", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index 39c811be03cea..b266879e40b16 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -193,6 +193,7 @@ response: "docs": [ { "doc": { + "_type": "_type", "_index": "_index", "_id": "_id", "_source": { @@ -253,6 +254,7 @@ POST _ingest/pipeline/_simulate "docs": [ { "doc": { + "_type": "_type", "_index": "_index", "_id": "_id", "_source": { From 395c533855fa2fbcf0318d5e48b748abf5a0cfa8 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 15:51:05 -0700 Subject: [PATCH 12/13] Review feedback, change to _doc --- .../java/org/elasticsearch/client/IngestClientIT.java | 1 - .../documentation/IngestClientDocumentationIT.java | 8 ++++---- docs/reference/ingest/apis/simulate-pipeline.asciidoc | 9 ++++++--- .../reference/ingest/processors/date-index-name.asciidoc | 2 +- docs/reference/ingest/processors/grok.asciidoc | 4 ++-- .../action/ingest/SimulatePipelineRequest.java | 6 +++--- .../ingest/SimulatePipelineRequestParsingTests.java | 8 ++++---- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 84bf43ab019d5..1c10f65fb3677 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -130,7 +130,6 @@ private void testSimulatePipeline(boolean isVerbose, { builder.startObject() .field("_index", "index") - .field("_type", "doc") .field("_id", "doc_" + 1) .startObject("_source").field("foo", "rab_" + 1).field("rank", rankValue).endObject() .endObject(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index 00bee27807f5f..df27b1f1c1a41 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -296,8 +296,8 @@ public void testSimulatePipeline() throws IOException { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( @@ -353,8 +353,8 @@ public void testSimulatePipelineAsync() throws Exception { "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + "}," + "\"docs\":[" + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + - "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + "]" + "}"; SimulatePipelineRequest request = new SimulatePipelineRequest( diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index c5ac6ab2f29e5..deb464eac7a53 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -92,6 +92,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_doc", "_source": { "field2": "_value", "foo": "bar" @@ -105,6 +106,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_doc", "_source": { "field2": "_value", "foo": "rab" @@ -161,7 +163,6 @@ POST _ingest/pipeline/_simulate?verbose }, { "_index": "index", - "_type": "_type", "_id": "id", "_source": { "foo": "rab" @@ -184,6 +185,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_doc", "_source": { "field2": "_value2", "foo": "bar" @@ -197,6 +199,7 @@ Response: "doc": { "_id": "id", "_index": "index", + "_type": "_doc", "_source": { "field3": "_value3", "field2": "_value2", @@ -215,7 +218,7 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_type", + "_type": "_doc", "_source": { "field2": "_value2", "foo": "rab" @@ -229,7 +232,7 @@ Response: "doc": { "_id": "id", "_index": "index", - "_type": "_type", + "_type": "_doc", "_source": { "field3": "_value3", "field2": "_value2", diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index fcece261bd440..6dd54dab056e8 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -112,7 +112,7 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", - "_type" : "_type", + "_type" : "_doc", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index b266879e40b16..f6f5fb3c92881 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -193,7 +193,7 @@ response: "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { @@ -254,7 +254,7 @@ POST _ingest/pipeline/_simulate "docs": [ { "doc": { - "_type": "_type", + "_type": "_doc", "_index": "_index", "_id": "_id", "_source": { diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 2d83e65a26311..eb15b56db31cc 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -185,11 +185,11 @@ private static List parseDocs(Map config) { String index = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"); if (dataMap.containsKey(MetaData.TYPE.getFieldName())) { - deprecationLogger.deprecatedAndMaybeLog("type_in_pipeline_simulation", - "Specifying _type in pipeline simulation requests is deprecated."); + deprecationLogger.deprecatedAndMaybeLog("simulate_pipeline_with_types", + "[types removal] specifying _type in pipeline simulation requests is deprecated"); } String type = ConfigurationUtils.readStringOrIntProperty(null, null, - dataMap, MetaData.TYPE.getFieldName(), "_type"); + dataMap, MetaData.TYPE.getFieldName(), "_doc"); String id = ConfigurationUtils.readStringOrIntProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"); String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 03076e760a9c9..01185cc800cab 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -102,7 +102,7 @@ private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Ex if (useExplicitType) { expectedDoc.put(TYPE.getFieldName(), type); } else { - expectedDoc.put(TYPE.getFieldName(), "_type"); + expectedDoc.put(TYPE.getFieldName(), "_doc"); } expectedDoc.put(ID.getFieldName(), id); expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue)); @@ -127,7 +127,7 @@ private void innerTestParseUsingPipelineStore(boolean useExplicitType) throws Ex assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1)); if (useExplicitType) { - assertWarnings("Specifying _type in pipeline simulation requests is deprecated."); + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); } } @@ -167,7 +167,7 @@ private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws doc.put(field.getFieldName(), value); expectedDoc.put(field.getFieldName(), value); } else { - expectedDoc.put(field.getFieldName(), "_type"); + expectedDoc.put(field.getFieldName(), "_doc"); } } else { if (randomBoolean()) { @@ -236,7 +236,7 @@ private void innerTestParseWithProvidedPipeline(boolean useExplicitType) throws assertThat(actualRequest.getPipeline().getDescription(), nullValue()); assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors)); if (useExplicitType) { - assertWarnings("Specifying _type in pipeline simulation requests is deprecated."); + assertWarnings("[types removal] specifying _type in pipeline simulation requests is deprecated"); } } From beaa82e2b821f5f6d5f46aba85d170665d363b3d Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Fri, 1 Feb 2019 17:13:43 -0700 Subject: [PATCH 13/13] Unused import --- .../action/ingest/SimulatePipelineRequestParsingTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java index 01185cc800cab..8e313e7cdbb1a 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.ingest.TestProcessor; import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList;