diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index d40488eaa34f8..b3b0fdb8cf850 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -141,6 +141,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 93d7f4b13380a..b21f47d8feb9b 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -92,6 +92,9 @@ public enum MergeReason { public static final String DEFAULT_MAPPING = "_default_"; public static final Setting INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope); + // maximum allowed number of nested json objects across all fields in a single document + public static final Setting INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING = + Setting.longSetting("index.mapping.nested_objects.limit", 10000L, 0, Property.Dynamic, Property.IndexScope); public static final Setting INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING = Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java index c6ed6b315a015..4a7af6cba47ea 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java @@ -305,6 +305,10 @@ public static class InternalParseContext extends ParseContext { private SeqNoFieldMapper.SequenceIDFields seqID; + private final long maxAllowedNumNestedDocs; + + private long numNestedDocs; + private final List dynamicMappers; @@ -321,6 +325,8 @@ public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperPars this.version = null; this.sourceToParse = source; this.dynamicMappers = new ArrayList<>(); + this.maxAllowedNumNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(indexSettings); + this.numNestedDocs = 0L; } @Override @@ -366,6 +372,13 @@ public Document doc() { @Override protected void addDoc(Document doc) { + numNestedDocs ++; + if (numNestedDocs > maxAllowedNumNestedDocs) { + throw new MapperParsingException( + "The number of nested documents has exceeded the allowed limit of [" + maxAllowedNumNestedDocs + "]." + + " This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting."); + } this.documents.add(doc); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 39d4de2359e78..e17d58d4a14b2 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -19,12 +19,12 @@ package org.elasticsearch.index.mapper; -import java.util.HashMap; import java.util.HashSet; import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -524,4 +524,144 @@ public void testParentObjectMapperAreNested() throws Exception { assertFalse(objectMapper.parentObjectMapperAreNested(mapperService)); } + public void testLimitNestedDocsDefaultSettings() throws Exception{ + Settings settings = Settings.builder().build(); + MapperService mapperService = createIndex("test1", settings).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + long defaultMaxNoNestedDocs = MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.get(settings); + + // parsing a doc with No. nested objects > defaultMaxNoNestedDocs fails + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + for(int i = 0; i <= defaultMaxNoNestedDocs; i++) { + docBuilder.startObject().field("f", i).endObject(); + } + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source1)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + defaultMaxNoNestedDocs + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); + } + + public void testLimitNestedDocs() throws Exception{ + // setting limit to allow only two nested objects in the whole doc + long maxNoNestedDocs = 2L; + MapperService mapperService = createIndex("test1", Settings.builder() + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // parsing a doc with 2 nested objects succeeds + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); + docBuilder.startObject().field("field1", "12").field("field2", "22").endObject(); + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); + ParsedDocument doc = docMapper.parse(source1); + assertThat(doc.docs().size(), equalTo(3)); + + // parsing a doc with 3 nested objects fails + XContentBuilder docBuilder2 = XContentFactory.jsonBuilder(); + docBuilder2.startObject(); + { + docBuilder2.startArray("nested1"); + { + docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject(); + docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject(); + docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject(); + } + docBuilder2.endArray(); + } + docBuilder2.endObject(); + SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); + } + + public void testLimitNestedDocsMultipleNestedFields() throws Exception{ + // setting limit to allow only two nested objects in the whole doc + long maxNoNestedDocs = 2L; + MapperService mapperService = createIndex("test1", Settings.builder() + .put(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey(), maxNoNestedDocs).build()).mapperService(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .startObject("nested2").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + DocumentMapper docMapper = mapperService.documentMapperParser().parse("type", new CompressedXContent(mapping)); + + // parsing a doc with 2 nested objects succeeds + XContentBuilder docBuilder = XContentFactory.jsonBuilder(); + docBuilder.startObject(); + { + docBuilder.startArray("nested1"); + { + docBuilder.startObject().field("field1", "11").field("field2", "21").endObject(); + } + docBuilder.endArray(); + docBuilder.startArray("nested2"); + { + docBuilder.startObject().field("field1", "21").field("field2", "22").endObject(); + } + docBuilder.endArray(); + } + docBuilder.endObject(); + SourceToParse source1 = SourceToParse.source("test1", "type", "1", docBuilder.bytes(), XContentType.JSON); + ParsedDocument doc = docMapper.parse(source1); + assertThat(doc.docs().size(), equalTo(3)); + + // parsing a doc with 3 nested objects fails + XContentBuilder docBuilder2 = XContentFactory.jsonBuilder(); + docBuilder2.startObject(); + { + docBuilder2.startArray("nested1"); + { + docBuilder2.startObject().field("field1", "11").field("field2", "21").endObject(); + } + docBuilder2.endArray(); + docBuilder2.startArray("nested2"); + { + docBuilder2.startObject().field("field1", "12").field("field2", "22").endObject(); + docBuilder2.startObject().field("field1", "13").field("field2", "23").endObject(); + } + docBuilder2.endArray(); + + } + docBuilder2.endObject(); + SourceToParse source2 = SourceToParse.source("test1", "type", "2", docBuilder2.bytes(), XContentType.JSON); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> docMapper.parse(source2)); + assertEquals( + "The number of nested documents has exceeded the allowed limit of [" + maxNoNestedDocs + + "]. This limit can be set by changing the [" + MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING.getKey() + + "] index level setting.", + e.getMessage() + ); + } + } diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index bc5c1def32f68..6f8f1b38d6f22 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -90,6 +90,12 @@ causing a mapping explosion: Indexing 1 document with 100 nested fields actually indexes 101 documents as each nested document is indexed as a separate hidden document. +`index.mapping.nested_objects.limit`:: + The maximum number of `nested` json objects within a single document across + all nested fields, defaults to 10000. Indexing one document with an array of + 100 objects within a nested field, will actually create 101 documents, as + each nested object will be indexed as a separate hidden document. + [float] == Dynamic mapping diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index b5b0d3394eafe..b5d96513cf745 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -201,3 +201,13 @@ Indexing a document with 100 nested fields actually indexes 101 documents as eac document is indexed as a separate document. To safeguard against ill-defined mappings the number of nested fields that can be defined per index has been limited to 50. See <>. + + +==== Limiting the number of `nested` json objects +Indexing a document with an array of 100 objects within a nested field, will actually +create 101 documents, as each nested object will be indexed as a separate document. +To prevent out of memory errors when a single document contains too many nested json +objects, the number of nested json objects that a single document may contain across all fields +has been limited to 10000. See <>. + + diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc index 4f8893829f9c7..215282c49d7bd 100644 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ b/docs/reference/migration/migrate_7_0/mappings.asciidoc @@ -7,4 +7,10 @@ The `_all` field deprecated in 6 have now been removed. ==== `index_options` for numeric fields has been removed -The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. \ No newline at end of file +The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. + +==== Limiting the number of `nested` json objects + +To safeguard against out of memory errors, the number of nested json objects within a single +document across all fields has been limited to 10000. This default limit can be changed with +the index setting `index.mapping.nested_objects.limit`. \ No newline at end of file diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml new file mode 100644 index 0000000000000..2c912a2165a83 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_nested.yml @@ -0,0 +1,41 @@ +--- +setup: + - do: + indices.create: + index: test_1 + body: + settings: + index.mapping.nested_objects.limit: 2 + mappings: + test_type: + properties: + nested1: + type: nested + +--- +"Indexing a doc with No. nested objects less or equal to index.mapping.nested_objects.limit should succeed": + - skip: + version: " - 6.99.99" + reason: index.mapping.nested_objects setting has been added in 7.0.0 + - do: + create: + index: test_1 + type: test_type + id: 1 + body: + "nested1" : [ { "foo": "bar" }, { "foo": "bar2" } ] + - match: { _version: 1} + +--- +"Indexing a doc with No. nested objects more than index.mapping.nested_objects.limit should fail": + - skip: + version: " - 6.99.99" + reason: index.mapping.nested_objects setting has been added in 7.0.0 + - do: + catch: /The number of nested documents has exceeded the allowed limit of \[2\]. This limit can be set by changing the \[index.mapping.nested_objects.limit\] index level setting\./ + create: + index: test_1 + type: test_type + id: 1 + body: + "nested1" : [ { "foo": "bar" }, { "foo": "bar2" }, { "foo": "bar3" } ]