diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 1556410ca0c2c..787d9ba7c7a6e 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.IgnoredFieldMapper; +import org.elasticsearch.index.mapper.LegacyTypeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -68,7 +69,9 @@ public Status needsField(FieldInfo fieldInfo) { } // support _uid for loading older indices if ("_uid".equals(fieldInfo.name)) { - return Status.YES; + if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) { + return Status.YES; + } } // All these fields are single-valued so we can stop when the set is // empty @@ -111,8 +114,9 @@ public void stringField(FieldInfo fieldInfo, String value) { if ("_uid".equals(fieldInfo.name)) { // 5.x-only int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can - // type = value.substring(0, delimiterIndex); + String type = value.substring(0, delimiterIndex); id = value.substring(delimiterIndex + 1); + addValue(LegacyTypeFieldMapper.NAME, type); } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) { // only applies to 5.x indices that have single_type = true id = value; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java new file mode 100644 index 0000000000000..a5e0ec86db775 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.sandbox.search.DocValuesTermsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.util.Collection; +import java.util.Collections; + +/** + * Field mapper to access the legacy _type that existed in Elasticsearch 5 + */ +public class LegacyTypeFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_type"; + + public static final String CONTENT_TYPE = "_type"; + + private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + + protected LegacyTypeFieldMapper() { + super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER); + } + + static final class LegacyTypeFieldType extends TermBasedFieldType { + + LegacyTypeFieldType() { + super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public boolean isSearchable() { + // The _type field is always searchable. + return true; + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value)); + } + + @Override + public Query termsQuery(Collection values, SearchExecutionContext context) { + BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new); + return new DocValuesTermsQuery(name(), bytesRefs); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + return SortedSetDocValuesField.newSlowRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); + } + + @Override + public boolean mayExistInIndex(SearchExecutionContext context) { + return true; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + return new StoredValueFetcher(context.lookup(), NAME); + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java index 0ed5e7682a365..034f056dda993 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperRegistry.java @@ -26,6 +26,7 @@ public final class MapperRegistry { private final Map runtimeFieldParsers; private final Map metadataMapperParsers; private final Map metadataMapperParsers7x; + private final Map metadataMapperParsers5x; private final Function> fieldFilter; public MapperRegistry( @@ -40,6 +41,9 @@ public MapperRegistry( Map metadata7x = new LinkedHashMap<>(metadataMapperParsers); metadata7x.remove(NestedPathFieldMapper.NAME); this.metadataMapperParsers7x = metadata7x; + Map metadata5x = new LinkedHashMap<>(metadata7x); + metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER); + this.metadataMapperParsers5x = metadata5x; this.fieldFilter = fieldFilter; } @@ -62,8 +66,11 @@ public Map getRuntimeFieldParsers() { public Map getMetadataMapperParsers(Version indexCreatedVersion) { if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) { return metadataMapperParsers; + } else if (indexCreatedVersion.major < 6) { + return metadataMapperParsers5x; + } else { + return metadataMapperParsers7x; } - return metadataMapperParsers7x; } /** diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 5b3fb0a331367..b8ab56bf69400 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -136,7 +137,7 @@ && randomBoolean()) { String id = "testdoc" + i; expectedIds.add(id); // use multiple types for ES versions < 6.0.0 - String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + String type = getType(oldVersion, id); Request doc = new Request("PUT", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); doc.setJsonEntity(sourceForDoc(i)); @@ -146,7 +147,7 @@ && randomBoolean()) { for (int i = 0; i < extraDocs; i++) { String id = randomFrom(expectedIds); expectedIds.remove(id); - String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0); + String type = getType(oldVersion, id); Request doc = new Request("DELETE", "/test/" + type + "/" + id); doc.addParameter("refresh", "true"); oldEs.performRequest(doc); @@ -267,6 +268,10 @@ && randomBoolean()) { } } + private String getType(Version oldVersion, String id) { + return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0); + } + private static String sourceForDoc(int i) { return "{\"test\":\"test" + i + "\",\"val\":" + i + "}"; } @@ -337,7 +342,7 @@ private void restoreMountAndVerify( } // run a search against the index - assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); // mount as full copy searchable snapshot RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots() @@ -363,7 +368,7 @@ private void restoreMountAndVerify( ); // run a search against the index - assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); // mount as shared cache searchable snapshot mountSnapshotResponse = client.searchableSnapshots() @@ -378,12 +383,18 @@ private void restoreMountAndVerify( assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards()); // run a search against the index - assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository); + assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion); } @SuppressWarnings("removal") - private void assertDocs(String index, int numDocs, Set expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository) - throws IOException { + private void assertDocs( + String index, + int numDocs, + Set expectedIds, + RestHighLevelClient client, + boolean sourceOnlyRepository, + Version oldVersion + ) throws IOException { // run a search against the index SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT); logger.info(searchResponse); @@ -420,9 +431,9 @@ private void assertDocs(String index, int numDocs, Set expectedIds, Rest // check that doc values can be accessed by (reverse) sorting on numeric val field // first add mapping for field (this will be done automatically in the future) XContentBuilder mappingBuilder = JsonXContent.contentBuilder(); - mappingBuilder.startObject().startObject("properties").startObject("val"); - mappingBuilder.field("type", "long"); - mappingBuilder.endObject().endObject().endObject(); + mappingBuilder.startObject().startObject("properties"); + mappingBuilder.startObject("val").field("type", "long").endObject(); + mappingBuilder.endObject().endObject(); assertTrue( client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged() ); @@ -442,6 +453,24 @@ private void assertDocs(String index, int numDocs, Set expectedIds, Rest expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()), Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList()) ); + + if (oldVersion.before(Version.fromString("6.0.0"))) { + // search on _type and check that results contain _type information + String randomType = getType(oldVersion, randomFrom(expectedIds)); + long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count(); + searchResponse = client.search( + new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))), + RequestOptions.DEFAULT + ); + logger.info(searchResponse); + assertEquals(typeCount, searchResponse.getHits().getTotalHits().value); + for (SearchHit hit : searchResponse.getHits().getHits()) { + DocumentField typeField = hit.field("_type"); + assertNotNull(typeField); + assertThat(typeField.getValue(), instanceOf(String.class)); + assertEquals(randomType, typeField.getValue()); + } + } } }