Skip to content

Commit

Permalink
Provide access to _type in 5.x indices (#83195)
Browse files Browse the repository at this point in the history
Allows running queries against _type on 5.x indices as well as returning _type in search results.

Relates #81210
  • Loading branch information
ywelsch authored Jan 27, 2022
1 parent b42ba64 commit ac9f30a
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.index.mapper.IdFieldMapper;
import org.elasticsearch.index.mapper.IgnoredFieldMapper;
import org.elasticsearch.index.mapper.LegacyTypeFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.RoutingFieldMapper;
import org.elasticsearch.index.mapper.SourceFieldMapper;
Expand Down Expand Up @@ -68,7 +69,9 @@ public Status needsField(FieldInfo fieldInfo) {
}
// support _uid for loading older indices
if ("_uid".equals(fieldInfo.name)) {
return Status.YES;
if (requiredFields.remove(IdFieldMapper.NAME) || requiredFields.remove(LegacyTypeFieldMapper.NAME)) {
return Status.YES;
}
}
// All these fields are single-valued so we can stop when the set is
// empty
Expand Down Expand Up @@ -111,8 +114,9 @@ public void stringField(FieldInfo fieldInfo, String value) {
if ("_uid".equals(fieldInfo.name)) {
// 5.x-only
int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can
// type = value.substring(0, delimiterIndex);
String type = value.substring(0, delimiterIndex);
id = value.substring(delimiterIndex + 1);
addValue(LegacyTypeFieldMapper.NAME, type);
} else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
// only applies to 5.x indices that have single_type = true
id = value;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.index.mapper;

import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.sandbox.search.DocValuesTermsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.query.SearchExecutionContext;

import java.util.Collection;
import java.util.Collections;

/**
* Field mapper to access the legacy _type that existed in Elasticsearch 5
*/
public class LegacyTypeFieldMapper extends MetadataFieldMapper {

public static final String NAME = "_type";

public static final String CONTENT_TYPE = "_type";

private static final LegacyTypeFieldMapper INSTANCE = new LegacyTypeFieldMapper();

public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE);

protected LegacyTypeFieldMapper() {
super(new LegacyTypeFieldType(), Lucene.KEYWORD_ANALYZER);
}

static final class LegacyTypeFieldType extends TermBasedFieldType {

LegacyTypeFieldType() {
super(NAME, false, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, Collections.emptyMap());
}

@Override
public String typeName() {
return CONTENT_TYPE;
}

@Override
public boolean isSearchable() {
// The _type field is always searchable.
return true;
}

@Override
public Query termQuery(Object value, SearchExecutionContext context) {
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
}

@Override
public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new);
return new DocValuesTermsQuery(name(), bytesRefs);
}

@Override
public Query rangeQuery(
Object lowerTerm,
Object upperTerm,
boolean includeLower,
boolean includeUpper,
SearchExecutionContext context
) {
return SortedSetDocValuesField.newSlowRangeQuery(
name(),
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
upperTerm == null ? null : indexedValueForSearch(upperTerm),
includeLower,
includeUpper
);
}

@Override
public boolean mayExistInIndex(SearchExecutionContext context) {
return true;
}

@Override
public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
return new StoredValueFetcher(context.lookup(), NAME);
}
}

@Override
protected String contentType() {
return CONTENT_TYPE;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public final class MapperRegistry {
private final Map<String, RuntimeField.Parser> runtimeFieldParsers;
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers7x;
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers5x;
private final Function<String, Predicate<String>> fieldFilter;

public MapperRegistry(
Expand All @@ -40,6 +41,9 @@ public MapperRegistry(
Map<String, MetadataFieldMapper.TypeParser> metadata7x = new LinkedHashMap<>(metadataMapperParsers);
metadata7x.remove(NestedPathFieldMapper.NAME);
this.metadataMapperParsers7x = metadata7x;
Map<String, MetadataFieldMapper.TypeParser> metadata5x = new LinkedHashMap<>(metadata7x);
metadata5x.put(LegacyTypeFieldMapper.NAME, LegacyTypeFieldMapper.PARSER);
this.metadataMapperParsers5x = metadata5x;
this.fieldFilter = fieldFilter;
}

Expand All @@ -62,8 +66,11 @@ public Map<String, RuntimeField.Parser> getRuntimeFieldParsers() {
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
return metadataMapperParsers;
} else if (indexCreatedVersion.major < 6) {
return metadataMapperParsers5x;
} else {
return metadataMapperParsers7x;
}
return metadataMapperParsers7x;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
Expand Down Expand Up @@ -136,7 +137,7 @@ && randomBoolean()) {
String id = "testdoc" + i;
expectedIds.add(id);
// use multiple types for ES versions < 6.0.0
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
String type = getType(oldVersion, id);
Request doc = new Request("PUT", "/test/" + type + "/" + id);
doc.addParameter("refresh", "true");
doc.setJsonEntity(sourceForDoc(i));
Expand All @@ -146,7 +147,7 @@ && randomBoolean()) {
for (int i = 0; i < extraDocs; i++) {
String id = randomFrom(expectedIds);
expectedIds.remove(id);
String type = "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Murmur3HashFunction.hash(id) % 2 : 0);
String type = getType(oldVersion, id);
Request doc = new Request("DELETE", "/test/" + type + "/" + id);
doc.addParameter("refresh", "true");
oldEs.performRequest(doc);
Expand Down Expand Up @@ -267,6 +268,10 @@ && randomBoolean()) {
}
}

private String getType(Version oldVersion, String id) {
return "doc" + (oldVersion.before(Version.fromString("6.0.0")) ? Math.abs(Murmur3HashFunction.hash(id) % 2) : 0);
}

private static String sourceForDoc(int i) {
return "{\"test\":\"test" + i + "\",\"val\":" + i + "}";
}
Expand Down Expand Up @@ -337,7 +342,7 @@ private void restoreMountAndVerify(
}

// run a search against the index
assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository);
assertDocs("restored_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);

// mount as full copy searchable snapshot
RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots()
Expand All @@ -363,7 +368,7 @@ private void restoreMountAndVerify(
);

// run a search against the index
assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository);
assertDocs("mounted_full_copy_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);

// mount as shared cache searchable snapshot
mountSnapshotResponse = client.searchableSnapshots()
Expand All @@ -378,12 +383,18 @@ private void restoreMountAndVerify(
assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards());

// run a search against the index
assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository);
assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client, sourceOnlyRepository, oldVersion);
}

@SuppressWarnings("removal")
private void assertDocs(String index, int numDocs, Set<String> expectedIds, RestHighLevelClient client, boolean sourceOnlyRepository)
throws IOException {
private void assertDocs(
String index,
int numDocs,
Set<String> expectedIds,
RestHighLevelClient client,
boolean sourceOnlyRepository,
Version oldVersion
) throws IOException {
// run a search against the index
SearchResponse searchResponse = client.search(new SearchRequest(index), RequestOptions.DEFAULT);
logger.info(searchResponse);
Expand Down Expand Up @@ -420,9 +431,9 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
// check that doc values can be accessed by (reverse) sorting on numeric val field
// first add mapping for field (this will be done automatically in the future)
XContentBuilder mappingBuilder = JsonXContent.contentBuilder();
mappingBuilder.startObject().startObject("properties").startObject("val");
mappingBuilder.field("type", "long");
mappingBuilder.endObject().endObject().endObject();
mappingBuilder.startObject().startObject("properties");
mappingBuilder.startObject("val").field("type", "long").endObject();
mappingBuilder.endObject().endObject();
assertTrue(
client.indices().putMapping(new PutMappingRequest(index).source(mappingBuilder), RequestOptions.DEFAULT).isAcknowledged()
);
Expand All @@ -442,6 +453,24 @@ private void assertDocs(String index, int numDocs, Set<String> expectedIds, Rest
expectedIds.stream().sorted(Comparator.comparingInt(this::getIdAsNumeric).reversed()).collect(Collectors.toList()),
Arrays.stream(searchResponse.getHits().getHits()).map(SearchHit::getId).collect(Collectors.toList())
);

if (oldVersion.before(Version.fromString("6.0.0"))) {
// search on _type and check that results contain _type information
String randomType = getType(oldVersion, randomFrom(expectedIds));
long typeCount = expectedIds.stream().filter(idd -> getType(oldVersion, idd).equals(randomType)).count();
searchResponse = client.search(
new SearchRequest(index).source(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("_type", randomType))),
RequestOptions.DEFAULT
);
logger.info(searchResponse);
assertEquals(typeCount, searchResponse.getHits().getTotalHits().value);
for (SearchHit hit : searchResponse.getHits().getHits()) {
DocumentField typeField = hit.field("_type");
assertNotNull(typeField);
assertThat(typeField.getValue(), instanceOf(String.class));
assertEquals(randomType, typeField.getValue());
}
}
}
}

Expand Down

0 comments on commit ac9f30a

Please sign in to comment.