From 95bc9fb577880632a38fadcda613b64b0425e7ad Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 6 Jan 2022 06:47:33 -0600 Subject: [PATCH] Script: fields API for x-pack constant_keyword (#82292) * Script: fields API for x-pack constant_keyword Adds the fields API for the constant_keyword field mapper Moves implementation to `AbstractKeywordDocValuesField`, allowing code sharing with `KeywordDocValuesField`. API: ``` field('const').get('default') field('const').get(0, 'default') ``` Refs: #79105 --- docs/changelog/82292.yaml | 5 + .../field/AbstractKeywordDocValuesField.java | 131 ++++++++++++++++++ .../script/field/KeywordDocValuesField.java | 118 +--------------- .../mapper-constant-keyword/build.gradle | 4 +- .../ConstantKeywordDocValuesField.java | 17 +++ .../ConstantKeywordPainlessExtension.java | 37 +++++ .../mapper/ConstantKeywordFieldMapper.java | 8 +- ...asticsearch.painless.spi.PainlessExtension | 1 + ...rg.elasticsearch.xpack.constantkeyword.txt | 12 ++ .../ConstantKeywordClientYamlTestSuiteIT.java | 27 ++++ .../rest-api-spec/test/10_script_values.yml | 45 ++++++ 11 files changed, 282 insertions(+), 123 deletions(-) create mode 100644 docs/changelog/82292.yaml create mode 100644 server/src/main/java/org/elasticsearch/script/field/AbstractKeywordDocValuesField.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordDocValuesField.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordPainlessExtension.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt create mode 100644 x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordClientYamlTestSuiteIT.java create mode 100644 x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml diff --git a/docs/changelog/82292.yaml b/docs/changelog/82292.yaml new file mode 100644 index 0000000000000..8cc35862af452 --- /dev/null +++ b/docs/changelog/82292.yaml @@ -0,0 +1,5 @@ +pr: 82292 +summary: "Script: fields API for x-pack `constant_keyword`" +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/script/field/AbstractKeywordDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/AbstractKeywordDocValuesField.java new file mode 100644 index 0000000000000..ff35943c32e16 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/AbstractKeywordDocValuesField.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class AbstractKeywordDocValuesField implements DocValuesField, ScriptDocValues.Supplier { + + protected final SortedBinaryDocValues input; + protected final String name; + + protected BytesRefBuilder[] values = new BytesRefBuilder[0]; + protected int count; + + // used for backwards compatibility for old-style "doc" access + // as a delegate to this field class + protected ScriptDocValues.Strings strings = null; + + public AbstractKeywordDocValuesField(SortedBinaryDocValues input, String name) { + this.input = input; + this.name = name; + } + + @Override + public void setNextDocId(int docId) throws IOException { + if (input.advanceExact(docId)) { + resize(input.docValueCount()); + for (int i = 0; i < count; i++) { + // We need to make a copy here, because BytesBinaryDVLeafFieldData's SortedBinaryDocValues + // implementation reuses the returned BytesRef. Otherwise we would end up with the same BytesRef + // instance for all slots in the values array. + values[i].copyBytes(input.nextValue()); + } + } else { + resize(0); + } + } + + private void resize(int newSize) { + count = newSize; + assert count >= 0 : "size must be positive (got " + count + "): likely integer overflow?"; + if (newSize > values.length) { + final int oldLength = values.length; + values = ArrayUtil.grow(values, count); + for (int i = oldLength; i < values.length; ++i) { + values[i] = new BytesRefBuilder(); + } + } + } + + @Override + public ScriptDocValues getScriptDocValues() { + if (strings == null) { + strings = new ScriptDocValues.Strings(this); + } + + return strings; + } + + // this method is required to support the Boolean return values + // for the old-style "doc" access in ScriptDocValues + @Override + public String getInternal(int index) { + return bytesToString(values[index].toBytesRef()); + } + + protected String bytesToString(BytesRef bytesRef) { + return bytesRef.utf8ToString(); + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isEmpty() { + return count == 0; + } + + @Override + public int size() { + return count; + } + + public String get(String defaultValue) { + return get(0, defaultValue); + } + + public String get(int index, String defaultValue) { + if (isEmpty() || index < 0 || index >= count) { + return defaultValue; + } + + return bytesToString(values[index].toBytesRef()); + } + + @Override + public Iterator iterator() { + return new Iterator() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < count; + } + + @Override + public String next() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + return bytesToString(values[index++].toBytesRef()); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/KeywordDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/KeywordDocValuesField.java index b3510e71b28d4..769c85caf7049 100644 --- a/server/src/main/java/org/elasticsearch/script/field/KeywordDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/KeywordDocValuesField.java @@ -8,124 +8,10 @@ package org.elasticsearch.script.field; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefBuilder; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import java.io.IOException; -import java.util.Iterator; -import java.util.NoSuchElementException; - -public class KeywordDocValuesField implements DocValuesField, ScriptDocValues.Supplier { - - private final SortedBinaryDocValues input; - private final String name; - - private BytesRefBuilder[] values = new BytesRefBuilder[0]; - private int count; - - // used for backwards compatibility for old-style "doc" access - // as a delegate to this field class - private ScriptDocValues.Strings strings = null; - +public class KeywordDocValuesField extends AbstractKeywordDocValuesField { public KeywordDocValuesField(SortedBinaryDocValues input, String name) { - this.input = input; - this.name = name; - } - - @Override - public void setNextDocId(int docId) throws IOException { - if (input.advanceExact(docId)) { - resize(input.docValueCount()); - for (int i = 0; i < count; i++) { - // We need to make a copy here, because BytesBinaryDVLeafFieldData's SortedBinaryDocValues - // implementation reuses the returned BytesRef. Otherwise we would end up with the same BytesRef - // instance for all slots in the values array. - values[i].copyBytes(input.nextValue()); - } - } else { - resize(0); - } - } - - private void resize(int newSize) { - count = newSize; - assert count >= 0 : "size must be positive (got " + count + "): likely integer overflow?"; - if (newSize > values.length) { - final int oldLength = values.length; - values = ArrayUtil.grow(values, count); - for (int i = oldLength; i < values.length; ++i) { - values[i] = new BytesRefBuilder(); - } - } - } - - @Override - public ScriptDocValues getScriptDocValues() { - if (strings == null) { - strings = new ScriptDocValues.Strings(this); - } - - return strings; - } - - // this method is required to support the Boolean return values - // for the old-style "doc" access in ScriptDocValues - @Override - public String getInternal(int index) { - return bytesToString(values[index].toBytesRef()); - } - - protected String bytesToString(BytesRef bytesRef) { - return bytesRef.utf8ToString(); - } - - @Override - public String getName() { - return name; - } - - @Override - public boolean isEmpty() { - return count == 0; - } - - @Override - public int size() { - return count; - } - - public String get(String defaultValue) { - return get(0, defaultValue); - } - - public String get(int index, String defaultValue) { - if (isEmpty() || index < 0 || index >= count) { - return defaultValue; - } - - return bytesToString(values[index].toBytesRef()); - } - - @Override - public Iterator iterator() { - return new Iterator() { - private int index = 0; - - @Override - public boolean hasNext() { - return index < count; - } - - @Override - public String next() { - if (hasNext() == false) { - throw new NoSuchElementException(); - } - return bytesToString(values[index++].toBytesRef()); - } - }; + super(input, name); } } diff --git a/x-pack/plugin/mapper-constant-keyword/build.gradle b/x-pack/plugin/mapper-constant-keyword/build.gradle index 3b80893515753..0a9a44f7ca211 100644 --- a/x-pack/plugin/mapper-constant-keyword/build.gradle +++ b/x-pack/plugin/mapper-constant-keyword/build.gradle @@ -1,15 +1,17 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { name 'constant-keyword' description 'Module for the constant-keyword field type, which is a specialization of keyword for the case when all documents have the same value.' classname 'org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-constant-keyword' dependencies { + compileOnly project(':modules:lang-painless:spi') compileOnly project(path: xpackModule('core')) internalClusterTestImplementation(testArtifact(project(xpackModule('core')))) } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordDocValuesField.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordDocValuesField.java new file mode 100644 index 0000000000000..99c58594350ef --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordDocValuesField.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.script.field.AbstractKeywordDocValuesField; + +public class ConstantKeywordDocValuesField extends AbstractKeywordDocValuesField { + public ConstantKeywordDocValuesField(SortedBinaryDocValues input, String name) { + super(input, name); + } +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordPainlessExtension.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordPainlessExtension.java new file mode 100644 index 0000000000000..77fb895606f86 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordPainlessExtension.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.script.ScriptModule.CORE_CONTEXTS; + +public class ConstantKeywordPainlessExtension implements PainlessExtension { + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles( + ConstantKeywordPainlessExtension.class, + "org.elasticsearch.xpack.constantkeyword.txt" + ); + + @Override + public Map, List> getContextWhitelists() { + List whitelist = singletonList(WHITELIST); + Map, List> contextWhitelists = new HashMap<>(CORE_CONTEXTS.size()); + for (ScriptContext scriptContext : CORE_CONTEXTS.values()) { + contextWhitelists.put(scriptContext, whitelist); + } + return contextWhitelists; + } +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index d3f754df95059..34b3b4b08a0d1 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -37,10 +36,10 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.script.field.DelegateDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.constantkeyword.ConstantKeywordDocValuesField; import org.elasticsearch.xpack.core.termsenum.action.SimpleTermCountEnum; import org.elasticsearch.xpack.core.termsenum.action.TermCount; @@ -137,10 +136,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S value, name(), CoreValuesSourceType.KEYWORD, - (dv, n) -> new DelegateDocValuesField( - new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), - n - ) + (dv, n) -> new ConstantKeywordDocValuesField(FieldData.toString(dv), n) ); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/mapper-constant-keyword/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..9faff79a0f890 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.constantkeyword.ConstantKeywordPainlessExtension diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt b/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt new file mode 100644 index 0000000000000..ff58c75b1ff8e --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt @@ -0,0 +1,12 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the Server Side Public License, v 1; you may not use this file except +# in compliance with, at your election, the Elastic License 2.0 or the Server +# Side Public License, v 1. +# + +class org.elasticsearch.xpack.constantkeyword.ConstantKeywordDocValuesField @dynamic_type { + String get(String) + String get(int, String) +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordClientYamlTestSuiteIT.java b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..789059d9e11c0 --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/java/org/elasticsearch/xpack/constantkeyword/ConstantKeywordClientYamlTestSuiteIT.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.constantkeyword; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** Runs yaml rest tests */ +public class ConstantKeywordClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public ConstantKeywordClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml new file mode 100644 index 0000000000000..25eff2afb624b --- /dev/null +++ b/x-pack/plugin/mapper-constant-keyword/src/yamlRestTest/resources/rest-api-spec/test/10_script_values.yml @@ -0,0 +1,45 @@ +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + keyword: + type: keyword + const: + type: constant_keyword + value: the_same_everywhere + const2: + type: constant_keyword + value: the_same_but_different + + - do: + bulk: + index: test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "keyword": "abc" } + { "index": {"_id" : "2"} } + { "keyword": "def" } + +--- +"Constant Keyword Fields API": + - do: + search: + index: test + body: + sort: [ { keyword: desc } ] + script_fields: + constOne: + script: + source: "field('const').get('doremi') + '_' + $('keyword', 'dne')" + constTwo: + script: + source: "field('const2').get(0, 'fasola') + '-' + $('keyword', 'dne')" + + - match: { hits.hits.0.fields.constOne.0: "the_same_everywhere_def" } + - match: { hits.hits.0.fields.constTwo.0: "the_same_but_different-def" } + - match: { hits.hits.1.fields.constOne.0: "the_same_everywhere_abc" } + - match: { hits.hits.1.fields.constTwo.0: "the_same_but_different-abc" }