diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt index cf8f1ff2560b5..6b6c2e32f1f75 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt @@ -8,18 +8,25 @@ # The whitelist for the fields api -# API class org.elasticsearch.script.field.Field @dynamic_type { String getName() boolean isEmpty() int size() } +class org.elasticsearch.script.field.EmptyField @dynamic_type { + def get(def) + def get(int, def) +} + class org.elasticsearch.script.DocBasedScript { org.elasticsearch.script.field.Field field(String) } class org.elasticsearch.script.field.DelegateDocValuesField @dynamic_type { - def getValue(def) - List getValues() } + +class org.elasticsearch.script.field.BinaryDocValuesField @dynamic_type { + ByteBuffer get(ByteBuffer) + ByteBuffer get(int, ByteBuffer) +} \ No newline at end of file diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml index d480a1e722d55..c4ae4fbf38cba 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml @@ -157,31 +157,4 @@ setup: - match: { error.failed_shards.0.reason.type: "script_exception" } - match: { error.failed_shards.0.reason.reason: "compile error" } ---- -"Scripted Field with error accessing an unsupported field via the script fields api": - - do: - catch: bad_request - search: - rest_total_hits_as_int: true - body: - script_fields: - bar: - script: - source: "field('foo').getValue('')" - - - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" } - - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead"} - - - do: - catch: bad_request - search: - rest_total_hits_as_int: true - body: - script_fields: - bar: - script: - source: "field('foo').getValues()" - - - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" } - - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead" } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 29d526a7c7187..8f00cdb90c106 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -440,3 +440,25 @@ setup: script: source: "doc['token_count'].value" - match: { hits.hits.0.fields.field.0: 5 } + +--- +"empty": + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + field: + script: + source: "int value = field('dne').get(1); value" + - match: { hits.hits.0.fields.field.0: 1} + + - do: + search: + rest_total_hits_as_int: true + body: + script_fields: + field: + script: + source: "int value = field('dne').get(1, 1); value" + - match: { hits.hits.0.fields.field.0: 1 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml index d6871412a73c9..ba8d15d82ae0c 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml @@ -20,7 +20,30 @@ index: test id: 1 body: - binary: U29tZSBiaW5hcnkgYmxvYg== + binary: "U29tZSBiaW5hcnkgYmxvYg==" + + - do: + #set the header so we won't randomize it + headers: + Content-Type: application/json + index: + index: test + id: 2 + body: + binary: [ + "U29tZSBiaW5hcnkgYmxvYg==", + "MTIzNA==", + "dGVzdA==" + ] + + - do: + #set the header so we won't randomize it + headers: + Content-Type: application/json + index: + index: test + id: 3 + body: {} - do: indices.refresh: {} @@ -31,9 +54,55 @@ script_fields: field1: script: - source: "doc['binary'].get(0).utf8ToString()" + source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].get(0).utf8ToString()" field2: script: - source: "doc['binary'].value.utf8ToString()" + source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].value.utf8ToString()" - match: { hits.hits.0.fields.field1.0: "Some binary blob" } - match: { hits.hits.0.fields.field2.0: "Some binary blob" } + + - do: + search: + body: + script_fields: + field1: + script: + source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.get(0)" + field2: + script: + source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.get(0)" + field3: + script: + source: "int total = 0; for (value in field('binary')) {total += value.get(0)} total" + - match: { hits.hits.0.fields.field1.0: 83 } + - match: { hits.hits.0.fields.field2.0: 83 } + - match: { hits.hits.0.fields.field3.0: 83 } + - match: { hits.hits.1.fields.field1.0: 49 } + - match: { hits.hits.1.fields.field2.0: 49 } + - match: { hits.hits.1.fields.field3.0: 248 } + - match: { hits.hits.2.fields.field1.0: -1 } + - match: { hits.hits.2.fields.field2.0: -1 } + - match: { hits.hits.2.fields.field3.0: 0 } + + - do: + search: + body: + script_fields: + field1: + script: + source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.limit()" + field2: + script: + source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.limit()" + field3: + script: + source: "int total = 0; for (ByteBuffer value : field('binary')) {total += value.limit()} total" + - match: { hits.hits.0.fields.field1.0: 16 } + - match: { hits.hits.0.fields.field2.0: 16 } + - match: { hits.hits.0.fields.field3.0: 16 } + - match: { hits.hits.1.fields.field1.0: 4 } + - match: { hits.hits.1.fields.field2.0: 4 } + - match: { hits.hits.1.fields.field3.0: 24 } + - match: { hits.hits.2.fields.field1.0: -1 } + - match: { hits.hits.2.fields.field2.0: -1 } + - match: { hits.hits.2.fields.field3.0: 0 } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 9e2a970d13e15..bf80100eced03 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -527,7 +527,7 @@ private static class ScaledFloatLeafFieldData implements LeafNumericFieldData { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Doubles(getDoubleValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java index efb9468aa631f..6075d5db84106 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java @@ -52,7 +52,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex IpFieldScript script = leafFactory.newInstance(context); return new BinaryScriptLeafFieldData() { @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new IpScriptDocValues(getBytesValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 85f938e3b9e1e..0e5d5551aee4e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -23,7 +23,7 @@ public interface LeafFieldData extends Accountable, Releasable { /** * Returns an {@code Field} for use in accessing field values in scripting. */ - DocValuesField getScriptField(String name); + DocValuesField getScriptField(String name); /** * Return a String representation of the values. diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index af5d9038478f2..d6afb24ab7067 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.script.field.BinaryDocValuesField; import java.io.IOException; import java.time.Instant; @@ -585,30 +586,33 @@ public final String getValue() { } } - public static final class BytesRefs extends BinaryScriptDocValues { + public static final class BytesRefs extends ScriptDocValues { - public BytesRefs(SortedBinaryDocValues in) { - super(in); + private final BinaryDocValuesField binaryDocValuesField; + + public BytesRefs(BinaryDocValuesField binaryDocValuesField) { + this.binaryDocValuesField = binaryDocValuesField; } @Override - public BytesRef get(int index) { - if (count == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " - + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } - /** - * We need to make a copy here because {@link BinaryScriptDocValues} might reuse the - * returned value and the same instance might be used to - * return values from multiple documents. - **/ - return values[index].toBytesRef(); + public void setNextDocId(int docId) throws IOException { + throw new UnsupportedOperationException(); } public BytesRef getValue() { + throwIfEmpty(); return get(0); } + + @Override + public BytesRef get(int index) { + throwIfEmpty(); + return binaryDocValuesField.getInternal(index); + } + + @Override + public int size() { + return binaryDocValuesField.size(); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java index 31e1e81c22396..641801ad32a5e 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java @@ -44,7 +44,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex StringFieldScript script = leafFactory.newInstance(context); return new BinaryScriptLeafFieldData() { @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java index 70b46f26f80dc..fd5018dff3bae 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java @@ -27,7 +27,7 @@ public final SortedBinaryDocValues getBytesValues() { } @Override - public final DocValuesField getScriptField(String name) { + public final DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.GeoPoints(getGeoPointValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java index cd8661eca2e30..ff479cde06669 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java @@ -35,7 +35,7 @@ protected AbstractLeafOrdinalsFieldData(Function getScriptField(String name) { return new DelegateDocValuesField(scriptFunction.apply(getOrdinalsValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java index 613d289f12f1a..b7db7ba8ee54a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java @@ -45,7 +45,7 @@ public SortedBinaryDocValues getBytesValues() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java index e39344963a4e4..b167b4242dec8 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java @@ -9,8 +9,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.BinaryDocValues; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.script.field.BinaryDocValuesField; import org.elasticsearch.script.field.DocValuesField; final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData { @@ -19,7 +18,7 @@ final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData { } @Override - public DocValuesField getScriptField(String name) { - return new DelegateDocValuesField(new ScriptDocValues.BytesRefs(getBytesValues()), name); + public DocValuesField getScriptField(String name) { + return new BinaryDocValuesField(getBytesValues(), name); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index a7991f5e872ea..3ab0ecd539eeb 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -41,7 +41,7 @@ public long ramBytesUsed() { } @Override - public final DocValuesField getScriptField(String name) { + public final DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Doubles(getDoubleValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index 5219575befd05..877c1d0df84b9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -44,7 +44,7 @@ public long ramBytesUsed() { } @Override - public final DocValuesField getScriptField(String name) { + public final DocValuesField getScriptField(String name) { switch (numericType) { // for now, dates and nanoseconds are treated the same, which also means, that the precision is only on millisecond level case DATE: diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java index 19d13383347de..d1861a23c16e7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java @@ -19,7 +19,7 @@ final class StringBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index fd61bdd1cd9ce..c44d978d26e46 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -218,7 +218,7 @@ public long ramBytesUsed() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java index 6fd629064747f..ce4ec5e806d36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java @@ -79,7 +79,7 @@ public void close() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION.apply(getOrdinalsValues()), name); } diff --git a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java index 6a05eb0b70ffb..ae26532ddfc88 100644 --- a/server/src/main/java/org/elasticsearch/script/DocBasedScript.java +++ b/server/src/main/java/org/elasticsearch/script/DocBasedScript.java @@ -23,14 +23,14 @@ public DocBasedScript(DocReader docReader) { this.docReader = docReader; } - public Field field(String fieldName) { + public Field field(String fieldName) { if (docReader == null) { return new EmptyField(fieldName); } return docReader.field(fieldName); } - public Stream fields(String fieldGlob) { + public Stream> fields(String fieldGlob) { if (docReader == null) { return Stream.empty(); } diff --git a/server/src/main/java/org/elasticsearch/script/DocReader.java b/server/src/main/java/org/elasticsearch/script/DocReader.java index 17ea73c576026..19f6f157f542b 100644 --- a/server/src/main/java/org/elasticsearch/script/DocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocReader.java @@ -22,10 +22,10 @@ */ public interface DocReader { /** New-style field access */ - Field field(String fieldName); + Field field(String fieldName); /** New-style field iterator */ - Stream fields(String fieldGlob); + Stream> fields(String fieldGlob); /** Set the underlying docId */ void setDocument(int docID); diff --git a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java index 3b293850a8346..24cd38bfbe7c6 100644 --- a/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java +++ b/server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java @@ -39,7 +39,7 @@ public DocValuesDocReader(SearchLookup searchLookup, LeafReaderContext leafConte } @Override - public Field field(String fieldName) { + public Field field(String fieldName) { LeafDocLookup leafDocLookup = leafSearchLookup.doc(); if (leafDocLookup.containsKey(fieldName) == false) { @@ -50,7 +50,7 @@ public Field field(String fieldName) { } @Override - public Stream fields(String fieldGlob) { + public Stream> fields(String fieldGlob) { throw new UnsupportedOperationException("not implemented"); } diff --git a/server/src/main/java/org/elasticsearch/script/field/BinaryDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/BinaryDocValuesField.java new file mode 100644 index 0000000000000..565afea2b3ade --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/BinaryDocValuesField.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script.field; + +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.ScriptDocValues.BytesRefs; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.NoSuchElementException; + +public class BinaryDocValuesField implements DocValuesField { + + private final SortedBinaryDocValues input; + private final String name; + + private BytesRefBuilder[] values = new BytesRefBuilder[0]; + private int count; + + // used for backwards compatibility for old-style "doc" access + // as a delegate to this field class + private BytesRefs bytesRefs = null; + + public BinaryDocValuesField(SortedBinaryDocValues input, String name) { + this.input = input; + this.name = name; + } + + @Override + public void setNextDocId(int docId) throws IOException { + if (input.advanceExact(docId)) { + resize(input.docValueCount()); + for (int i = 0; i < count; i++) { + // We need to make a copy here, because BytesBinaryDVLeafFieldData's SortedBinaryDocValues + // implementation reuses the returned BytesRef. Otherwise, we would end up with the same BytesRef + // instance for all slots in the values array. + values[i].copyBytes(input.nextValue()); + } + } else { + resize(0); + } + } + + private void resize(int newSize) { + count = newSize; + + if (newSize > values.length) { + int oldLength = values.length; + values = ArrayUtil.grow(values, count); + + for (int i = oldLength; i < values.length; ++i) { + values[i] = new BytesRefBuilder(); + } + } + } + + @Override + public ScriptDocValues getScriptDocValues() { + if (bytesRefs == null) { + bytesRefs = new BytesRefs(this); + } + + return bytesRefs; + } + + // this method is required to support the ByteRef return values + // for the old-style "doc" access in ScriptDocValues + public BytesRef getInternal(int index) { + return values[index].toBytesRef(); + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isEmpty() { + return count == 0; + } + + @Override + public int size() { + return count; + } + + protected ByteBuffer toWrapped(int index) { + return ByteBuffer.wrap(values[index].toBytesRef().bytes); + } + + public ByteBuffer get(ByteBuffer defaultValue) { + return get(0, defaultValue); + } + + public ByteBuffer get(int index, ByteBuffer defaultValue) { + if (isEmpty() || index < 0 || index >= count) { + return defaultValue; + } + + return toWrapped(index); + } + + @Override + public Iterator iterator() { + return new Iterator() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < count; + } + + @Override + public ByteBuffer next() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + + return toWrapped(index++); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/DelegateDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/DelegateDocValuesField.java index f84f4b4446f1b..b90920f3fec1f 100644 --- a/server/src/main/java/org/elasticsearch/script/field/DelegateDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/DelegateDocValuesField.java @@ -11,13 +11,13 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import java.io.IOException; -import java.util.List; +import java.util.Iterator; /** * A default {@link Field} to provide {@code ScriptDocValues} for fields * that are not supported by the script fields api. */ -public class DelegateDocValuesField implements DocValuesField { +public class DelegateDocValuesField implements DocValuesField { private final ScriptDocValues scriptDocValues; private final String name; @@ -39,24 +39,21 @@ public ScriptDocValues getScriptDocValues() { @Override public String getName() { - throw new UnsupportedOperationException("field [" + name + "] is not supported through the fields api, use [doc] instead"); + return name; } @Override public boolean isEmpty() { - throw new UnsupportedOperationException("field [" + name + "] is not supported through the fields api, use [doc] instead"); + throw new UnsupportedOperationException(); } @Override public int size() { - throw new UnsupportedOperationException("field [" + name + "] is not supported through the fields api, use [doc] instead"); + throw new UnsupportedOperationException(); } - public Object getValue(Object defaultValue) { - throw new UnsupportedOperationException("field [" + name + "] is not supported through the fields api, use [doc] instead"); - } - - public List getValues() { - throw new UnsupportedOperationException("field [" + name + "] is not supported through the fields api, use [doc] instead"); + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); } } diff --git a/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java index 9061cf81ec5a9..34336e632a731 100644 --- a/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/DocValuesField.java @@ -12,7 +12,7 @@ import java.io.IOException; -public interface DocValuesField extends Field { +public interface DocValuesField extends Field { /** Set the current document ID. */ void setNextDocId(int docId) throws IOException; diff --git a/server/src/main/java/org/elasticsearch/script/field/EmptyField.java b/server/src/main/java/org/elasticsearch/script/field/EmptyField.java index c2b7252af5bfd..2ca171acf6c40 100644 --- a/server/src/main/java/org/elasticsearch/script/field/EmptyField.java +++ b/server/src/main/java/org/elasticsearch/script/field/EmptyField.java @@ -8,10 +8,13 @@ package org.elasticsearch.script.field; +import java.util.Iterator; +import java.util.NoSuchElementException; + /** - * Script field with no mapping, always returns {@code defaultValue}. + * A script {@code Field} with no mapping, always returns {@code defaultValue}. */ -public class EmptyField implements Field { +public class EmptyField implements Field { private final String name; @@ -33,4 +36,27 @@ public boolean isEmpty() { public int size() { return 0; } + + public Object get(Object defaultValue) { + return get(0, defaultValue); + } + + public Object get(int index, Object defaultValue) { + return defaultValue; + } + + @Override + public Iterator iterator() { + return new Iterator<>() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public Object next() { + throw new NoSuchElementException(); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/script/field/Field.java b/server/src/main/java/org/elasticsearch/script/field/Field.java index 5ba4437bcca2c..cb03bb257b44f 100644 --- a/server/src/main/java/org/elasticsearch/script/field/Field.java +++ b/server/src/main/java/org/elasticsearch/script/field/Field.java @@ -9,7 +9,7 @@ package org.elasticsearch.script.field; /** A field in a document accessible via scripting. */ -public interface Field { +public interface Field extends Iterable { /** Returns the name of this field. */ String getName(); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 98f7e85c31723..76cd231a4d250 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -31,7 +31,7 @@ public class LeafDocLookup implements Map> { private int docId = -1; - private final Map localCacheScriptFieldData = new HashMap<>(4); + private final Map> localCacheScriptFieldData = new HashMap<>(4); LeafDocLookup( Function fieldTypeLookup, @@ -47,8 +47,8 @@ public void setDocument(int docId) { this.docId = docId; } - public DocValuesField getScriptField(String fieldName) { - DocValuesField field = localCacheScriptFieldData.get(fieldName); + public DocValuesField getScriptField(String fieldName) { + DocValuesField field = localCacheScriptFieldData.get(fieldName); if (field == null) { final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); @@ -59,9 +59,9 @@ public DocValuesField getScriptField(String fieldName) { // Load the field data on behalf of the script. Otherwise, it would require // additional permissions to deal with pagedbytes/ramusagestimator/etc. - field = AccessController.doPrivileged(new PrivilegedAction() { + field = AccessController.doPrivileged(new PrivilegedAction>() { @Override - public DocValuesField run() { + public DocValuesField run() { return fieldDataLookup.apply(fieldType).load(reader).getScriptField(fieldName); } }); @@ -87,7 +87,7 @@ public ScriptDocValues get(Object key) { @Override public boolean containsKey(Object key) { String fieldName = key.toString(); - DocValuesField docValuesField = localCacheScriptFieldData.get(fieldName); + DocValuesField docValuesField = localCacheScriptFieldData.get(fieldName); return docValuesField != null || fieldTypeLookup.apply(fieldName) != null; } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java index e2717d83af7cf..44ef2b4fef53c 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/BinaryDVFieldDataTests.java @@ -17,10 +17,12 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.script.field.BinaryDocValuesField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -114,29 +116,29 @@ public void testDocValue() throws Exception { assertEquals(bytesList2.get(0), bytesValues.nextValue()); assertEquals(bytesList2.get(1), bytesValues.nextValue()); - // Test whether ScriptDocValues.BytesRefs makes a deepcopy + // Test whether BinaryDocValuesField makes a deepcopy fieldData = indexFieldData.load(reader); - ScriptDocValues scriptValues = fieldData.getScriptField("test").getScriptDocValues(); - Object[][] retValues = new BytesRef[4][0]; + BinaryDocValuesField binaryDocValuesField = (BinaryDocValuesField) fieldData.getScriptField("test"); + ByteBuffer[][] retValues = new ByteBuffer[4][]; for (int i = 0; i < 4; i++) { - scriptValues.setNextDocId(i); - retValues[i] = new BytesRef[scriptValues.size()]; + binaryDocValuesField.setNextDocId(i); + retValues[i] = new ByteBuffer[binaryDocValuesField.size()]; for (int j = 0; j < retValues[i].length; j++) { - retValues[i][j] = scriptValues.get(j); + retValues[i][j] = binaryDocValuesField.get(j, null); } } assertEquals(2, retValues[0].length); - assertEquals(bytesList1.get(0), retValues[0][0]); - assertEquals(bytesList1.get(1), retValues[0][1]); + assertArrayEquals(bytesList1.get(0).bytes, retValues[0][0].array()); + assertArrayEquals(bytesList1.get(1).bytes, retValues[0][1].array()); assertEquals(1, retValues[1].length); - assertEquals(bytes1, retValues[1][0]); + assertArrayEquals(bytes1.bytes, retValues[1][0].array()); assertEquals(0, retValues[2].length); assertEquals(2, retValues[3].length); - assertEquals(bytesList2.get(0), retValues[3][0]); - assertEquals(bytesList2.get(1), retValues[3][1]); + assertArrayEquals(bytesList2.get(0).bytes, retValues[3][0].array()); + assertArrayEquals(bytesList2.get(1).bytes, retValues[3][1].array()); } private static BytesRef randomBytes() { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index c2dad776a5459..9b503346f8daf 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -498,7 +498,7 @@ public ValuesSourceType getValuesSourceType() { public LeafFieldData load(LeafReaderContext context) { return new LeafFieldData() { @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new ScriptDocValues() { String value; diff --git a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java index b192663d2b05d..8b9fbd7a86bf8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java @@ -91,7 +91,7 @@ public LeafFieldData load(LeafReaderContext context) { return new LeafFieldData() { @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { throw new UnsupportedOperationException(UNSUPPORTED); } @@ -209,7 +209,7 @@ public double nextValue() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { throw new UnsupportedOperationException(UNSUPPORTED); } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 1240221dbccb4..df4fecdaaebfb 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -207,7 +207,7 @@ public HistogramValue histogram() throws IOException { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { throw new UnsupportedOperationException("The [" + CONTENT_TYPE + "] field does not " + "support scripts"); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 9a0da27b54a29..995e5f70de845 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -421,7 +421,7 @@ public double nextValue() throws IOException { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { // getAggregateMetricValues returns all metric as doubles, including `value_count` return new DelegateDocValuesField(new ScriptDocValues.Doubles(getAggregateMetricValues(defaultMetric)), name); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java index 2c854f432cf17..3633b423bdc72 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongDocValuesField.java @@ -17,11 +17,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; +import java.util.PrimitiveIterator; import static org.elasticsearch.search.DocValueFormat.MASK_2_63; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; -public class UnsignedLongDocValuesField implements UnsignedLongField, DocValuesField { +public class UnsignedLongDocValuesField implements UnsignedLongField, DocValuesField { private final SortedNumericDocValues input; private final String name; @@ -50,6 +52,11 @@ public void setNextDocId(int docId) throws IOException { } } + private void resize(int newSize) { + count = newSize; + values = ArrayUtil.grow(values, count); + } + @Override public ScriptDocValues getScriptDocValues() { if (unsignedLongScriptDocValues == null) { @@ -59,11 +66,6 @@ public ScriptDocValues getScriptDocValues() { return unsignedLongScriptDocValues; } - protected void resize(int newSize) { - count = newSize; - values = ArrayUtil.grow(values, count); - } - @Override public String getName() { return name; @@ -116,6 +118,32 @@ public long getValue(int index, long defaultValue) { return toFormatted(index); } + @Override + public PrimitiveIterator.OfLong iterator() { + return new PrimitiveIterator.OfLong() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < count; + } + + @Override + public Long next() { + return nextLong(); + } + + @Override + public long nextLong() { + if (hasNext() == false) { + throw new NoSuchElementException(); + } + + return toFormatted(index++); + } + }; + } + protected BigInteger toBigInteger(int index) { return BigInteger.valueOf(toFormatted(index)).and(BIGINTEGER_2_64_MINUS_ONE); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java index 35843fc35cf0f..971261ea01f2b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongField.java @@ -12,7 +12,7 @@ import java.math.BigInteger; import java.util.List; -public interface UnsignedLongField extends Field { +public interface UnsignedLongField extends Field { /** Return all the values as a {@code List}. */ List getValues(); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java index 2072a23e16a54..eb18b039dc0af 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongLeafFieldData.java @@ -73,7 +73,7 @@ public int docValueCount() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { return new UnsignedLongDocValuesField(getLongValues(), name); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java index 768ff44ba853e..69018c3ed0803 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/fielddata/plain/AbstractAtomicGeoShapeShapeFieldData.java @@ -32,7 +32,7 @@ public final SortedBinaryDocValues getBytesValues() { } @Override - public final DocValuesField getScriptField(String name) { + public final DocValuesField getScriptField(String name) { return new DelegateDocValuesField(new GeoShapeScriptValues(getGeoShapeValues()), name); } diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java index 1aca1f12f7379..e0184303e65f7 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVLeafFieldData.java @@ -54,7 +54,7 @@ public SortedBinaryDocValues getBytesValues() { } @Override - public DocValuesField getScriptField(String name) { + public DocValuesField getScriptField(String name) { try { if (indexed) { VectorValues values = reader.getVectorValues(field);