Add BinaryDocValuesField to replace BytesRef (ScriptDocValues)

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt

@@ -20,6 +20,9 @@ class org.elasticsearch.script.DocBasedScript {
 }
 
 class org.elasticsearch.script.field.DelegateDocValuesField @dynamic_type {
-  def getValue(def)
-  List getValues()
 }
+
+class org.elasticsearch.script.field.BinaryDocValuesField @dynamic_type {
+  ByteBuffer get(ByteBuffer)
+  ByteBuffer get(int, ByteBuffer)
+}

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/20_scriptfield.yml

@@ -157,31 +157,4 @@ setup:
     - match: { error.failed_shards.0.reason.type: "script_exception" }
     - match: { error.failed_shards.0.reason.reason: "compile error" }
 
----
-"Scripted Field with error accessing an unsupported field via the script fields api":
-  - do:
-      catch: bad_request
-      search:
-        rest_total_hits_as_int: true
-        body:
-          script_fields:
-            bar:
-              script:
-                source: "field('foo').getValue('')"
-
-  - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" }
-  - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead"}
-
-  - do:
-      catch: bad_request
-      search:
-        rest_total_hits_as_int: true
-        body:
-          script_fields:
-            bar:
-              script:
-                source: "field('foo').getValues()"
-
-  - match: { error.failed_shards.0.reason.caused_by.type: "unsupported_operation_exception" }
-  - match: { error.failed_shards.0.reason.caused_by.reason: "field [foo] is not supported through the fields api, use [doc] instead" }
 

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/60_script_doc_values_binary.yml

@@ -20,7 +20,30 @@
             index: test
             id: 1
             body:
-                binary: U29tZSBiaW5hcnkgYmxvYg==
+                binary: "U29tZSBiaW5hcnkgYmxvYg=="
+
+    - do:
+        #set the header so we won't randomize it
+        headers:
+          Content-Type: application/json
+        index:
+          index: test
+          id: 2
+          body:
+            binary: [
+              "U29tZSBiaW5hcnkgYmxvYg==",
+              "MTIzNA==",
+              "dGVzdA=="
+            ]
+
+    - do:
+        #set the header so we won't randomize it
+        headers:
+          Content-Type: application/json
+        index:
+          index: test
+          id: 3
+          body: {}
 
     - do:
         indices.refresh: {}
@@ -31,9 +54,55 @@
                 script_fields:
                     field1:
                         script:
-                            source: "doc['binary'].get(0).utf8ToString()"
+                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].get(0).utf8ToString()"
                     field2:
                         script:
-                            source: "doc['binary'].value.utf8ToString()"
+                            source: "if (doc['binary'].size() == 0) {return 'empty'} doc['binary'].value.utf8ToString()"
     - match: { hits.hits.0.fields.field1.0: "Some binary blob" }
     - match: { hits.hits.0.fields.field2.0: "Some binary blob" }
+
+    - do:
+        search:
+          body:
+            script_fields:
+              field1:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.get(0)"
+              field2:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.get(0)"
+              field3:
+                script:
+                  source: "int total = 0; for (value in field('binary')) {total += value.get(0)} total"
+    - match: { hits.hits.0.fields.field1.0: 83 }
+    - match: { hits.hits.0.fields.field2.0: 83 }
+    - match: { hits.hits.0.fields.field3.0: 83 }
+    - match: { hits.hits.1.fields.field1.0: 49 }
+    - match: { hits.hits.1.fields.field2.0: 49 }
+    - match: { hits.hits.1.fields.field3.0: 248 }
+    - match: { hits.hits.2.fields.field1.0: -1 }
+    - match: { hits.hits.2.fields.field2.0: -1 }
+    - match: { hits.hits.2.fields.field3.0: 0 }
+
+    - do:
+        search:
+          body:
+            script_fields:
+              field1:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(null); if (bb == null) {return -1;} return bb.limit()"
+              field2:
+                script:
+                  source: "ByteBuffer bb = field('binary').get(0, null); if (bb == null) {return -1;} return bb.limit()"
+              field3:
+                script:
+                  source: "int total = 0; for (ByteBuffer value : field('binary')) {total += value.limit()} total"
+    - match: { hits.hits.0.fields.field1.0: 16 }
+    - match: { hits.hits.0.fields.field2.0: 16 }
+    - match: { hits.hits.0.fields.field3.0: 16 }
+    - match: { hits.hits.1.fields.field1.0: 4 }
+    - match: { hits.hits.1.fields.field2.0: 4 }
+    - match: { hits.hits.1.fields.field3.0: 24 }
+    - match: { hits.hits.2.fields.field1.0: -1 }
+    - match: { hits.hits.2.fields.field2.0: -1 }
+    - match: { hits.hits.2.fields.field3.0: 0 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -527,7 +527,7 @@ private static class ScaledFloatLeafFieldData implements LeafNumericFieldData {
         }
 
         @Override
-        public DocValuesField getScriptField(String name) {
+        public DocValuesField<?> getScriptField(String name) {
             return new DelegateDocValuesField(new ScriptDocValues.Doubles(getDoubleValues()), name);
         }
 

server/src/main/java/org/elasticsearch/index/fielddata/IpScriptFieldData.java

@@ -52,7 +52,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex
         IpFieldScript script = leafFactory.newInstance(context);
         return new BinaryScriptLeafFieldData() {
             @Override
-            public DocValuesField getScriptField(String name) {
+            public DocValuesField<?> getScriptField(String name) {
                 return new DelegateDocValuesField(new IpScriptDocValues(getBytesValues()), name);
             }
 

server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java

@@ -23,7 +23,7 @@ public interface LeafFieldData extends Accountable, Releasable {
     /**
      * Returns an {@code Field} for use in accessing field values in scripting.
      */
-    DocValuesField getScriptField(String name);
+    DocValuesField<?> getScriptField(String name);
 
     /**
      * Return a String representation of the values.

server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.geometry.utils.Geohash;
+import org.elasticsearch.script.field.BinaryDocValuesField;
 
 import java.io.IOException;
 import java.time.Instant;
@@ -585,30 +586,33 @@ public final String getValue() {
         }
     }
 
-    public static final class BytesRefs extends BinaryScriptDocValues<BytesRef> {
+    public static final class BytesRefs extends ScriptDocValues<BytesRef> {
 
-        public BytesRefs(SortedBinaryDocValues in) {
-            super(in);
+        private final BinaryDocValuesField binaryDocValuesField;
+
+        public BytesRefs(BinaryDocValuesField binaryDocValuesField) {
+            this.binaryDocValuesField = binaryDocValuesField;
         }
 
         @Override
-        public BytesRef get(int index) {
-            if (count == 0) {
-                throw new IllegalStateException(
-                    "A document doesn't have a value for a field! "
-                        + "Use doc[<field>].size()==0 to check if a document is missing a field!"
-                );
-            }
-            /**
-             * We need to make a copy here because {@link BinaryScriptDocValues} might reuse the
-             * returned value and the same instance might be used to
-             * return values from multiple documents.
-             **/
-            return values[index].toBytesRef();
+        public void setNextDocId(int docId) throws IOException {
+            throw new UnsupportedOperationException();
         }
 
         public BytesRef getValue() {
+            throwIfEmpty();
             return get(0);
         }
+
+        @Override
+        public BytesRef get(int index) {
+            throwIfEmpty();
+            return binaryDocValuesField.getInternal(index);
+        }
+
+        @Override
+        public int size() {
+            return binaryDocValuesField.size();
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/fielddata/StringScriptFieldData.java

@@ -44,7 +44,7 @@ public BinaryScriptLeafFieldData loadDirect(LeafReaderContext context) throws Ex
         StringFieldScript script = leafFactory.newInstance(context);
         return new BinaryScriptLeafFieldData() {
             @Override
-            public DocValuesField getScriptField(String name) {
+            public DocValuesField<?> getScriptField(String name) {
                 return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name);
             }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafGeoPointFieldData.java

@@ -27,7 +27,7 @@ public final SortedBinaryDocValues getBytesValues() {
     }
 
     @Override
-    public final DocValuesField getScriptField(String name) {
+    public final DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(new ScriptDocValues.GeoPoints(getGeoPointValues()), name);
     }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLeafOrdinalsFieldData.java

@@ -35,7 +35,7 @@ protected AbstractLeafOrdinalsFieldData(Function<SortedSetDocValues, ScriptDocVa
     }
 
     @Override
-    public final DocValuesField getScriptField(String name) {
+    public final DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(scriptFunction.apply(getOrdinalsValues()), name);
     }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/BinaryDVLeafFieldData.java

@@ -45,7 +45,7 @@ public SortedBinaryDocValues getBytesValues() {
     }
 
     @Override
-    public DocValuesField getScriptField(String name) {
+    public DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name);
     }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java

@@ -9,8 +9,7 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.index.BinaryDocValues;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.script.field.DelegateDocValuesField;
+import org.elasticsearch.script.field.BinaryDocValuesField;
 import org.elasticsearch.script.field.DocValuesField;
 
 final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData {
@@ -19,7 +18,7 @@ final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData {
     }
 
     @Override
-    public DocValuesField getScriptField(String name) {
-        return new DelegateDocValuesField(new ScriptDocValues.BytesRefs(getBytesValues()), name);
+    public DocValuesField<?> getScriptField(String name) {
+        return new BinaryDocValuesField(getBytesValues(), name);
     }
 }

server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java

@@ -41,7 +41,7 @@ public long ramBytesUsed() {
     }
 
     @Override
-    public final DocValuesField getScriptField(String name) {
+    public final DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(new ScriptDocValues.Doubles(getDoubleValues()), name);
     }
 

server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java

@@ -44,7 +44,7 @@ public long ramBytesUsed() {
     }
 
     @Override
-    public final DocValuesField getScriptField(String name) {
+    public final DocValuesField<?> getScriptField(String name) {
         switch (numericType) {
             // for now, dates and nanoseconds are treated the same, which also means, that the precision is only on millisecond level
             case DATE:

server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java

@@ -19,7 +19,7 @@ final class StringBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData {
     }
 
     @Override
-    public DocValuesField getScriptField(String name) {
+    public DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name);
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -218,7 +218,7 @@ public long ramBytesUsed() {
             }
 
             @Override
-            public DocValuesField getScriptField(String name) {
+            public DocValuesField<?> getScriptField(String name) {
                 return new DelegateDocValuesField(new ScriptDocValues.Strings(getBytesValues()), name);
             }
 

server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java

@@ -79,7 +79,7 @@ public void close() {
     }
 
     @Override
-    public DocValuesField getScriptField(String name) {
+    public DocValuesField<?> getScriptField(String name) {
         return new DelegateDocValuesField(AbstractLeafOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION.apply(getOrdinalsValues()), name);
     }
 

server/src/main/java/org/elasticsearch/script/DocBasedScript.java

@@ -23,14 +23,14 @@ public DocBasedScript(DocReader docReader) {
         this.docReader = docReader;
     }
 
-    public Field field(String fieldName) {
+    public Field<?> field(String fieldName) {
         if (docReader == null) {
             return new EmptyField(fieldName);
         }
         return docReader.field(fieldName);
     }
 
-    public Stream<Field> fields(String fieldGlob) {
+    public Stream<Field<?>> fields(String fieldGlob) {
         if (docReader == null) {
             return Stream.empty();
         }

server/src/main/java/org/elasticsearch/script/DocReader.java

@@ -22,10 +22,10 @@
  */
 public interface DocReader {
     /** New-style field access */
-    Field field(String fieldName);
+    Field<?> field(String fieldName);
 
     /** New-style field iterator */
-    Stream<Field> fields(String fieldGlob);
+    Stream<Field<?>> fields(String fieldGlob);
 
     /** Set the underlying docId */
     void setDocument(int docID);

server/src/main/java/org/elasticsearch/script/DocValuesDocReader.java

@@ -39,7 +39,7 @@ public DocValuesDocReader(SearchLookup searchLookup, LeafReaderContext leafConte
     }
 
     @Override
-    public Field field(String fieldName) {
+    public Field<?> field(String fieldName) {
         LeafDocLookup leafDocLookup = leafSearchLookup.doc();
 
         if (leafDocLookup.containsKey(fieldName) == false) {
@@ -50,7 +50,7 @@ public Field field(String fieldName) {
     }
 
     @Override
-    public Stream<Field> fields(String fieldGlob) {
+    public Stream<Field<?>> fields(String fieldGlob) {
         throw new UnsupportedOperationException("not implemented");
     }