Extra testing and some cleanups for filtering on field caps

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FieldCapsIT.java

@@ -8,16 +8,21 @@
 
 package org.elasticsearch.upgrades;
 
+import org.apache.http.HttpHost;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.xcontent.json.JsonXContent;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
@@ -27,6 +32,9 @@
  * reduce the transport message size between nodes and clusters, and the memory usage to hold these internal responses.
  * As the optimization is applied for only field-caps requests without index-filter and nodes on 8.2 or later,
  * these BWC tests verify these combinations of field-caps requests: (old|new|mixed indices) and (with|without index filter)
+ *
+ * In 8.2 we also added the ability to filter fields by type and metadata, with some post-hoc filtering applied on
+ * the co-ordinating node if older nodes were included in the system
  */
 public class FieldCapsIT extends AbstractRollingTestCase {
     private static boolean indicesCreated = false;
@@ -42,6 +50,7 @@ public void setupIndices() throws Exception {
                "red_field": { "type": "keyword" },
                "yellow_field": { "type": "integer" },
                "blue_field": { "type": "keyword" },
+               "multi_field" : { "type" : "ip", "fields" : { "keyword" : { "type" : "keyword" } } },
                "timestamp": {"type": "date"}
              }
             """;
@@ -50,6 +59,7 @@ public void setupIndices() throws Exception {
                "green_field": { "type": "keyword" },
                "yellow_field": { "type": "long" },
                "blue_field": { "type": "keyword" },
+               "multi_field" : { "type" : "ip", "fields" : { "keyword" : { "type" : "keyword" } } },
                "timestamp": {"type": "date"}
              }
             """;
@@ -88,7 +98,7 @@ public void setupIndices() throws Exception {
 
     public void testOldIndicesOnly() throws Exception {
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), null, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2", "old_red_empty" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -98,7 +108,7 @@ public void testOldIndicesOnly() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), null, null, null);
             assertThat(
                 resp.getIndices(),
                 equalTo(new String[] { "old_green_1", "old_green_2", "old_green_empty", "old_red_1", "old_red_2", "old_red_empty" })
@@ -116,7 +126,7 @@ public void testOldIndicesOnly() throws Exception {
     public void testOldIndicesWithIndexFilter() throws Exception {
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_red_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_red_1", "old_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -126,7 +136,7 @@ public void testOldIndicesWithIndexFilter() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "old_green_1", "old_green_2", "old_red_1", "old_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -141,7 +151,7 @@ public void testOldIndicesWithIndexFilter() throws Exception {
     public void testNewIndicesOnly() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), null, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2", "new_red_empty" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -151,7 +161,7 @@ public void testNewIndicesOnly() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), null, null, null);
             assertThat(
                 resp.getIndices(),
                 equalTo(new String[] { "new_green_1", "new_green_2", "new_green_empty", "new_red_1", "new_red_2", "new_red_empty" })
@@ -170,7 +180,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_red_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_red_1", "new_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -180,7 +190,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
             assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
         }
         {
-            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter);
+            FieldCapabilitiesResponse resp = fieldCaps(List.of("new_*"), List.of("*"), indexFilter, null, null);
             assertThat(resp.getIndices(), equalTo(new String[] { "new_green_1", "new_green_2", "new_red_1", "new_red_2" }));
             assertThat(resp.getField("red_field").keySet(), contains("keyword"));
             assertTrue(resp.getField("red_field").get("keyword").isSearchable());
@@ -194,7 +204,7 @@ public void testNewIndicesOnlyWithIndexFilter() throws Exception {
 
     public void testAllIndices() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
-        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null);
+        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), null, null, null);
         assertThat(
             resp.getIndices(),
             equalTo(
@@ -227,7 +237,7 @@ public void testAllIndices() throws Exception {
     public void testAllIndicesWithIndexFilter() throws Exception {
         assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
         final QueryBuilder indexFilter = QueryBuilders.rangeQuery("timestamp").gte("2020-01-01").lte("2020-12-12");
-        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter);
+        FieldCapabilitiesResponse resp = fieldCaps(List.of("old_*", "new_*"), List.of("*"), indexFilter, null, null);
         assertThat(
             resp.getIndices(),
             equalTo(
@@ -252,4 +262,46 @@ public void testAllIndicesWithIndexFilter() throws Exception {
         assertThat(resp.getField("blue_field").keySet(), contains("keyword"));
         assertTrue(resp.getField("blue_field").get("keyword").isSearchable());
     }
+
+    @SuppressWarnings("unchecked")
+    private RestClient getUpgradedNodeClient() throws IOException {
+        for (HttpHost host : getClusterHosts()) {
+            RestClient client = RestClient.builder(host).build();
+            Request nodesRequest = new Request("GET", "_nodes/_local/_none");
+            Map<String, ?> nodeMap = (Map<String, ?>) entityAsMap(client.performRequest(nodesRequest)).get("nodes");
+            Map<String, ?> nameMap = (Map<String, ?>) nodeMap.values().iterator().next();
+            String version = (String) nameMap.get("version");
+            if (version.equals(Version.CURRENT.toString())) {
+                return client;
+            }
+            client.close();
+        }
+        throw new IllegalStateException("Couldn't find node on version " + Version.CURRENT);
+    }
+
+    // Test field type filtering on mixed cluster
+    public void testAllIndicesWithFieldTypeFilter() throws Exception {
+        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        RestClient restClient = getUpgradedNodeClient();
+        FieldCapabilitiesResponse resp = fieldCaps(restClient, List.of("old_*", "new_*"), List.of("*"), null, "keyword", null);
+        assertThat(resp.getField("red_field").keySet(), contains("keyword"));
+        assertNull(resp.getField("yellow_field"));
+        restClient.close();
+    }
+
+    // Test multifield exclusion on mixed cluster
+    public void testAllIndicesWithExclusionFilter() throws Exception {
+        assumeFalse("required mixed or upgraded cluster", CLUSTER_TYPE == ClusterType.OLD);
+        RestClient client = getUpgradedNodeClient();
+        {
+            FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, null);
+            assertThat(resp.getField("multi_field.keyword").keySet(), contains("keyword"));
+        }
+        {
+            FieldCapabilitiesResponse resp = fieldCaps(client, List.of("old_*", "new_*"), List.of("*"), null, null, "-multifield");
+            assertThat(resp.getField("multi_field").keySet(), contains("ip"));
+            assertNull(resp.getField("multi_field.keyword"));
+        }
+        client.close();
+    }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFetcher.java

@@ -109,11 +109,11 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
 
         boolean includeParentObjects = checkIncludeParents(filters);
 
-        FieldCapsFilter filter = buildFilter(indexFieldfilter, filters, types);
+        Predicate<MappedFieldType> filter = buildFilter(indexFieldfilter, filters, types, context);
         Map<String, IndexFieldCapabilities> responseMap = new HashMap<>();
         for (String field : fieldNames) {
             MappedFieldType ft = context.getFieldType(field);
-            if (filter.matches(ft, context)) {
+            if (filter.test(ft)) {
                 IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
                     field,
                     ft.familyTypeName(),
@@ -190,30 +190,27 @@ private boolean canMatchShard(
         return SearchService.queryStillMatchesAfterRewrite(searchRequest, searchExecutionContext);
     }
 
-    private interface FieldCapsFilter {
-        boolean matches(MappedFieldType fieldType, SearchExecutionContext context);
-
-        default FieldCapsFilter and(FieldCapsFilter other) {
-            return (ft, context) -> matches(ft, context) && other.matches(ft, context);
-        }
-    }
-
-    private static FieldCapsFilter buildFilter(Predicate<String> fieldFilter, String[] filters, String[] fieldTypes) {
+    private static Predicate<MappedFieldType> buildFilter(
+        Predicate<String> fieldFilter,
+        String[] filters,
+        String[] fieldTypes,
+        SearchExecutionContext context
+    ) {
         // security filters don't exclude metadata fields
-        FieldCapsFilter fcf = (ft, c) -> fieldFilter.test(ft.name()) || c.isMetadataField(ft.name());
+        Predicate<MappedFieldType> fcf = ft -> fieldFilter.test(ft.name()) || context.isMetadataField(ft.name());
         if (fieldTypes.length > 0) {
             Set<String> acceptedTypes = Set.of(fieldTypes);
-            fcf = fcf.and((ft, c) -> acceptedTypes.contains(ft.familyTypeName()));
+            fcf = fcf.and(ft -> acceptedTypes.contains(ft.familyTypeName()));
         }
         for (String filter : filters) {
             if ("parent".equals(filter) || "-parent".equals(filter)) {
                 continue;
             }
-            FieldCapsFilter next = switch (filter) {
-                case "+metadata" -> (ft, c) -> c.isMetadataField(ft.name());
-                case "-metadata" -> (ft, c) -> c.isMetadataField(ft.name()) == false;
-                case "-nested" -> (ft, c) -> c.nestedLookup().getNestedParent(ft.name()) == null;
-                case "-multifield" -> (ft, c) -> c.isMultiField(ft.name()) == false;
+            Predicate<MappedFieldType> next = switch (filter) {
+                case "+metadata" -> ft -> context.isMetadataField(ft.name());
+                case "-metadata" -> ft -> context.isMetadataField(ft.name()) == false;
+                case "-nested" -> ft -> context.nestedLookup().getNestedParent(ft.name()) == null;
+                case "-multifield" -> ft -> context.isMultiField(ft.name()) == false;
                 default -> throw new IllegalArgumentException("Unknown field caps filter [" + filter + "]");
             };
             fcf = fcf.and(next);

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -37,7 +37,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
     private String[] fields = Strings.EMPTY_ARRAY;
     private String[] filters = Strings.EMPTY_ARRAY;
-    private String[] allowedTypes = Strings.EMPTY_ARRAY;
+    private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
@@ -57,7 +57,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
         runtimeFields = in.readMap();
         if (in.getVersion().onOrAfter(Version.V_8_2_0)) {
             filters = in.readStringArray();
-            allowedTypes = in.readStringArray();
+            types = in.readStringArray();
         }
     }
 
@@ -95,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(runtimeFields);
         if (out.getVersion().onOrAfter(Version.V_8_2_0)) {
             out.writeStringArray(filters);
-            out.writeStringArray(allowedTypes);
+            out.writeStringArray(types);
         }
     }
 
@@ -137,13 +137,13 @@ public String[] filters() {
         return filters;
     }
 
-    public FieldCapabilitiesRequest allowedTypes(String... types) {
-        this.allowedTypes = types;
+    public FieldCapabilitiesRequest types(String... types) {
+        this.types = types;
         return this;
     }
 
-    public String[] allowedTypes() {
-        return allowedTypes;
+    public String[] types() {
+        return types;
     }
 
     /**
@@ -243,7 +243,7 @@ public boolean equals(Object o) {
             && Objects.equals(indexFilter, that.indexFilter)
             && Objects.equals(nowInMillis, that.nowInMillis)
             && Arrays.equals(filters, that.filters)
-            && Arrays.equals(allowedTypes, that.allowedTypes)
+            && Arrays.equals(types, that.types)
             && Objects.equals(runtimeFields, that.runtimeFields);
     }
 
@@ -253,7 +253,7 @@ public int hashCode() {
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         result = 31 * result + Arrays.hashCode(filters);
-        result = 31 * result + Arrays.hashCode(allowedTypes);
+        result = 31 * result + Arrays.hashCode(types);
         return result;
     }
 
@@ -266,7 +266,7 @@ public String getDescription() {
         stringBuilder.append("], filters[");
         stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(filters), ","));
         stringBuilder.append("], types[");
-        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(allowedTypes), ","));
+        stringBuilder.append(Strings.collectionToDelimitedString(Arrays.asList(types), ","));
         stringBuilder.append("]");
         return stringBuilder.toString();
     }

server/src/main/java/org/elasticsearch/action/fieldcaps/IndexFieldCapabilities.java

@@ -35,7 +35,7 @@ public class IndexFieldCapabilities implements Writeable {
     private final TimeSeriesParams.MetricType metricType;
     private final Map<String, String> meta;
 
-    public static IndexFieldCapabilities withMetadata(IndexFieldCapabilities input, boolean isMetadata) {
+    static IndexFieldCapabilities withMetadata(IndexFieldCapabilities input, boolean isMetadata) {
         return new IndexFieldCapabilities(
             input.getName(),
             input.getType(),

server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java

@@ -175,7 +175,7 @@ private void sendRequestToNode(String nodeId, List<ShardId> shardIds) {
             shardIds,
             fieldCapsRequest.fields(),
             fieldCapsRequest.filters(),
-            fieldCapsRequest.allowedTypes(),
+            fieldCapsRequest.types(),
             originalIndices,
             fieldCapsRequest.indexFilter(),
             nowInMillis,

server/src/main/java/org/elasticsearch/action/fieldcaps/ResponseRewriter.java

@@ -17,6 +17,10 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+/**
+ * Applies field type filters to responses that come from earlier versions of ES
+ * that do not support filtering directly.
+ */
 final class ResponseRewriter {
 
     public static Map<String, IndexFieldCapabilities> rewriteOldResponses(
@@ -26,7 +30,7 @@ public static Map<String, IndexFieldCapabilities> rewriteOldResponses(
         String[] allowedTypes,
         Predicate<String> isMetadata
     ) {
-        if (version.onOrAfter(Version.V_8_1_0)) {
+        if (version.onOrAfter(Version.V_8_2_0)) {
             return input;   // nothing needs to be done
         }
         Function<IndexFieldCapabilities, IndexFieldCapabilities> transformer = buildTransformer(

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -221,7 +221,7 @@ private static FieldCapabilitiesRequest prepareRemoteRequest(
         remoteRequest.indices(originalIndices.indices());
         remoteRequest.fields(request.fields());
         remoteRequest.filters(request.filters());
-        remoteRequest.allowedTypes(request.allowedTypes());
+        remoteRequest.types(request.types());
         remoteRequest.runtimeFields(request.runtimeFields());
         remoteRequest.indexFilter(request.indexFilter());
         remoteRequest.nowInMillis(nowInMillis);
@@ -281,7 +281,7 @@ private void innerMerge(
             response.getOriginVersion(),
             response.get(),
             request.filters(),
-            request.allowedTypes(),
+            request.types(),
             metadataFieldPred
         );
         for (Map.Entry<String, IndexFieldCapabilities> entry : fields.entrySet()) {