Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…112704_fix

.buildkite/pipelines/periodic.yml

@@ -584,7 +584,7 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
       diskSizeGb: 250
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

distribution/archives/integ-test-zip/src/javaRestTest/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java

@@ -27,10 +27,9 @@ public void testIndexMissingBody() throws IOException {
     }
 
     public void testBulkMissingBody() throws IOException {
-        ResponseException responseException = expectThrows(
-            ResponseException.class,
-            () -> client().performRequest(new Request(randomBoolean() ? "POST" : "PUT", "/_bulk"))
-        );
+        Request request = new Request(randomBoolean() ? "POST" : "PUT", "/_bulk");
+        request.setJsonEntity("");
+        ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(request));
         assertResponseException(responseException, "request body is required");
     }
 

docs/build.gradle

@@ -123,6 +123,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
+  requiresFeature 'es.ccs_telemetry_feature_flag_enabled', Version.fromString("8.16.0")
 
   // TODO Rene: clean up this kind of cross project file references
   extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json")

docs/changelog/112481.yaml

@@ -0,0 +1,5 @@
+pr: 112481
+summary: Validate streaming HTTP Response
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112571.yaml

@@ -0,0 +1,17 @@
+pr: 112571
+summary: Deprecate dot-prefixed indices and composable template index patterns
+area: CRUD
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate dot-prefixed indices and composable template index patterns
+  area: CRUD
+  details: "Indices beginning with a dot '.' are reserved for system and internal\
+    \ indices, and should not be used by and end-user. Additionally, composable index\
+    \ templates that contain patterns for dot-prefixed indices should also be avoided,\
+    \ as these patterns are meant for internal use only. In a future Elasticsearch\
+    \ version, creation of these dot-prefixed indices will no longer be allowed."
+  impact: "Requests performing an action that would create an index beginning with\
+    \ a dot (indexing a document, manual creation, reindex), or creating an index\
+    \ template with index patterns beginning with a dot, will contain a deprecation\
+    \ header warning about dot-prefixed indices in the response."

docs/changelog/112895.yaml

@@ -0,0 +1,5 @@
+pr: 112895
+summary: (logger) change from error to warn for short circuiting user
+area: Security
+type: enhancement
+issues: []

docs/changelog/113027.yaml

@@ -0,0 +1,6 @@
+pr: 113027
+summary: Retrieve the source for objects and arrays in a separate parsing phase
+area: Mapping
+type: bug
+issues:
+ - 112374

docs/reference/modules/discovery/bootstrapping.asciidoc

@@ -5,7 +5,7 @@ Starting an Elasticsearch cluster for the very first time requires the initial
 set of <<master-node,master-eligible nodes>> to be explicitly defined on one or
 more of the master-eligible nodes in the cluster. This is known as _cluster
 bootstrapping_. This is only required the first time a cluster starts up.
-Freshly-started nodes that are joining a running cluster obtain this 
+Freshly-started nodes that are joining a running cluster obtain this
 information from the cluster's elected master.
 
 The initial set of master-eligible nodes is defined in the
@@ -27,17 +27,20 @@ node:
   if it is not possible to use the `node.name` of the node and there are
   multiple nodes sharing a single IP address.
 
+Do not set `cluster.initial_master_nodes` on master-ineligible nodes.
+
 [IMPORTANT]
 ====
 After the cluster has formed, remove the `cluster.initial_master_nodes` setting
-from each node's configuration. It should not be set for master-ineligible
-nodes, master-eligible nodes joining an existing cluster, or nodes which are
-restarting.
-
-If you leave `cluster.initial_master_nodes` in place once the cluster has
-formed then there is a risk that a future misconfiguration may result in
-bootstrapping a new cluster alongside your existing cluster. It may not be
-possible to recover from this situation without losing data.
+from each node's configuration and never set it again for this cluster. Do not
+configure this setting on nodes joining an existing cluster. Do not configure
+this setting on nodes which are restarting. Do not configure this setting when
+performing a full-cluster restart.
+
+If you leave `cluster.initial_master_nodes` in place once the cluster has formed
+then there is a risk that a future misconfiguration may result in bootstrapping
+a new cluster alongside your existing cluster. It may not be possible to recover
+from this situation without losing data.
 ====
 
 The simplest way to create a new cluster is for you to select one of your

docs/reference/modules/discovery/discovery-settings.asciidoc

@@ -46,8 +46,11 @@ setting, see <<single-node-discovery>>.
 Sets the initial set of master-eligible nodes in a brand-new cluster. By
 default this list is empty, meaning that this node expects to join a cluster
 that has already been bootstrapped. Remove this setting once the cluster has
-formed. Do not use this setting when restarting nodes or when adding new nodes
-to an existing cluster. See <<initial_master_nodes>>.
+formed, and never set it again for this cluster. Do not configure this setting
+on master-ineligible nodes. Do not configure this setting on nodes joining an
+existing cluster. Do not configure this setting on nodes which are restarting.
+Do not configure this setting when performing a full-cluster restart. See
+<<initial_master_nodes>>.
 
 [discrete]
 ==== Expert settings

docs/reference/setup/bootstrap-checks.asciidoc

@@ -245,10 +245,9 @@ properties:
 - `discovery.seed_providers`
 - `cluster.initial_master_nodes`
 
-Note that you should remove `cluster.initial_master_nodes` from the
-configuration after the cluster has started for the first time. Do not use this
-setting when restarting nodes or when adding new nodes to an existing cluster.
-Instead, configure `discovery.seed_hosts` or `discovery.seed_providers`. If you
-do not need any discovery configuration, for instance if running a single-node
-cluster, set `discovery.seed_hosts: []` to disable discovery and satisfy this
-bootstrap check.
+Note that you must <<initial_master_nodes,remove `cluster.initial_master_nodes`
+from the configuration of every node>> after the cluster has started for the
+first time. Instead, configure `discovery.seed_hosts` or
+`discovery.seed_providers`. If you do not need any discovery configuration, for
+instance if running a single-node cluster, set `discovery.seed_hosts: []` to
+disable discovery and satisfy this bootstrap check.

docs/reference/setup/important-settings/discovery-settings.asciidoc

@@ -53,16 +53,18 @@ first election. In <<dev-vs-prod-mode,development mode>>, with no discovery
 settings configured, this step is performed automatically by the nodes
 themselves.
 
-Because auto-bootstrapping is <<modules-discovery-quorums,inherently
-unsafe>>, when starting a new cluster in production
-mode, you must explicitly list the master-eligible nodes whose votes should be
-counted in the very first election. You set this list using the
-`cluster.initial_master_nodes` setting.
+Because auto-bootstrapping is <<modules-discovery-quorums,inherently unsafe>>,
+when starting a new cluster in production mode, you must explicitly list the
+master-eligible nodes whose votes should be counted in the very first election.
+You set this list using the `cluster.initial_master_nodes` setting on every
+master-eligible node. Do not configure this setting on master-ineligible nodes.
 
 IMPORTANT: After the cluster forms successfully for the first time, remove the
-`cluster.initial_master_nodes` setting from each node's configuration. Do not
-use this setting when restarting a cluster or adding a new node to an existing
-cluster.
+`cluster.initial_master_nodes` setting from each node's configuration and never
+set it again for this cluster. Do not configure this setting on nodes joining
+an existing cluster. Do not configure this setting on nodes which are
+restarting. Do not configure this setting when performing a full-cluster
+restart. See <<modules-discovery-bootstrap-cluster>>.
 
 [source,yaml]
 --------------------------------------------------

modules/data-streams/build.gradle

@@ -40,3 +40,7 @@ if (BuildParams.isSnapshotBuild() == false) {
     systemProperty 'es.failure_store_feature_flag_enabled', 'true'
   }
 }
+
+tasks.named("yamlRestCompatTestTransform").configure({ task ->
+  task.skipTest("data_stream/10_basic/Create hidden data stream", "warning does not exist for compatibility")
+})

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DataStreamsRestIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.datastreams;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -71,10 +72,12 @@ public void testHiddenDataStreamImplicitHiddenSearch() throws IOException {
         // Create a template
         Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/hidden");
         putComposableIndexTemplateRequest.setJsonEntity("{\"index_patterns\": [\".hidden\"], \"data_stream\": {\"hidden\": true}}");
+        putComposableIndexTemplateRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
         assertOK(client().performRequest(putComposableIndexTemplateRequest));
 
         Request createDocRequest = new Request("POST", "/.hidden/_doc?refresh=true");
         createDocRequest.setJsonEntity("{ \"@timestamp\": \"2020-10-22\", \"a\": 1 }");
+        createDocRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("X-elastic-product-origin", "elastic"));
 
         assertOK(client().performRequest(createDocRequest));
 

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceTransforms.java

@@ -11,10 +11,10 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -32,7 +32,7 @@ class SourceTransforms {
      * @return flattened map
      */
     public static Map<String, List<Object>> normalize(Map<String, Object> map) {
-        var flattened = new HashMap<String, List<Object>>();
+        var flattened = new TreeMap<String, List<Object>>();
 
         descend(null, map, flattened);
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -115,8 +115,10 @@ setup:
 "Create hidden data stream":
   - requires:
       cluster_features: ["gte_v7.11.0"]
+      test_runner_features: ["warnings", "headers"]
       reason: "hidden data streams only available in 7.11"
   - do:
+      headers: { X-elastic-product-origin: elastic }
       allowed_warnings:
         - "index template [my-template3] has index patterns [.hidden-data-stream, hidden-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template3] will take precedence during new index creation"
       indices.put_index_template:

modules/dot-prefix-validation/build.gradle

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+apply plugin: 'elasticsearch.yaml-rest-compat-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
+
+esplugin {
+  description 'Validation for dot-prefixed indices for non-operator users'
+  classname 'org.elasticsearch.validation.DotPrefixValidationPlugin'
+}
+
+restResources {
+  restApi {
+    include '_common', 'indices', 'index', 'cluster', 'nodes', 'get', 'ingest', 'bulk', 'reindex'
+  }
+}
+
+tasks.named('yamlRestTest') {
+  usesDefaultDistribution()
+}

modules/dot-prefix-validation/src/main/java/module-info.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module org.elasticsearch.validation {
+    requires org.elasticsearch.server;
+    requires org.elasticsearch.base;
+}

modules/dot-prefix-validation/src/main/java/org/elasticsearch/validation/AutoCreateDotValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.validation;
+
+import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Set;
+
+public class AutoCreateDotValidator extends DotPrefixValidator<CreateIndexRequest> {
+    public AutoCreateDotValidator(ThreadContext threadContext, ClusterService clusterService) {
+        super(threadContext, clusterService);
+    }
+
+    @Override
+    protected Set<String> getIndicesFromRequest(CreateIndexRequest request) {
+        return Set.of(request.index());
+    }
+
+    @Override
+    public String actionName() {
+        return AutoCreateAction.NAME;
+    }
+}