Merge branch 'main' into chart_expressions_xy-reference-line

.buildkite/ftr_configs.yml

@@ -32,6 +32,7 @@ disabled:
   - x-pack/test/security_solution_cypress/upgrade_config.ts
   - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
+  - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
 

.buildkite/pipelines/artifacts.yml

@@ -52,6 +52,17 @@ steps:
           limit: 1
 
   - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pull_requests.json

@@ -16,7 +16,25 @@
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
-      "skip_target_branches": ["6.8", "7.11", "7.12"]
+      "skip_target_branches": ["6.8", "7.11", "7.12"],
+      "skip_ci_on_only_changed": [
+        "^docs/",
+        "^rfcs/",
+        "^.ci/.+\\.yml$",
+        "^.ci/es-snapshots/",
+        "^.ci/pipeline-library/",
+        "^.ci/Jenkinsfile_[^/]+$",
+        "^\\.github/",
+        "\\.md$",
+        "^\\.backportrc\\.json$",
+        "^nav-kibana-dev\\.docnav\\.json$",
+        "^src/dev/prs/kibana_qa_pr_list\\.json$",
+        "^\\.buildkite/pull_requests\\.json$"
+      ],
+      "always_require_ci_on_changed": [
+        "^docs/developer/plugin-list.asciidoc$",
+        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
+      ]
     }
   ]
 }

.buildkite/scripts/common/env.sh

@@ -38,6 +38,7 @@ export TEST_BROWSER_HEADLESS=1
 export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 export ELASTIC_APM_SERVER_URL=https://kibana-ci-apm.apm.us-central1.gcp.cloud.es.io
+# Not really a secret, if APM supported public auth we would use it and APM requires that we use this name
 export ELASTIC_APM_SECRET_TOKEN=7YKhoXsO4MzjhXjx2c
 
 if is_pr; then

.buildkite/scripts/lifecycle/post_command.sh

@@ -26,6 +26,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/apps/reporting/reports/session/*.pdf'
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
+  buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
 
   echo "--- Run Failed Test Reporter"
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -9,14 +9,11 @@
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
-const { SKIPPABLE_PR_MATCHERS } = require('./skippable_pr_matchers');
-
-const REQUIRED_PATHS = [
-  // this file is auto-generated and changes to it need to be validated with CI
-  /^docs\/developer\/plugin-list.asciidoc$/,
-  // don't skip CI on prs with changes to plugin readme files /i is for case-insensitive matching
-  /\/plugins\/[^\/]+\/readme\.(md|asciidoc)$/i,
-];
+const prConfigs = require('../../../pull_requests.json');
+const prConfig = prConfigs.jobs.find((job) => job.pipelineSlug === 'kibana-pull-request');
+
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -19,6 +19,8 @@ if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
   DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
 elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
   DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
 fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -69,7 +69,6 @@ echo "--- Build Elasticsearch"
   :distribution:archives:darwin-aarch64-tar:assemble \
   :distribution:archives:darwin-tar:assemble \
   :distribution:docker:docker-export:assemble \
-  :distribution:docker:cloud-docker-export:assemble \
   :distribution:archives:linux-aarch64-tar:assemble \
   :distribution:archives:linux-tar:assemble \
   :distribution:archives:windows-zip:assemble \
@@ -86,19 +85,26 @@ docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 
 echo "--- Create kibana-ci docker cloud image archives"
-ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
-ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
-KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
-KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
-
-docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
-
-echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-trap 'docker logout docker.elastic.co' EXIT
-docker image push "$KIBANA_ES_CLOUD_IMAGE"
-
-export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
-export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+# Ignore build failures.  This docker image downloads metricbeat and filebeat.
+# When we bump versions, these dependencies may not exist yet, but we don't want to
+# block the rest of the snapshot promotion process
+set +e
+./gradlew :distribution:docker:cloud-docker-export:assemble && {
+  ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
+  ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+  KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
+  KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+  echo $ES_CLOUD_ID $ES_CLOUD_VERSION $KIBANA_ES_CLOUD_VERSION $KIBANA_ES_CLOUD_IMAGE
+  docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
+
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  trap 'docker logout docker.elastic.co' EXIT
+  docker image push "$KIBANA_ES_CLOUD_IMAGE"
+
+  export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
+  export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+}
+set -e
 
 echo "--- Create checksums for snapshot files"
 cd "$destination"

dev_docs/getting_started/troubleshooting.mdx

@@ -26,3 +26,17 @@ git clean -fdxn -e /config -e /.vscode
 # review the files which will be deleted, consider adding some more excludes (-e)
 # re-run without the dry-run (-n) flag to actually delete the files
 ```
+
+### search.check_ccs_compatibility error
+
+If you run into an error that says something like:
+
+```
+[class org.elasticsearch.action.search.SearchRequest] is not compatible version 8.1.0 and the 'search.check_ccs_compatibility' setting is enabled.
+```
+
+it means you are using a new Elasticsearch feature that will not work in a CCS environment because the feature does not exist in older versions. If you are working on an experimental feature and are okay with this limitation, you will have to move the failing test into a special test suite that does not use this setting to get ci to pass. Take this path cautiously. If you do not remember to move the test back into the default test suite when the feature is GA'ed, it will not have proper CCS test coverage.
+
+We added this test coverage in version `8.1` because we accidentally broke core Kibana features (for example, when Discover started using the new fields parameter) for our CCS users. CCS is not a corner case and (excluding certain experimental features) Kibana should always work for our CCS users. This setting is our way of ensuring test coverage.
+
+Please reach out to the [Kibana Operations team](https://github.com/orgs/elastic/teams/kibana-operations) if you have further questions.

dev_docs/operations/ci_stats.mdx

@@ -0,0 +1,15 @@
+---
+id: kibDevDocsOpsCiStats
+slug: /kibana-dev-docs/ops/ci-stats
+title: "Kibana CI Stats"
+description: A service that we run to track little bits of data about CI runs.
+tags: ['kibana', 'dev', 'contributor', 'operations', 'ci']
+---
+
+Kibana CI Stats (sometimes written "ci-stats") is a service run at https://ci-stats.kibana.dev. This service runs a series of APIs in front of an Elasticsearch Cluster which allows us to record data about each run of CI. This data includes metrics, times, test results, and more. We are working on storing performance data in there too, and eventually will spend time to write good UIs on top of this data.
+
+The service implementation is available at https://github.com/elastic/kibana-ci-stats/ (private because there doesn't seem to be a good reason to make it public).
+
+The service is run on Google Cloud Run, which allows us to build a container, push it to GCR, define a memory limit, vCPU count, and concurrent request limit per container, and Google will automatically scale the container for us. It works pretty well and hides a lot of the complexity of running the service. The repo uses Buildkite CI to build and deploy the container when pushing to the main branch. All changes to the main branch must come from PRs, but at this time we don't require review for PRs.
+
+The website at https://ci-stats.kibana.dev uses EUI and Elastic Charts, and currently has users powered by Github OAuth. When someone authenticates with ci-stats they are first redirected to Github for authentication, then their membership in the Elastic org is checked. Users in the Elastic org will be able to do things that other users can't, like [trigger flaky test runner jobs](https://ci-stats.kibana.dev/trigger_flaky_test_runner).

dev_docs/operations/operations_landing.mdx

@@ -0,0 +1,49 @@
+---
+id: kibDevDocsOpsOverview
+slug: /kibana-dev-docs/ops
+title: Kibana Operations
+description: Links to all the documentation maintained by the Kibana Operations team
+tags: ['kibana', 'dev', 'contributor', 'operations']
+layout: landing
+---
+
+<DocLandingHero
+  title='Welcome to the ops docs!'
+  description="👋 Here you will find documentation about all the things the Kibana Operations team manages. Feel free to look around and ask questions!"
+/>
+
+<DocRelatedArticles
+  sectionTitle='CI'
+  items={[
+    { pageId: "kibDevDocsOpsCiStats" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Build tooling'
+  items={[
+    { pageId: "kibDevDocsOpsOptimizer" },
+    { pageId: "kibDevDocsOpsBabelPreset" },
+    { pageId: "kibDevDocsOpsTypeSummarizer" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Linting & Validation'
+  items={[
+    { pageId: "kibDevDocsOpsEslintConfig" },
+    { pageId: "kibDevDocsOpsEslintWithTypes" },
+    { pageId: "kibDevDocsOpsEslintPluginImports" },
+  ]}
+/>
+
+<DocRelatedArticles
+  sectionTitle='Utilities'
+  items={[
+    { pageId: "kibDevDocsToolingLog" },
+    { pageId: "kibDevDocsOpsJestSerializers"},
+    { pageId: "kibDevDocsOpsExpect" },
+    { pageId: "kibDevDocsOpsAmbientStorybookTypes" },
+    { pageId: "kibDevDocsOpsAmbientUiTypes"},
+  ]}
+/>

docs/apm/services.asciidoc

@@ -12,3 +12,41 @@ and requires anomaly detection to be enabled.
 
 [role="screenshot"]
 image::apm/images/apm-services-overview.png[Example view of services table the APM app in Kibana]
+
+[float]
+[[service-groups]]
+==== Service groups
+
+preview::[]
+
+Group services together to build meaningful views that remove noise and simplify investigations across services.
+Service groups are {kib} space-specific and available for any users with appropriate access.
+
+[role="screenshot"]
+image::apm/images/apm-service-group.png[Example view of service group in the APM app in Kibana]
+
+To enable Service groups, open {kib} and navigate to **Stack Management** > **Advanced Settings** > **Observability**,
+and enable the **Service groups feature**.
+
+To create a service group, navigate to **Observability** > **APM** > **Services** and select **Create group**.
+Specify a name, color, and description.
+Then, using the <<kuery-query, Kibana Query Language (KQL)>>, specify a query to select services for the group.
+Services that match the query within the last 24 hours will be assigned to the group.
+
+[NOTE]
+====
+Once a service group has been saved, this list of services within it is static.
+If a newly added service matches the KQL query, it will not be automatically added to the service group.
+Similarly, if a service stops matching the KQL query, it will not be removed from the group.
+
+To update the list of services within a group,
+edit the service group, click **Refresh** next to the KQL query, and click **Save group**.
+====
+
+**Examples**
+
+Not sure where to get started? Here are some sample queries you can build from:
+
+* Group services by environment--in this example, "production": `service.environment : "production"`
+* Group services by name--this example groups those that end in "beat": `service.name : *beat` (matches services named "Auditbeat", "Heartbeat", "Filebeat", etc.)
+* Group services with a high transaction duration in the last 24 hours: `transaction.duration.us >= 50000000`

docs/development/core/server/kibana-plugin-core-server.savedobjectsupdateoptions.md

@@ -18,6 +18,7 @@ export interface SavedObjectsUpdateOptions<Attributes = unknown> extends SavedOb
 |  --- | --- | --- |
 |  [references?](./kibana-plugin-core-server.savedobjectsupdateoptions.references.md) | SavedObjectReference\[\] | <i>(Optional)</i> A reference to another saved object. |
 |  [refresh?](./kibana-plugin-core-server.savedobjectsupdateoptions.refresh.md) | MutatingOperationRefreshSetting | <i>(Optional)</i> The Elasticsearch Refresh setting for this operation |
+|  [retryOnConflict?](./kibana-plugin-core-server.savedobjectsupdateoptions.retryonconflict.md) | number | <i>(Optional)</i> The Elasticsearch <code>retry_on_conflict</code> setting for this operation. Defaults to <code>0</code> when <code>version</code> is provided, <code>3</code> otherwise. |
 |  [upsert?](./kibana-plugin-core-server.savedobjectsupdateoptions.upsert.md) | Attributes | <i>(Optional)</i> If specified, will be used to perform an upsert if the document doesn't exist |
 |  [version?](./kibana-plugin-core-server.savedobjectsupdateoptions.version.md) | string | <i>(Optional)</i> An opaque version number which changes on each successful write operation. Can be used for implementing optimistic concurrency control. |