Merge branch 'main' of github.com:elastic/kibana into adds-related-in…

…tegrations-ui

.buildkite/ftr_configs.yml

@@ -162,10 +162,12 @@ enabled:
   - x-pack/test/functional/apps/maps/group2/config.ts
   - x-pack/test/functional/apps/maps/group3/config.ts
   - x-pack/test/functional/apps/maps/group4/config.ts
+  - x-pack/test/functional/apps/ml/anomaly_detection/config.ts
+  - x-pack/test/functional/apps/ml/data_frame_analytics/config.ts
   - x-pack/test/functional/apps/ml/data_visualizer/config.ts
-  - x-pack/test/functional/apps/ml/group1/config.ts
-  - x-pack/test/functional/apps/ml/group2/config.ts
-  - x-pack/test/functional/apps/ml/group3/config.ts
+  - x-pack/test/functional/apps/ml/permissions/config.ts
+  - x-pack/test/functional/apps/ml/short_tests/config.ts
+  - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts
   - x-pack/test/functional/apps/reporting_management/config.ts

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#ef419d4f761dd256cb59bfab9b59f8b91029eb40"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#ae4994aba5f2e72edcc5914e2aa208086e4b7ea3"
   }
 }

.buildkite/pipelines/artifacts.yml

@@ -41,7 +41,17 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/artifacts/docker_context.sh
+  - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/on_merge.yml

@@ -107,6 +107,16 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh
+    label: 'Populate local dev bazel cache (Linux)'
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 15
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/scripts/build_kibana.sh

@@ -5,9 +5,9 @@ set -euo pipefail
 export KBN_NP_PLUGINS_BUILT=true
 
 echo "--- Build Kibana Distribution"
-if [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-all-platforms"* ]]; then
+if is_pr_with_label "ci:build-all-platforms"; then
   node scripts/build --all-platforms --skip-os-packages
-elif [[ "${GITHUB_PR_LABELS:-}" == *"ci:build-os-packages"* ]]; then
+elif is_pr_with_label "ci:build-os-packages"; then
   node scripts/build --all-platforms --docker-cross-compile
 else
   node scripts/build

.buildkite/scripts/common/env.sh

@@ -41,7 +41,7 @@ export ELASTIC_APM_SERVER_URL=https://kibana-ci-apm.apm.us-central1.gcp.cloud.es
 export ELASTIC_APM_SECRET_TOKEN=7YKhoXsO4MzjhXjx2c
 
 if is_pr; then
-  if [[ "${GITHUB_PR_LABELS:-}" == *"ci:collect-apm"* ]]; then
+  if is_pr_with_label "ci:collect-apm"; then
     export ELASTIC_APM_ACTIVE=true
     export ELASTIC_APM_CONTEXT_PROPAGATION_ONLY=false
   else

.buildkite/scripts/common/setup_bazel.sh

@@ -32,6 +32,15 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
+if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
+  echo "[bazel] enabling caching with GCS buckets for local dev"
+
+cat <<EOF >> $KIBANA_DIR/.bazelrc
+  build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
+  build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+EOF
+fi
+
 if [[ "$BAZEL_CACHE_MODE" == "buildbuddy" ]]; then
   echo "[bazel] enabling caching with Buildbuddy"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
@@ -43,7 +52,7 @@ cat <<EOF >> $KIBANA_DIR/.bazelrc
 EOF
 fi
 
-if [[ "$BAZEL_CACHE_MODE" != @(gcs|buildbuddy|none|) ]]; then
-  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,buildbuddy,none]"
+if [[ "$BAZEL_CACHE_MODE" != @(gcs|populate-local-gcs|buildbuddy|none|) ]]; then
+  echo "invalid value for BAZEL_CACHE_MODE received ($BAZEL_CACHE_MODE), expected one of [gcs,populate-local-gcs|buildbuddy,none]"
   exit 1
 fi

.buildkite/scripts/common/util.sh

@@ -14,6 +14,25 @@ is_pr() {
   false
 }
 
+is_pr_with_label() {
+  match="$1"
+
+  IFS=',' read -ra labels <<< "${GITHUB_PR_LABELS:-}"
+
+  for label in "${labels[@]}"
+  do
+    if [ "$label" == "$match" ]; then
+      return
+    fi
+  done
+
+  false
+}
+
+is_auto_commit_disabled() {
+  is_pr_with_label "ci:no-auto-commit"
+}
+
 check_for_changed_files() {
   RED='\033[0;31m'
   YELLOW='\033[0;33m'
@@ -23,7 +42,7 @@ check_for_changed_files() {
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
 
   if [ "$GIT_CHANGES" ]; then
-    if [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+    if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
       NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'"
       PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
 

.buildkite/scripts/lifecycle/pre_command.sh

@@ -132,6 +132,10 @@ export SYNTHETICS_REMOTE_KIBANA_URL
 KIBANA_BUILDBUDDY_CI_API_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
 export KIBANA_BUILDBUDDY_CI_API_KEY
 
+BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
+export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
+retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+
 # By default, all steps should set up these things to get a full environment before running
 # It can be skipped for pipeline upload steps though, to make job start time a little faster
 if [[ "${SKIP_CI_SETUP:-}" != "true" ]]; then

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -9,19 +9,7 @@
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
-
-const SKIPPABLE_PATHS = [
-  /^docs\//,
-  /^rfcs\//,
-  /^.ci\/.+\.yml$/,
-  /^.ci\/es-snapshots\//,
-  /^.ci\/pipeline-library\//,
-  /^.ci\/Jenkinsfile_[^\/]+$/,
-  /^\.github\//,
-  /\.md$/,
-  /^\.backportrc\.json$/,
-  /^nav-kibana-dev\.docnav\.json$/,
-];
+const { SKIPPABLE_PR_MATCHERS } = require('./skippable_pr_matchers');
 
 const REQUIRED_PATHS = [
   // this file is auto-generated and changes to it need to be validated with CI
@@ -47,7 +35,7 @@ const uploadPipeline = (pipelineContent) => {
 
 (async () => {
   try {
-    const skippable = await areChangesSkippable(SKIPPABLE_PATHS, REQUIRED_PATHS);
+    const skippable = await areChangesSkippable(SKIPPABLE_PR_MATCHERS, REQUIRED_PATHS);
 
     if (skippable) {
       console.log('All changes in PR are skippable. Skipping CI.');

.buildkite/scripts/pipelines/pull_request/skippable_pr_matchers.js

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  SKIPPABLE_PR_MATCHERS: [
+    /^docs\//,
+    /^rfcs\//,
+    /^.ci\/.+\.yml$/,
+    /^.ci\/es-snapshots\//,
+    /^.ci\/pipeline-library\//,
+    /^.ci\/Jenkinsfile_[^\/]+$/,
+    /^\.github\//,
+    /\.md$/,
+    /^\.backportrc\.json$/,
+    /^nav-kibana-dev\.docnav\.json$/,
+    /^src\/dev\/prs\/kibana_qa_pr_list\.json$/,
+    /^\.buildkite\/scripts\/pipelines\/pull_request\/skippable_pr_matchers\.js$/,
+  ],
+};

.buildkite/scripts/steps/artifacts/build.sh

@@ -24,3 +24,10 @@ buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
 buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
 buildkite-agent artifact upload 'i18n/*.json'
 cd -
+
+if [ -d .beats ]; then
+  cd .beats
+  buildkite-agent artifact upload 'metricbeat-*'
+  buildkite-agent artifact upload 'filebeat-*'
+  cd -
+fi

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -6,17 +6,29 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/artifacts/env.sh
 
+KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
+
 echo "--- Create contexts"
 mkdir -p target
 node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact $(echo "$BUILD_ARGS")
 
-echo "--- Setup default context"
+echo "--- Setup context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
 
-tar -xf target/kibana-[0-9]*-docker-build-context.tar.gz -C "$DOCKER_BUILD_FOLDER"
+if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+fi
+
+tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
 buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+if [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
+  buildkite-agent artifact download "metricbeat-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  buildkite-agent artifact download "filebeat-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+fi
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+export BAZEL_CACHE_MODE=populate-local-gcs
+export DISABLE_BOOTSTRAP_VALIDATION=true
+
+# Clear out bazel cache between runs to make sure that any artifacts that don't exist in the cache are uploaded
+rm -rf ~/.bazel-cache
+
+.buildkite/scripts/bootstrap.sh