diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 00000000000..577310d0104 --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,23 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Triggers +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. + +### Pipeline steps + +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 00000000000..fb396328d8e --- /dev/null +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,231 @@ +# yaml-language-server: $schema=https://github.com/raw/buildkite/pipeline-schema/main/schema.json +# TODO: Pre-cache beats-dev/golang-crossbuild container image + +env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "linux/arm64" + +steps: + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dashboards-snapshot + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: true + artifact_paths: + - build/distributions/**/* + + - label: Staging dashboards + if: "build.branch =~ /\\d+\\.\\d+/" + key: dashboards-staging + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + artifact_paths: + - build/distributions/**/* + + - group: Packaging snapshot + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: packaging-snapshot + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: Packaging staging + + key: packaging-staging + ## Only for release + if: "build.branch =~ /\\d+\\.\\d+/" + steps: + # x86 + - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + ## ARM + - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/** + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + ## Agentbeat needs more CPUs because it builds many other beats + - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD}" + SNAPSHOT: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + ## Only for release branches and main + if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + key: dra-snapshot + env: + DRA_WORKFLOW: snapshot + depends_on: + - packaging-snapshot + - dashboards-snapshot + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" + + - label: DRA Staging + ## Only for release branches + if: "build.branch =~ /\\d+\\.\\d+/" + key: dra-staging + env: + DRA_WORKFLOW: staging + depends_on: + - packaging-staging + - dashboards-staging + command: | + buildkite-agent artifact download "build/**" . + .buildkite/scripts/packaging/prepare-release-manager.sh + .buildkite/scripts/dra.sh + agents: + provider: "gcp" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100755 index 00000000000..ee0f4b286f0 --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## TODO: Set to empty string when Jenkins is disabled +if [[ "$DRY_RUN" == "false" ]]; then echo "--- Running in publish mode"; DRY_RUN=""; else echo "--- Running in dry-run mode"; DRY_RUN="--dry-run"; fi +set -euo pipefail +BRANCH="${BUILDKITE_BRANCH}" + +if [[ "${BUILDKITE_PULL_REQUEST:="false"}" != "false" ]]; then + BRANCH=main + DRY_RUN="--dry-run" + echo "+++ Running in PR and setting branch main and --dry-run" +fi + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +release_manager_login + +echo "+++ Changing permissions for the BK API commands" +sudo chown -R :1000 build/distributions/ + +echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} \ No newline at end of file diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 00000000000..a70bd455278 --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ueo pipefail +BEAT_DIR=${1:-""} + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo "~~~Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 00000000000..2727fd986cc --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true