From 9457f64afaac498c2356089a7c0d7ab98537478f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 25 Apr 2024 16:15:45 +0300 Subject: [PATCH] Fixes for Buildkite packaging pipeline (#39207) This PR fixes a number of bugs from #39126 --- .buildkite/packaging.pipeline.yml | 66 ++++++++++----------- .buildkite/scripts/dra.sh | 18 +++++- .buildkite/scripts/packaging/package-dra.sh | 2 +- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fb396328d8e..63ea44795ef 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,7 +8,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PLATFORMS_ARM: "linux/arm64" steps: @@ -16,13 +16,13 @@ steps: key: dashboards steps: - label: Snapshot dashboards - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dashboards-snapshot # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -32,13 +32,13 @@ steps: - build/distributions/**/* - label: Staging dashboards - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dashboards-staging # TODO: container with go and make agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -46,19 +46,18 @@ steps: - build/distributions/**/* - group: Packaging snapshot - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: packaging-snapshot steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -78,8 +77,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot" + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -105,36 +103,33 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "SNAPSHOT: x-pack/agentbeat" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" artifact_paths: - build/distributions/**/* - matrix: - - x-pack/agentbeat - - group: Packaging staging + - group: Packaging Staging key: packaging-staging ## Only for release - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ steps: - # x86 - - label: ":ubuntu: {{matrix}}/Packaging Linux Staging" + - label: "STAGING: {{matrix}}" env: - PLATFORMS: "${PLATFORMS_AMD}" + PLATFORMS: "${PLATFORMS}" SNAPSHOT: false command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - build/distributions/**/* matrix: @@ -154,8 +149,7 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - ## ARM - - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging" + - label: "STAGING: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" PACKAGES: "docker" @@ -181,26 +175,24 @@ steps: - x-pack/packetbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot" + - label: "STAGING: x-pack/agentbeat" env: - PLATFORMS: "${PLATFORMS_AMD}" - SNAPSHOT: true - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: false + command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" artifact_paths: - build/distributions/**/* - matrix: - - x-pack/agentbeat - group: DRA publish key: dra steps: - label: DRA Snapshot ## Only for release branches and main - if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true" + if: build.branch =~ /^\d+\.\d+$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" key: dra-snapshot env: DRA_WORKFLOW: snapshot @@ -212,11 +204,13 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - label: DRA Staging ## Only for release branches - if: "build.branch =~ /\\d+\\.\\d+/" + if: build.branch =~ /^\d+\.\d+$/ key: dra-staging env: DRA_WORKFLOW: staging @@ -228,4 +222,6 @@ steps: .buildkite/scripts/packaging/prepare-release-manager.sh .buildkite/scripts/dra.sh agents: - provider: "gcp" + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh index ee0f4b286f0..4957ef36199 100755 --- a/.buildkite/scripts/dra.sh +++ b/.buildkite/scripts/dra.sh @@ -28,6 +28,22 @@ release_manager_login echo "+++ Changing permissions for the BK API commands" sudo chown -R :1000 build/distributions/ +echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..." +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..." docker run --rm \ --name release-manager \ @@ -43,4 +59,4 @@ docker run --rm \ --workflow "${DRA_WORKFLOW}" \ --version "${BEAT_VERSION}" \ --artifact-set "main" \ - ${DRY_RUN} \ No newline at end of file + ${DRY_RUN} diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh index a70bd455278..10d5425da36 100755 --- a/.buildkite/scripts/packaging/package-dra.sh +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then exit 1 fi -echo "~~~Packaging : $BEAT_DIR" +echo "~~~ Packaging : $BEAT_DIR" WORKSPACE=$(pwd) BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')