Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes for packaging pipeline #39207

Merged
merged 9 commits into from
Apr 25, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 20 additions & 28 deletions .buildkite/packaging.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ env:
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"

PLATFORMS_AMD: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
PLATFORMS_ARM: "linux/arm64"

steps:
- group: Beats dashboards
key: dashboards
steps:
- label: Snapshot dashboards
if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
key: dashboards-snapshot
# TODO: container with go and make
agents:
Expand All @@ -32,7 +32,7 @@ steps:
- build/distributions/**/*

- label: Staging dashboards
if: "build.branch =~ /\\d+\\.\\d+/"
if: build.branch =~ /\d+\.\d+/
key: dashboards-staging
# TODO: container with go and make
agents:
Expand All @@ -46,13 +46,12 @@ steps:
- build/distributions/**/*

- group: Packaging snapshot
if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
key: packaging-snapshot
steps:
# x86
- label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
- label: "SNAPSHOT: {{matrix}}/Packaging cross architecture artifacts"
env:
PLATFORMS: "${PLATFORMS_AMD}"
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: true
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
Expand All @@ -78,8 +77,7 @@ steps:
- x-pack/packetbeat
- x-pack/winlogbeat

## ARM
- label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
- label: "SNAPSHOT: {{matrix}}/Packaging docker artifacts for Linux/arm64"
env:
PLATFORMS: "${PLATFORMS_ARM}"
PACKAGES: "docker"
Expand All @@ -105,30 +103,27 @@ steps:
- x-pack/packetbeat

## Agentbeat needs more CPUs because it builds many other beats
- label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
- label: "SNAPSHOT: x-pack/agentbeat/Packaging cross architecture artifacts"
env:
PLATFORMS: "${PLATFORMS_AMD}"
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: true
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- build/distributions/**/*
matrix:
- x-pack/agentbeat

- group: Packaging staging

key: packaging-staging
## Only for release
if: "build.branch =~ /\\d+\\.\\d+/"
if: build.branch =~ /\d+\.\d+/
steps:
# x86
- label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
- label: "STAGING: {{matrix}}/Packaging cross architecture artifacts"
env:
PLATFORMS: "${PLATFORMS_AMD}"
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: false
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
agents:
Expand All @@ -154,8 +149,7 @@ steps:
- x-pack/packetbeat
- x-pack/winlogbeat

## ARM
- label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
- label: "STAGING: {{matrix}}/Packaging docker artifacts for Linux/arm64"
env:
PLATFORMS: "${PLATFORMS_ARM}"
PACKAGES: "docker"
Expand All @@ -181,26 +175,24 @@ steps:
- x-pack/packetbeat

## Agentbeat needs more CPUs because it builds many other beats
- label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
- label: "STAGING: x-pack/agentbeat/Packaging cross architecture artifacts"
env:
PLATFORMS: "${PLATFORMS_AMD}"
SNAPSHOT: true
command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
PLATFORMS: "${PLATFORMS}"
SNAPSHOT: false
command: ".buildkite/scripts/packaging/package-dra.sh x-pack/agentbeat"
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "c2-standard-16"
artifact_paths:
- build/distributions/**/*
matrix:
- x-pack/agentbeat

- group: DRA publish
key: dra
steps:
- label: DRA Snapshot
## Only for release branches and main
if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT')==true"
if: build.branch =~ /\d+\.\d+/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true"
key: dra-snapshot
env:
DRA_WORKFLOW: snapshot
Expand All @@ -216,7 +208,7 @@ steps:

- label: DRA Staging
## Only for release branches
if: "build.branch =~ /\\d+\\.\\d+/"
if: build.branch =~ /\d+\.\d+/
key: dra-staging
env:
DRA_WORKFLOW: staging
Expand Down
18 changes: 17 additions & 1 deletion .buildkite/scripts/dra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,22 @@ release_manager_login
echo "+++ Changing permissions for the BK API commands"
sudo chown -R :1000 build/distributions/

echo "+++ :hammer_and_pick: Listing $BRANCH $DRA_WORKFLOW DRA artifacts..."
docker run --rm \
--name release-manager \
-e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
-e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
-e VAULT_SECRET_ID="${VAULT_SECRET}" \
--mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
docker.elastic.co/infra/release-manager:latest \
cli list \
--project "beats" \
--branch "${BRANCH}" \
--commit "${BUILDKITE_COMMIT}" \
--workflow "${DRA_WORKFLOW}" \
--version "${BEAT_VERSION}" \
--artifact-set "main"

echo "+++ :hammer_and_pick: Publishing $BRANCH $DRA_WORKFLOW DRA artifacts..."
docker run --rm \
--name release-manager \
Expand All @@ -43,4 +59,4 @@ docker run --rm \
--workflow "${DRA_WORKFLOW}" \
--version "${BEAT_VERSION}" \
--artifact-set "main" \
${DRY_RUN}
${DRY_RUN}
2 changes: 1 addition & 1 deletion .buildkite/scripts/packaging/package-dra.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ if [ -z "$BEAT_DIR" ]; then
exit 1
fi

echo "~~~Packaging : $BEAT_DIR"
echo "~~~ Packaging : $BEAT_DIR"

WORKSPACE=$(pwd)
BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g')
Expand Down
Loading