enabled synamic packaging step

elastic · Jan 29, 2024 · 937fb3c · 937fb3c
1 parent 40fb6ad
commit 937fb3c
Show file tree

Hide file tree

Showing 11 changed files with 165 additions and 189 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -9,135 +9,133 @@ env:
   IMAGE_RHEL9: "family/core-rhel-9"
 
 steps:
-#  - group: "Auditbeat Mandatory Testing"
-#    key: "mandatory-tests"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
-#
-#    steps:
-#      - label: ":ubuntu: Unit Tests"
-#        command:
-#          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat: Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#        artifact_paths:
-#          - "auditbeat/build/*.xml"
-#          - "auditbeat/build/*.json"
-#
-#      - label: ":rhel: Unit Tests"
-#        command:
-#          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat: Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_RHEL9}"
-#        artifact_paths:
-#          - "auditbeat/build/*.xml"
-#          - "auditbeat/build/*.json"
-#
-#      - label: ":windows:-{{matrix.image}} Unit Tests"
-#        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat: Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "{{matrix.image}}"
-#          machine_type: "n2-standard-8"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
-#        matrix:
-#          setup:
-#            image:
-#              - "${IMAGE_WIN_2016}"
-#              - "${IMAGE_WIN_2022}"
-#        artifact_paths:
-#          - "auditbeat/build/*.xml"
-#          - "auditbeat/build/*.json"
-#
-#      - label: ":linux: Crosscompile"
-#        command:
-#          - ".buildkite/auditbeat/scripts/crosscompile.sh"
-#        env:
-#          GOX_FLAGS: "-arch amd64"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat: Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_UBUNTU_X86_64}"
-#
-#  - group: "Extended Testing"
-#    key: "extended-tests"
-#    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat extended tests"
-#
-#    steps:
-#      - label: ":linux: ARM64 Unit Tests"
-#        key: "arm-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-#        command:
-#          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat/Extended: Unit Tests ARM"
-#        agents:
-#          provider: "aws"
-#          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
-#          instanceType: "t4g.large"
-#        artifact_paths: "auditbeat/build/*.xml"
-#
-#      - label: ":mac: MacOS Unit Tests"
-#        key: "macos-extended"
-#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-#        command:
-#          - ".buildkite/auditbeat/scripts/unit-tests.sh"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat/Extended: MacOS Unit Tests"
-#        agents:
-#          provider: "orka"
-#          imagePrefix: "generic-13-ventura-x64"
-#        artifact_paths: "auditbeat/build/*.xml"
-#
-#  - group: "Windows Extended Testing"
-#    key: "extended-tests-win"
-#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
-#
-#    steps:
-#      - label: ":windows: Win 2019 Unit Tests"
-#        key: "win-extended-2019"
-#        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
-#        notify:
-#          - github_commit_status:
-#              context: "auditbeat/Extended: Win-2019 Unit Tests"
-#        agents:
-#          provider: "gcp"
-#          image: "${IMAGE_WIN_2019}"
-#          machine_type: "n2-standard-8"
-#          disk_size: 200
-#          disk_type: "pd-ssd"
-#        artifact_paths:
-#          - "auditbeat/build/*.xml"
-#          - "auditbeat/build/*.json"
+  - group: "Auditbeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":ubuntu: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":rhel: Unit Tests"
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":windows:-{{matrix.image}} Unit Tests"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
+
+      - label: ":linux: Crosscompile"
+        command:
+          - ".buildkite/auditbeat/scripts/crosscompile.sh"
+        env:
+          GOX_FLAGS: "-arch amd64"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat extended tests"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "auditbeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/auditbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "generic-13-ventura-x64"
+        artifact_paths: "auditbeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "auditbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "auditbeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "auditbeat/build/*.xml"
+          - "auditbeat/build/*.json"
 
   - group: "Packaging"
     key: "packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-#    depends_on:
-#      - "mandatory-tests"
-#      - "extended-tests"
-#      - "extended-tests-win"
+    depends_on:
+      - "mandatory-tests"
+      - "extended-tests"
+      - "extended-tests-win"
 
     steps:
-#      - label: Package pipeline
-#        env:
-#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-#        commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload"
+      - label: Package pipeline
+        commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload"
 
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"

diff --git a/.buildkite/auditbeat/scripts/crosscompile.sh b/.buildkite/auditbeat/scripts/crosscompile.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/unix-env.sh
+source .buildkite/env-scripts/linux-env.sh
 
 echo ":: Executing Crosscompile ::"
 make -C auditbeat crosscompile
diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh
@@ -17,11 +17,13 @@ if are_files_changed "$changeset"; then
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
         command:
           - ".buildkite/auditbeat/scripts/package.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Packaging: Linux X86"
+              context: "Auditbeat/Packaging: Linux X86"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -35,11 +37,10 @@ if are_files_changed "$changeset"; then
           - ".buildkite/auditbeat/scripts/package.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Packaging: Linux ARM"
+              context: "Auditbeat/Packaging: ARM"
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
 EOF
 fi
-
diff --git a/.buildkite/auditbeat/scripts/package.sh b/.buildkite/auditbeat/scripts/package.sh
@@ -2,9 +2,10 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/unix-env.sh
+source .buildkite/env-scripts/linux-env.sh
 
-echo ":: Evaluate Auditbeat Changes ::"
+echo ":: Docker Version ::"
+docker --version
 
 echo ":: Start Packaging ::"
 cd auditbeat

diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/env-scripts/unix-env.sh
+source .buildkite/env-scripts/linux-env.sh
 
 echo ":: Running Unit Tests ::"
 sudo chmod -R go-w auditbeat/

diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh
@@ -1,21 +1,44 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 source .buildkite/env-scripts/util.sh
 
 DEBIAN_FRONTEND="noninteractive"
 
-export DEBIAN_FRONTEND
-
 sudo mkdir -p /etc/needrestart
 echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
 
-# Remove this code once beats specific agent is set up
+echo ":: PLATFORM TYPE $PLATFORM_TYPE ::"
+
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
-  echo ":: Installing libs ::"
-  sudo apt-get update
-  sudo apt-get install -y libsystemd-dev
-  sudo apt install -y python3-pip
-  sudo apt-get install -y python3-venv
+  # Remove this code once beats specific agent is set up
+  if grep -q 'Ubuntu' /etc/*release; then
+    export DEBIAN_FRONTEND
+
+    echo ":: Ubuntu - Installing libs ::"
+    sudo apt-get update
+    sudo apt-get install -y libsystemd-dev
+    sudo apt install -y python3-pip
+    sudo apt-get install -y python3-venv
+  fi
+
+  # Remove this code once beats specific agent is set up
+  if grep -q 'Red Hat' /etc/*release; then
+    echo ":: RHL - Installing libs ::"
+    sudo yum update -y
+    sudo yum install -y systemd-devel
+    sudo yum install -y python3-pip
+    sudo yum install -y python3
+    pip3 install virtualenv
+  fi
+fi
+
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 50000
+  echo ":: ULIMIT :: $(ulimit -n)"
 fi
 
 echo ":: Setting up environment ::"