From 82f94f488ad255bd954fdd0dbd9d823e0bffb049 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 16 Oct 2020 09:07:08 +0100 Subject: [PATCH] [CI] Enable build stage for arm architecture (#21284) (#21839) --- .ci/scripts/install-go.sh | 27 +++++++++++++++++++++++++-- Jenkinsfile | 23 +++++++++++++++++------ auditbeat/Jenkinsfile.yml | 15 ++++++++++++++- filebeat/Jenkinsfile.yml | 13 +++++++++++++ heartbeat/Jenkinsfile.yml | 13 +++++++++++++ journalbeat/Jenkinsfile.yml | 13 +++++++++++++ libbeat/Jenkinsfile.yml | 11 +++++++++++ packetbeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/auditbeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/elastic-agent/Jenkinsfile.yml | 13 +++++++++++++ x-pack/filebeat/Jenkinsfile.yml | 13 +++++++++++++ x-pack/functionbeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/libbeat/Jenkinsfile.yml | 13 +++++++++++++ 13 files changed, 182 insertions(+), 9 deletions(-) diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 5af9f338ca1..6e4afa0fb41 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -5,12 +5,35 @@ MSG="parameter missing." GO_VERSION=${GO_VERSION:?$MSG} PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} HOME=${HOME:?$MSG} -ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +OS=$(uname -s| tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m| tr '[:upper:]' '[:lower:]') GVM_CMD="${HOME}/bin/gvm" +if command -v go +then + echo "Found Go. Checking version.." + FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//) + if [ $FOUND_GO_VERSION == $GO_VERSION ] + then + echo "Versions match. No need to install Go. Exiting." + exit 0 + fi +fi + +if [ "${ARCH}" == "aarch64" ] ; then + GVM_ARCH_SUFFIX=arm64 +elif [ "${ARCH}" == "x86_64" ] ; then + GVM_ARCH_SUFFIX=amd64 +elif [ "${ARCH}" == "i686" ] ; then + GVM_ARCH_SUFFIX=386 +else + GVM_ARCH_SUFFIX=arm +fi + +echo "UNMET DEP: Installing Go" mkdir -p "${HOME}/bin" -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}" chmod +x "${GVM_CMD}" gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} diff --git a/Jenkinsfile b/Jenkinsfile index 08abbe40e3c..1f8310682c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,6 @@ pipeline { DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' @@ -30,7 +29,7 @@ pipeline { XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') @@ -47,6 +46,7 @@ pipeline { booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`') } @@ -224,10 +224,17 @@ def withBeatsEnv(Map args = [:], Closure body) { def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def goRoot, path, magefile, pythonEnv, testResults, artifacts + def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags if(isUnix()) { - goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + if (isArm() && is64arm()) { + // TODO: nodeOS() should support ARM + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64" + gox_flags = '-arch arm' + } else { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + gox_flags = '-arch amd64' + } path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" magefile = "${WORKSPACE}/.magefile" pythonEnv = "${WORKSPACE}/python-env" @@ -241,6 +248,7 @@ def withBeatsEnv(Map args = [:], Closure body) { magefile = "${env.WORKSPACE}\\.magefile" testResults = "**\\build\\TEST*.xml" artifacts = "**\\build\\TEST*.out" + gox_flags = '-arch amd64' } deleteDir() @@ -258,7 +266,8 @@ def withBeatsEnv(Map args = [:], Closure body) { "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", "TEST_COVERAGE=true", - "TEST_TAGS=${env.TEST_TAGS},oracle" + "TEST_TAGS=${env.TEST_TAGS},oracle", + "GOX_FLAGS=${gox_flags}" ]) { if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") @@ -362,7 +371,9 @@ def archiveTestOutput(Map args = [:]) { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() + // TODO: nodeOS() should support ARM + def os_suffix = isArm() ? 'linux' : nodeOS() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix tar(file: "${name}.tgz", archive: true, dir: folder) } } diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 6f5374224d4..7eed06a497c 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -13,9 +13,22 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" - crosscompile: + crosscompile: make: "make -C auditbeat crosscompile" macos: mage: "mage build unitTest" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index ffe2cec98b4..09dbe948c72 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index c465138791b..c054be0cd53 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test heartbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 12bb63f4cc6..5715712dd4a 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -13,5 +13,18 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test journalbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags unitTest: mage: "mage build unitTest" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml index 64a43269b13..692400e7253 100644 --- a/libbeat/Jenkinsfile.yml +++ b/libbeat/Jenkinsfile.yml @@ -12,6 +12,17 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test" crosscompile: diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index f755f322477..7ecdbbfca8f 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test packetbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 969aa0a8e08..5850d9947b4 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/auditbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage update build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 04a6b227721..3a4c54c414d 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/elastic-agent for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" macos: diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index b2db448f532..d8cef3dd206 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -13,6 +13,19 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/filebeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index a8d42ce5fb9..098746ff606 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -13,6 +13,17 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/functionbeat for arm" + labels: + - "arm" + parameters: + - "armTest" build: mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" macos: diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml index 87019f071a0..ed22a8dfe70 100644 --- a/x-pack/libbeat/Jenkinsfile.yml +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -13,5 +13,18 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/libbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags build: mage: "mage build test"