Support snapshot uploading to s3 for every build

jenkins/opensearch/distribution-build.jenkinsfile

@@ -27,7 +27,7 @@ pipeline {
         }
         stage('build') {
             parallel {
-                stage('build-snapshot') {
+                stage('build-snapshot-x64') {
                     environment {
                         SNAPSHOT_REPO_URL = "https://aws.oss.sonatype.org/content/repositories/snapshots/"
                     }
@@ -52,6 +52,41 @@ pipeline {
                             } else {
                                 echo "Skipping publishing snapshots, builds/opensearch/maven does not exist."
                             }
+
+
+                            if (fileExists("$WORKSPACE/builds/opensearch/dist")){
+                                uploadMinSnapshotsToS3()
+                            } else {
+                                echo "Dist folder not found, skipping snapshots upload to s3"
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            postCleanup()
+                        }
+                    }
+                }
+                stage('build-snapshot-arm64') {
+                    agent {
+                        docker {
+                            label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
+                            image dockerAgent.image
+                            args dockerAgent.args
+                            alwaysPull true
+                        }
+                    }
+                    steps {
+                        script {
+                            git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
+                            buildManifest(
+                                snapshot: true
+                            )
+                            if (fileExists("$WORKSPACE/builds/opensearch/dist")){
+                                uploadMinSnapshotsToS3()
+                            } else {
+                                echo "Dist folder not found, skipping snapshots upload to s3"
+                            }
                         }
                     }
                     post {

tests/jenkins/UploadMinSnapshotsToS3.groovy

@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins.tests
+
+import org.junit.*
+import java.util.*
+import java.nio.file.*
+
+class UploadMinSnapshotsToS3 extends BuildPipelineTest {
+    private Path target;
+
+    @Before
+    void setUp() {
+        super.setUp()
+
+        binding.setVariable('WORKSPACE', 'workspace')
+
+        binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'dummy_role')
+        binding.setVariable('AWS_ACCOUNT_ARTIFACT', '1234')
+        binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'dummy_bucket')
+
+        Path source = Path.of("tests/data/opensearch-build-1.1.0.yml");
+        target = Path.of("workspace/builds/opensearch/manifest.yml");
+        Files.createDirectories(target.getParent());
+        Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+
+        helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
+            closure.delegate = delegate
+            return helper.callClosure(closure)
+        })
+    }
+    @Test
+    public void test() {
+        super.testPipeline("tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile")
+    }
+
+    @After
+    void after() {
+        super.setUp()
+        Files.delete(target) // Test file needs to be cleaned up
+    }
+}
+

tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile

@@ -0,0 +1,14 @@
+pipeline {
+    agent none
+    stages {
+        stage('uploadMinSnapshotsToS3') {
+            steps {
+                script {
+                    uploadMinSnapshotsToS3(
+                        manifest: "tests/jenkins/data/opensearch-1.3.0.yml"
+                        )
+                }
+            }
+        }
+    }
+}

tests/jenkins/jobs/uploadSnapshotsToS3_Jenkinsfile.txt

@@ -0,0 +1,14 @@
+   uploadSnapshotsToS3_Jenkinsfile.run()
+      uploadSnapshotsToS3_Jenkinsfile.pipeline(groovy.lang.Closure)
+         uploadSnapshotsToS3_Jenkinsfile.echo(Executing on agent [label:none])
+         uploadSnapshotsToS3_Jenkinsfile.stage(uploadMinSnapshotsToS3, groovy.lang.Closure)
+            uploadSnapshotsToS3_Jenkinsfile.script(groovy.lang.Closure)
+               uploadSnapshotsToS3_Jenkinsfile.uploadMinSnapshotsToS3({manifest=tests/jenkins/data/opensearch-1.3.0.yml})
+                  uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure)
+                  uploadMinSnapshotsToS3.library({identifier=jenkins@20211123, retriever=null})
+                  uploadMinSnapshotsToS3.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml})
+                  InputManifest.asBoolean()
+                  uploadMinSnapshotsToS3.readYaml({file=workspace/builds/opensearch/manifest.yml})
+                  BuildManifest.asBoolean()
+                  uploadMinSnapshotsToS3.withAWS({role=dummy_role, roleAccount=1234, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                     uploadMinSnapshotsToS3.s3Upload({file=workspace/builds/opensearch/dist/, bucket=dummy_bucket, path=snapshots/core/opensearch/1.1.0/opensearch-min-1.1.0-linux-x64-latest.tar.gz})

vars/uploadMinSnapshotsToS3.groovy

@@ -0,0 +1,16 @@
+void call(Map args = [:]) {
+    def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
+    String manifest = args.manifest ?: "manifests/${INPUT_MANIFEST}"
+
+    def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest))
+    def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: "$WORKSPACE/builds/${inputManifest.build.getFilename()}/manifest.yml"))
+    version = buildManifest.build.version
+    architecture = buildManifest.build.architecture
+    platform = buildManifest.build.platform
+    minArtifactName = "opensearch-min-${version}-${platform}-${architecture}-latest.tar.gz"
+
+
+    withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
+        s3Upload(file: "$WORKSPACE/builds/opensearch/dist/", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "snapshots/core/opensearch/${version}/${minArtifactName}")
+        }
+}