Merge remote-tracking branch 'upstream/main'

.github/ISSUE_TEMPLATE/release_retrospective_template.md

@@ -0,0 +1,14 @@
+---
+name: Release retrospective
+title: "[Retrospective] Release Version {{ env.VERSION }}"
+labels: untriaged, release, v{{ env.VERSION }}
+---
+
+### **Related release issue?**
+{{ env.RELEASE_ISSUE_URL }}
+
+### **How to use this issue?**
+Please add comments to this issue, they can be small or large in scope. Honest feedback is important to improve our processes, suggestions are also welcomed but not required.
+
+### **What will happen to this issue post release?**
+There will be a discussion(s) about how the release went and how the next release can be improved. Then this issue will be updated with the notes of that discussion along side action items.

.github/workflows/os-increment-plugin-versions.yml

@@ -48,13 +48,15 @@ jobs:
           - {repo: custom-codecs}
           - {repo: flow-framework}
           - {repo: skills}
+          - {repo: query-insights}
         branch:
           - 1.x
           - '1.3'
           - 2.x
           - main
           - '2.15'
           - '2.16'
+          - '2.17'
         exclude:
           - {entry: {repo: geospatial}, branch: '1.3'}
           - {entry: {repo: neural-search}, branch: '1.3'}

.github/workflows/os-release-issues.yml

@@ -60,6 +60,7 @@ jobs:
           - {repo: custom-codecs}
           - {repo: flow-framework}
           - {repo: skills}
+          - {repo: query-insights}
         release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }}
     steps:
       - name: GitHub App token

.github/workflows/osd-increment-plugin-versions.yml

@@ -31,6 +31,7 @@ jobs:
           - {repo: dashboards-query-workbench}
           - {repo: dashboards-assistant}
           - {repo: dashboards-maps}
+          - {repo: dashboards-flow-framework}
           - {repo: anomaly-detection-dashboards-plugin}
           - {repo: ml-commons-dashboards}
           - {repo: index-management-dashboards-plugin}
@@ -47,6 +48,7 @@ jobs:
           - main
           - '2.15'
           - '2.16'
+          - '2.17'
     steps:
       - name: Check out OpenSearch Dashboards repo
         uses: actions/checkout@v3
@@ -81,7 +83,7 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/${{ matrix.entry.repo }}
           if [ ${{ matrix.entry.path }} ]; then
-              yarn osd bootstrap
+              yarn osd bootstrap --single-version=loose
               cp -R ${{ matrix.entry.path }} ../
               cd ../${{ matrix.entry.path }}
               node ../../scripts/plugin_helpers version --sync legacy
@@ -97,7 +99,7 @@ jobs:
                 OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version")
                 echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV
           else
-              yarn osd bootstrap
+              yarn osd bootstrap --single-version=loose
               node ../../scripts/plugin_helpers version --sync legacy
               OSD_PLUGIN_VERSION=$(node -p "require('./package.json').version")
               echo "OSD_PLUGIN_VERSION=$OSD_PLUGIN_VERSION" >> $GITHUB_ENV

.github/workflows/osd-release-issues.yml

@@ -44,6 +44,7 @@ jobs:
           - {repo: dashboards-query-workbench}
           - {repo: dashboards-assistant}
           - {repo: dashboards-maps}
+          - {repo: dashboards-flow-framework}
           - {repo: anomaly-detection-dashboards-plugin}
           - {repo: ml-commons-dashboards}
           - {repo: index-management-dashboards-plugin}

.github/workflows/releases.yml

@@ -25,11 +25,24 @@ jobs:
         release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
-      - uses: dblock/create-a-github-issue@v3.0.0
+      - name: Create release issue
+        uses: dblock/create-a-github-issue@v3.0.0
+        id: release-issue
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VERSION: ${{ matrix.release_version }}
         with:
           search_existing: all
           update_existing: false
           filename: .github/ISSUE_TEMPLATE/release_template.md
+      - name: Create retrospective issue
+        uses: dblock/create-a-github-issue@v3.0.0
+        if: ${{ steps.release-issue.outputs.status == 'created' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VERSION: ${{ matrix.release_version }}
+          RELEASE_ISSUE_URL: ${{ steps.release-issue.outputs.url }}
+        with:
+          search_existing: all
+          update_existing: false
+          filename: .github/ISSUE_TEMPLATE/release_retrospective_template.md

jenkins/check-for-build.jenkinsfile

@@ -23,9 +23,9 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
-            H 1 * * * %INPUT_MANIFEST=2.15.1/opensearch-2.15.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-dashboards-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 1 * * * %INPUT_MANIFEST=1.3.19/opensearch-1.3.19.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
         '''

jenkins/docker/docker-copy.jenkinsfile

@@ -48,7 +48,7 @@ pipeline {
         stage("Image Copy") {
             agent {
                 docker {
-                    label 'Jenkins-Agent-AL2023-X64-C54xlarge-Docker-Host'
+                    label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder'
                     image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.9.1-qemu5.0-v1'
                     args '-u root -v /var/run/docker.sock:/var/run/docker.sock'
                     registryUrl 'https://public.ecr.aws/'

jenkins/gradle/gradle-check.jenkinsfile

@@ -16,7 +16,7 @@ pipeline {
     agent { label AGENT_LABEL }
     options {
         timeout(time: 2, unit: 'HOURS')
-        buildDiscarder(logRotator(daysToKeepStr: '180'))
+        buildDiscarder(logRotator(daysToKeepStr: '90'))
         throttleJobProperty(
             categories: [],
             limitOneJobWithMatchingParams: false,

jenkins/opensearch-dashboards/distribution-build.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@6.5.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@6.7.2', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))
@@ -60,6 +60,11 @@ pipeline {
             defaultValue: 'linux windows',
             trim: true
         )
+        string(
+            name: 'RC_NUMBER',
+            description: 'The RC build count. Default is 0 which means its not an RC build.',
+            defaultValue: '0'
+        )
         string( // Note: need to update 'verify-parameters' entries if you add new distribution(s)
             name: 'BUILD_DISTRIBUTION',
             description: 'Build selected distribution related artifacts, choices include tar, rpm, deb, zip. Can combine multiple distributions with space in between (docker is only available on tar)',
@@ -941,6 +946,15 @@ pipeline {
             node(AGENT_LINUX_X64) {
                 checkout scm
                 script {
+                    def rc = (params.RC_NUMBER.toInteger() > 0)
+                    publishDistributionBuildResults(
+                        failureMessages: buildMessage(search: 'Error building'),
+                        passMessages: buildMessage(search: 'Successfully built'),
+                        inputManifestPath: "manifests/$INPUT_MANIFEST",
+                        rc: rc,
+                        rcNumber: env.RC_NUMBER,
+                        componentCategory: "OpenSearch Dashboards"
+                    )
                     if (params.UPDATE_GITHUB_ISSUE) {
                         UpdateBuildFailureIssues(
                             failureMessages: buildMessage(search: 'Error building'),
@@ -1029,6 +1043,7 @@ def triggerIntegrationTests(String buildManifestUrl, String buildManifestUrlOpen
                 parameters: [
                     string(name: 'TEST_MANIFEST', value: TEST_MANIFEST),
                     string(name: 'BUILD_MANIFEST_URL', value: buildManifestUrl),
+                    string(name: 'RC_NUMBER', value: RC_NUMBER),
                     string(name: 'BUILD_MANIFEST_URL_OPENSEARCH', value: buildManifestUrlOpenSearch),
                     booleanParam(name: 'UPDATE_GITHUB_ISSUES', value: true)
                 ]

jenkins/opensearch-dashboards/integ-test.jenkinsfile

@@ -7,8 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@6.5.0', retriever: modernSCM([
-
+lib = library(identifier: 'jenkins@6.7.2', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))
@@ -36,7 +35,7 @@ def agent_nodes = [
 pipeline {
     options {
         timeout(time: 7, unit: 'HOURS')
-        buildDiscarder(logRotator(daysToKeepStr: '30'))
+        buildDiscarder(logRotator(daysToKeepStr: '60'))
     }
     agent none
     environment {
@@ -72,6 +71,11 @@ pipeline {
             description: 'If true results will create / update / close github issues',
             defaultValue: false
         )
+        string(
+            name: 'RC_NUMBER',
+            description: 'The RC build count. Default is 0 which means its not an RC build.',
+            defaultValue: '0'
+        )
         booleanParam(
             name: 'VALIDATE_ARTIFACTS',
             description: 'If true the provided artifacts are validated before triggering integ-tests else skip the validation stage',
@@ -335,31 +339,53 @@ pipeline {
             post {
                 always {
                     script {
-                        node(AGENT_LABEL) {
-                            docker.withRegistry('https://public.ecr.aws/') {
-                                docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
-                                    checkout scm
-                                    downloadBuildManifest(
-                                        url: BUILD_MANIFEST_URL,
-                                        path: BUILD_MANIFEST
-                                    )
-                                    downloadBuildManifest(
-                                        url: BUILD_MANIFEST_URL_OPENSEARCH,
-                                        path: BUILD_MANIFEST_OPENSEARCH
-                                    )
-                                    createUploadTestReportManifest(
-                                        testManifest: "manifests/${TEST_MANIFEST}",
-                                        buildManifest: BUILD_MANIFEST_OPENSEARCH,
-                                        dashboardsBuildManifest: BUILD_MANIFEST,
-                                        testRunID: "${env.BUILD_NUMBER}",
-                                        testType: "integ-test",
-                                        componentName: "${COMPONENT_NAME}",
-                                    )
-                                    echo("Download report manifest from https://ci.opensearch.org/ci/dbc/integ-test/${env.version}/${env.buildId}/${env.platform}/${env.architecture}/${env.distribution}/test-results/${env.BUILD_NUMBER}/integ-test/test-report.yml")
-                                    archiveArtifacts artifacts: 'test-report.yml'
+                        retry(5) {
+                            node(AGENT_LABEL) {
+                                docker.withRegistry('https://public.ecr.aws/') {
+                                    docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
+                                        checkout scm
+                                        downloadBuildManifest(
+                                            url: BUILD_MANIFEST_URL,
+                                            path: BUILD_MANIFEST
+                                        )
+                                        downloadBuildManifest(
+                                            url: BUILD_MANIFEST_URL_OPENSEARCH,
+                                            path: BUILD_MANIFEST_OPENSEARCH
+                                        )
+                                        createUploadTestReportManifest(
+                                            testManifest: "manifests/${TEST_MANIFEST}",
+                                            buildManifest: BUILD_MANIFEST_OPENSEARCH,
+                                            dashboardsBuildManifest: BUILD_MANIFEST,
+                                            testRunID: "${env.BUILD_NUMBER}",
+                                            testType: "integ-test",
+                                            componentName: "${COMPONENT_NAME}",
+                                        )
+                                        echo("Download report manifest from https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/${env.version}/${env.buildId}/${env.platform}/${env.architecture}/${env.distribution}/test-results/${env.BUILD_NUMBER}/integ-test/test-report.yml")
+                                        archiveArtifacts artifacts: 'test-report.yml'
+                                    }
                                 }
                             }
                         }
+                        retry(5) {
+                            node(agent_nodes['linux_x64']) {
+                                def rc = (params.RC_NUMBER.toInteger() > 0)
+                                sh "mkdir -p test-results-osd-${env.BUILD_NUMBER}"
+                                sh "curl -sSL https://ci.opensearch.org/ci/dbc/integ-test-opensearch-dashboards/${env.version}/${env.buildId}/${env.platform}/${env.architecture}/${env.distribution}/test-results/${env.BUILD_NUMBER}/integ-test/test-report.yml --output test-results-osd-${env.BUILD_NUMBER}/test-report.yml"
+                                publishIntegTestResults(
+                                    version: env.version,
+                                    distributionBuildNumber: env.buildId,
+                                    distributionBuildUrl: "https://build.ci.opensearch.org/blue/organizations/jenkins/${env.BUILD_JOB_NAME}/detail/${env.BUILD_JOB_NAME}/${env.buildId}/pipeline",
+                                    rc: rc,
+                                    jobName: env.JOB_NAME,
+                                    rcNumber: env.RC_NUMBER,
+                                    platform: env.platform,
+                                    architecture: env.architecture,
+                                    distribution: env.distribution,
+                                    testReportManifestYml: "test-results-osd-${env.BUILD_NUMBER}/test-report.yml"
+                                )
+                                postCleanup()
+                            }
+                        }
                     }
                     postCleanup()
                 }