diff --git a/.github/ISSUE_TEMPLATE/release_template.md b/.github/ISSUE_TEMPLATE/release_template.md index 2cb98550fc..2905537aa3 100644 --- a/.github/ISSUE_TEMPLATE/release_template.md +++ b/.github/ISSUE_TEMPLATE/release_template.md @@ -47,6 +47,7 @@ To aid in understanding the state of the release there is a table with status in - [ ] Increment each component version to {{ env.VERSION }} and ensure working CI in component repositories. - [ ] Make pull requests to add each component to [manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-{{ env.VERSION }}.yml) and [manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}/opensearch-dashboards-{{ env.VERSION }}.yml) with the corresponding checks. - [ ] OpenSearch / OpenSearch-Dashboards core and components teams finalize their features +- [ ] OpenSearch / OpenSearch-Dashboards core cut branch `.` early. ### Campaigns @@ -60,7 +61,9 @@ __REPLACE with OpenSearch wide initiatives to improve quality and consistency.__ ### Release testing - _Ends __REPLACE_RELEASE-minus-6-days___ +- [ ] All components should have cut branch `.` for the release. - [ ] Declare a release candidate build, and provide the instructions with the release candidates for teams on testing (__REPLACE_RELEASE-minus-8-days__). +- [ ] Stop builds for this version of OpenSearch and/or OpenSearch Dashboards in order to avoid accidental commits going in unknowingly. Restart only if necessary else manually run the build workflow and declare new release candidate. - [ ] Sanity Testing (__REPLACE_RELEASE-minus-8-days__ - __REPLACE_RELEASE-minus-6-days__): Sanity testing and fixing of critical issues found by teams. Teams test their components within the distribution, ensuring integration, backwards compatibility, and perf tests pass. - [ ] Publish all test results in the comments of this issue. @@ -81,11 +84,11 @@ __REPLACE with OpenSearch wide initiatives to improve quality and consistency.__ - [ ] Create [release tags](https://github.com/opensearch-project/opensearch-build/blob/main/jenkins/release-tag/release-tag.jenkinsfile) for each component (Jenkins job name: release-tag-creation). - [ ] Replace refs in [manifests/{{ env.VERSION }}](/opensearch-project/opensearch-build/tree/main/manifests/{{ env.VERSION }}) with tags and remove checks. +- [ ] If this is a major or minor version release, stop building previous patch version. - [ ] Generate distribution release notes reviewed by PM team for opensearch-build repository. - [ ] Increment version for Helm Charts [(sample PR)](https://github.com/opensearch-project/helm-charts/pull/246) for the `{{ env.VERSION }}` release. - [ ] Increment version for Ansible Charts [(sample PR)](https://github.com/opensearch-project/ansible-playbook/pull/50) for the `{{ env.VERSION }}` release. - [ ] Prepare [for next patch release](https://github.com/opensearch-project/opensearch-plugins/blob/main/META.md#increment-a-version-in-every-plugin) by incrementing patch versions for each component. -- [ ] Lower the [frequency of builds](https://github.com/opensearch-project/opensearch-build/pull/1475) for this version of OpenSearch and/or OpenSearch Dashboards. - [ ] Update [this template](https://github.com/opensearch-project/opensearch-build/blob/main/.github/ISSUE_TEMPLATE/release_template.md) with any new or missed steps. - [ ] Create an issue for a retrospective, solicit feedback, and publish a summary. @@ -93,7 +96,7 @@ __REPLACE with OpenSearch wide initiatives to improve quality and consistency.__ __Replace with links to all component tracking issues.__ -| Component | On track | Notes | +| Component | On track | Release Notes | | --------- | -------- | ----- | | {COMPONENT_ISSUE_LINK} | {INDICATOR}} | {STATUS} | diff --git a/.github/workflows/increment-plugin-versions.yml b/.github/workflows/increment-plugin-versions.yml new file mode 100644 index 0000000000..8164fe4bdb --- /dev/null +++ b/.github/workflows/increment-plugin-versions.yml @@ -0,0 +1,97 @@ +--- +name: Increment Plugin Versions + +on: + workflow_dispatch: + inputs: + logLevel: + description: Log level + required: true + default: warning + type: choice + options: + - info + - warning + - debug + +jobs: + plugin-version-increment-sync: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + entry: + - {repo: alerting} + - {repo: anomaly-detection} + - {repo: asynchronous-search} + - {repo: common-utils} + - {repo: cross-cluster-replication} + - {repo: dashboards-reports, path: reports-scheduler} + - {repo: geospatial} + - {repo: index-management} + - {repo: job-scheduler} + - {repo: k-NN} + - {repo: ml-commons} + - {repo: notifications, path: notifications} + - {repo: observability, path: opensearch-observability} + - {repo: performance-analyzer} + - {repo: performance-analyzer-rca} + - {repo: security} + - {repo: sql} + branch: + - 'main' + - '2.x' + - '2.1' + - '2.2' + steps: + - name: Check out OpenSearch repo + uses: actions/checkout@v3 + with: + repository: opensearch-project/OpenSearch + ref: ${{ matrix.branch }} + - name: Fetch OpenSearch version + run: | + OPENSEARCH_VERSION=$(cat buildSrc/version.properties | grep opensearch | cut -d= -f2 | grep -oE '[0-9.]+') + echo "OPENSEARCH_VERSION_NUMBER=$OPENSEARCH_VERSION" >> $GITHUB_ENV + OPENSEARCH_VERSION=$OPENSEARCH_VERSION-SNAPSHOT + echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION" >> $GITHUB_ENV + - name: Check out plugin repo + uses: actions/checkout@v3 + with: + repository: opensearch-project/${{ matrix.entry.repo }} + ref: ${{ matrix.branch }} + - name: Increment Version in ${{ matrix.entry.repo }} + run: | + echo "OpenSearch Core repo version on branch ${{ matrix.branch }} is ${{ env.OPENSEARCH_VERSION_NUMBER }}" + if [ ${{ matrix.entry.path }} ]; then + echo "The gradle path is ${{ matrix.entry.path }}" + cd ${{ matrix.entry.path }} + fi + ./gradlew updateVersion -DnewVersion=${{ env.OPENSEARCH_VERSION }} + - name: GitHub App token + id: github_app_token + uses: tibdex/github-app-token@v1.6.0 + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + installation_id: 27448020 + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ steps.github_app_token.outputs.token }} + committer: opensearch-ci-bot + author: opensearch-ci-bot + commit-message: | + Increment version to ${{ env.OPENSEARCH_VERSION }} + + Signed-off-by: opensearch-ci-bot + delete-branch: true + branch: create-pull-request/${{ env.OPENSEARCH_VERSION }} + title: '[AUTO] Increment version to ${{ env.OPENSEARCH_VERSION }}' + body: | + - Incremented version to **${{ env.OPENSEARCH_VERSION }}**. + - name: Check outputs + run: |- + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/manifests.yml b/.github/workflows/manifests.yml index e01a5282cb..4495916b3b 100644 --- a/.github/workflows/manifests.yml +++ b/.github/workflows/manifests.yml @@ -5,30 +5,65 @@ on: push: pull_request: paths: - - manifests/**/*.yml + - 'manifests/**/*.yml' + - '!manifests/templates/**/' schedule: - cron: 0 0 * * * jobs: + list-manifests11: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - id: set-matrix + run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | awk -F/ '{if($2<2)print$0}' | jq -R -s -c 'split("\n")[:-1]')" - list-manifests: + list-manifests17: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v2 - id: set-matrix - run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | jq -R -s -c 'split("\n")[:-1]')" + run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | awk -F/ '{if($2>2)print$0}' | jq -R -s -c 'split("\n")[:-1]')" + + manifest-checks-jdk11: + needs: list-manifests11 + runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.7 + JDK_VERSION: 11 + strategy: + matrix: + manifest: ${{ fromJson(needs.list-manifests11.outputs.matrix) }} + steps: + - uses: actions/checkout@v2 + - name: Set Up JDK ${{ env.JDK_VERSION }} + uses: actions/setup-java@v1 + with: + java-version: ${{ env.JDK_VERSION }} + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install Pipenv and Dependencies + run: | + python -m pip install --upgrade pipenv wheel + - name: OpenSearch Manifests + run: |- + ./ci.sh ${{ matrix.manifest }} --snapshot - check: - needs: list-manifests + manifest-checks-jdk17: + needs: list-manifests17 runs-on: ubuntu-latest env: PYTHON_VERSION: 3.7 - JDK_VERSION: 14 + JDK_VERSION: 17 strategy: matrix: - manifest: ${{ fromJson(needs.list-manifests.outputs.matrix) }} + manifest: ${{ fromJson(needs.list-manifests17.outputs.matrix) }} steps: - uses: actions/checkout@v2 - name: Set Up JDK ${{ env.JDK_VERSION }} diff --git a/Pipfile b/Pipfile index 4bf457ba7a..abcc663357 100644 --- a/Pipfile +++ b/Pipfile @@ -24,7 +24,10 @@ atomicwrites = "*" validators = "*" yamlfix = "*" yamllint = "*" - +pytablewriter = "*" +# With pytablewriter the python version expects >3.6, importlib-metadata and typed-ast has to be reimported with new versions. +importlib-metadata = "*" +typed-ast = "*" [dev-packages] [requires] diff --git a/Pipfile.lock b/Pipfile.lock index f5e633f5ee..09dbde7780 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3da3cb40cf6d9fb381ad4499ae93f01db922e93f1bed2fd233bb880b2ad422cd" + "sha256": "79ce852ba8c77dd843d326387e74fc581f46636c10e07437e900cdcca0d8d8d9" }, "pipfile-spec": 6, "requires": { @@ -18,19 +18,18 @@ "default": { "atomicwrites": { "hashes": [ - "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", - "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a" + "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11" ], "index": "pypi", - "version": "==1.4.0" + "version": "==1.4.1" }, "attrs": { "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" + "markers": "python_version >= '3.5'", + "version": "==22.1.0" }, "cerberus": { "hashes": [ @@ -41,10 +40,11 @@ }, "certifi": { "hashes": [ - "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872", - "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "version": "==2021.10.8" + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" }, "cfgv": { "hashes": [ @@ -54,21 +54,29 @@ "markers": "python_full_version >= '3.6.1'", "version": "==3.3.1" }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, "charset-normalizer": { "hashes": [ - "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", - "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_version >= '3'", - "version": "==2.0.12" + "markers": "python_version >= '3.6'", + "version": "==2.1.0" }, "click": { "hashes": [ - "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", - "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" ], - "markers": "python_version >= '3.6'", - "version": "==8.0.4" + "markers": "python_version >= '3.7'", + "version": "==8.1.3" }, "coverage": { "hashes": [ @@ -108,6 +116,14 @@ "index": "pypi", "version": "==4.5.4" }, + "dataproperty": { + "hashes": [ + "sha256:73ccf10f8b123968210438a1a1aa859ea6d5a16b4e1f4d307da7a81b838e79fa", + "sha256:a8f29175950f4b2c33a387aa3809130d87b9a8d3b92a916c906c49efdb566b32" + ], + "markers": "python_version >= '3.6'", + "version": "==0.55.0" + }, "decorator": { "hashes": [ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", @@ -118,10 +134,10 @@ }, "distlib": { "hashes": [ - "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b", - "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579" + "sha256:a7f75737c70be3b25e2bee06288cec4e4c221de18455b2dd037fe2a795cab2fe", + "sha256:b710088c59f06338ca514800ad795a132da19fda270e3ce4affc74abf955a26c" ], - "version": "==0.3.4" + "version": "==0.3.5" }, "distro": { "hashes": [ @@ -133,11 +149,11 @@ }, "filelock": { "hashes": [ - "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", - "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" + "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404", + "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04" ], "markers": "python_version >= '3.7'", - "version": "==3.6.0" + "version": "==3.7.1" }, "flake8": { "hashes": [ @@ -149,27 +165,27 @@ }, "identify": { "hashes": [ - "sha256:3f3244a559290e7d3deb9e9adc7b33594c1bc85a9dd82e0f1be519bf12a1ec17", - "sha256:5f06b14366bd1facb88b00540a1de05b69b310cbc2654db3c7e07fa3a4339323" + "sha256:a3d4c096b384d50d5e6dc5bc8b9bc44f1f61cefebd750a7b3e9f939b53fb214d", + "sha256:feaa9db2dc0ce333b453ce171c0cf1247bbfde2c55fc6bb785022d411a1b78b5" ], "markers": "python_version >= '3.7'", - "version": "==2.4.12" + "version": "==2.5.2" }, "idna": { "hashes": [ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], - "markers": "python_version >= '3'", + "markers": "python_version >= '3.5'", "version": "==3.3" }, "importlib-metadata": { "hashes": [ - "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6", - "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539" + "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670", + "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23" ], - "markers": "python_version < '3.8'", - "version": "==4.11.3" + "index": "pypi", + "version": "==4.12.0" }, "iniconfig": { "hashes": [ @@ -194,6 +210,14 @@ "index": "pypi", "version": "==2.1.1" }, + "mbstrdecoder": { + "hashes": [ + "sha256:153443c34b5e0c9b2263a1f8480d33156190b3e03e1e89cccd9239581aaef0ed", + "sha256:f4dfd549e424ad8dfc985e6af8b55cb4ec0c208782f610d57439fe6a9a44c244" + ], + "markers": "python_version >= '3.6'", + "version": "==1.1.0" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -203,32 +227,32 @@ }, "mypy": { "hashes": [ - "sha256:080097eee5393fd740f32c63f9343580aaa0fb1cda0128fd859dfcf081321c3d", - "sha256:0d3bcbe146247997e03bf030122000998b076b3ac6925b0b6563f46d1ce39b50", - "sha256:0dd441fbacf48e19dc0c5c42fafa72b8e1a0ba0a39309c1af9c84b9397d9b15a", - "sha256:108f3c7e14a038cf097d2444fa0155462362c6316e3ecb2d70f6dd99cd36084d", - "sha256:3bada0cf7b6965627954b3a128903a87cac79a79ccd83b6104912e723ef16c7b", - "sha256:3cf77f138efb31727ee7197bc824c9d6d7039204ed96756cc0f9ca7d8e8fc2a4", - "sha256:42c216a33d2bdba08098acaf5bae65b0c8196afeb535ef4b870919a788a27259", - "sha256:465a6ce9ca6268cadfbc27a2a94ddf0412568a6b27640ced229270be4f5d394d", - "sha256:6a8e1f63357851444940351e98fb3252956a15f2cabe3d698316d7a2d1f1f896", - "sha256:745071762f32f65e77de6df699366d707fad6c132a660d1342077cbf671ef589", - "sha256:818cfc51c25a5dbfd0705f3ac1919fff6971eb0c02e6f1a1f6a017a42405a7c0", - "sha256:8e5974583a77d630a5868eee18f85ac3093caf76e018c510aeb802b9973304ce", - "sha256:8eaf55fdf99242a1c8c792247c455565447353914023878beadb79600aac4a2a", - "sha256:98f61aad0bb54f797b17da5b82f419e6ce214de0aa7e92211ebee9e40eb04276", - "sha256:b2ce2788df0c066c2ff4ba7190fa84f18937527c477247e926abeb9b1168b8cc", - "sha256:b30d29251dff4c59b2e5a1fa1bab91ff3e117b4658cb90f76d97702b7a2ae699", - "sha256:bf446223b2e0e4f0a4792938e8d885e8a896834aded5f51be5c3c69566495540", - "sha256:cbcc691d8b507d54cb2b8521f0a2a3d4daa477f62fe77f0abba41e5febb377b7", - "sha256:d051ce0946521eba48e19b25f27f98e5ce4dbc91fff296de76240c46b4464df0", - "sha256:d61b73c01fc1de799226963f2639af831307fe1556b04b7c25e2b6c267a3bc76", - "sha256:eea10982b798ff0ccc3b9e7e42628f932f552c5845066970e67cd6858655d52c", - "sha256:f79137d012ff3227866222049af534f25354c07a0d6b9a171dba9f1d6a1fdef4", - "sha256:fc5ecff5a3bbfbe20091b1cad82815507f5ae9c380a3a9bf40f740c70ce30a9b" + "sha256:02ef476f6dcb86e6f502ae39a16b93285fef97e7f1ff22932b657d1ef1f28655", + "sha256:0d054ef16b071149917085f51f89555a576e2618d5d9dd70bd6eea6410af3ac9", + "sha256:19830b7dba7d5356d3e26e2427a2ec91c994cd92d983142cbd025ebe81d69cf3", + "sha256:1f7656b69974a6933e987ee8ffb951d836272d6c0f81d727f1d0e2696074d9e6", + "sha256:23488a14a83bca6e54402c2e6435467a4138785df93ec85aeff64c6170077fb0", + "sha256:23c7ff43fff4b0df93a186581885c8512bc50fc4d4910e0f838e35d6bb6b5e58", + "sha256:25c5750ba5609a0c7550b73a33deb314ecfb559c350bb050b655505e8aed4103", + "sha256:2ad53cf9c3adc43cf3bea0a7d01a2f2e86db9fe7596dfecb4496a5dda63cbb09", + "sha256:3fa7a477b9900be9b7dd4bab30a12759e5abe9586574ceb944bc29cddf8f0417", + "sha256:40b0f21484238269ae6a57200c807d80debc6459d444c0489a102d7c6a75fa56", + "sha256:4b21e5b1a70dfb972490035128f305c39bc4bc253f34e96a4adf9127cf943eb2", + "sha256:5a361d92635ad4ada1b1b2d3630fc2f53f2127d51cf2def9db83cba32e47c856", + "sha256:77a514ea15d3007d33a9e2157b0ba9c267496acf12a7f2b9b9f8446337aac5b0", + "sha256:855048b6feb6dfe09d3353466004490b1872887150c5bb5caad7838b57328cc8", + "sha256:9796a2ba7b4b538649caa5cecd398d873f4022ed2333ffde58eaf604c4d2cb27", + "sha256:98e02d56ebe93981c41211c05adb630d1d26c14195d04d95e49cd97dbc046dc5", + "sha256:b793b899f7cf563b1e7044a5c97361196b938e92f0a4343a5d27966a53d2ec71", + "sha256:d1ea5d12c8e2d266b5fb8c7a5d2e9c0219fedfeb493b7ed60cd350322384ac27", + "sha256:d2022bfadb7a5c2ef410d6a7c9763188afdb7f3533f22a0a32be10d571ee4bbe", + "sha256:d3348e7eb2eea2472db611486846742d5d52d1290576de99d59edeb7cd4a42ca", + "sha256:d744f72eb39f69312bc6c2abf8ff6656973120e2eb3f3ec4f758ed47e414a4bf", + "sha256:ef943c72a786b0f8d90fd76e9b39ce81fb7171172daf84bf43eaf937e9f220a9", + "sha256:f2899a3cbd394da157194f913a931edfd4be5f274a88041c9dc2d9cdcb1c315c" ], "index": "pypi", - "version": "==0.941" + "version": "==0.971" }, "mypy-extensions": { "hashes": [ @@ -239,10 +263,11 @@ }, "nodeenv": { "hashes": [ - "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b", - "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7" + "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e", + "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b" ], - "version": "==1.6.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==1.7.0" }, "packaging": { "hashes": [ @@ -259,13 +284,21 @@ ], "version": "==0.9.0" }, + "pathvalidate": { + "hashes": [ + "sha256:8dbbc64e78e874ddff049ac187499d8bf34f890adb8b7f657e134a842832d3b9", + "sha256:bbc27e653335aba7935a2ade2299622e76a9487bc9004cdcf1441ce8d2ff4a54" + ], + "markers": "python_version >= '3.6'", + "version": "==2.5.1" + }, "platformdirs": { "hashes": [ - "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", - "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" ], "markers": "python_version >= '3.7'", - "version": "==2.5.1" + "version": "==2.5.2" }, "pluggy": { "hashes": [ @@ -285,41 +318,41 @@ }, "psutil": { "hashes": [ - "sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5", - "sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a", - "sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4", - "sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841", - "sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d", - "sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d", - "sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0", - "sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845", - "sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf", - "sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b", - "sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07", - "sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618", - "sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2", - "sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd", - "sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666", - "sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce", - "sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3", - "sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d", - "sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25", - "sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492", - "sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b", - "sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d", - "sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2", - "sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203", - "sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2", - "sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94", - "sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9", - "sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64", - "sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56", - "sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3", - "sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c", - "sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3" + "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685", + "sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc", + "sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36", + "sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1", + "sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329", + "sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81", + "sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de", + "sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4", + "sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574", + "sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237", + "sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22", + "sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b", + "sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0", + "sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954", + "sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021", + "sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537", + "sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87", + "sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0", + "sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc", + "sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af", + "sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4", + "sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453", + "sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689", + "sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8", + "sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680", + "sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e", + "sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9", + "sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b", + "sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d", + "sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2", + "sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5", + "sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676" ], "index": "pypi", - "version": "==5.9.0" + "version": "==5.9.1" }, "py": { "hashes": [ @@ -347,19 +380,27 @@ }, "pyparsing": { "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pytablewriter": { + "hashes": [ + "sha256:99409d401d6ef5f06d1bc40f265a8e3053afe4cbfbaf709f71124076afb40dbb", + "sha256:c46d1ddc40ef4d084213a86f8626cee33b3aa0119535aa8555da64cb5b65e382" + ], + "index": "pypi", + "version": "==0.64.2" }, "pytest": { "hashes": [ - "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63", - "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea" + "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", + "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" ], "index": "pypi", - "version": "==7.1.1" + "version": "==7.1.2" }, "pytest-cov": { "hashes": [ @@ -369,6 +410,20 @@ "index": "pypi", "version": "==2.10.1" }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, "pyyaml": { "hashes": [ "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", @@ -406,11 +461,11 @@ }, "requests": { "hashes": [ - "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", - "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], "index": "pypi", - "version": "==2.27.1" + "version": "==2.28.1" }, "retry": { "hashes": [ @@ -430,11 +485,11 @@ }, "setuptools": { "hashes": [ - "sha256:6599055eeb23bfef457d5605d33a4d68804266e6cb430b0fb12417c5efeae36c", - "sha256:782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96" + "sha256:273b6847ae61f7829c1affcdd9a32f67aa65233be508f4fbaab866c5faa4e408", + "sha256:d5340d16943a0f67057329db59b564e938bb3736c6e50ae16ea84d5e5d9ba6d0" ], "markers": "python_version >= '3.7'", - "version": "==60.10.0" + "version": "==63.3.0" }, "six": { "hashes": [ @@ -452,6 +507,22 @@ "index": "pypi", "version": "==2.4.0" }, + "tabledata": { + "hashes": [ + "sha256:2016fa561552bbf2266682fe328e9161359e605620084bac4754e91c238880f1", + "sha256:54541b0c9e58f8fa38251ea0a60965dbaf95737027fa80e6ab56f98d7e4d61e9" + ], + "markers": "python_version >= '3.6'", + "version": "==1.3.0" + }, + "tcolorpy": { + "hashes": [ + "sha256:88b1a5c1f4d14fa0a3c9fb97d93f84f0540c7a0e3f3957b1967b1b8ea93dbdbc", + "sha256:8a669c29aada6e6715048ae04116e6b620f0864541fecb8722ede568f403b76e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.1.2" + }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", @@ -465,38 +536,49 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.11'", "version": "==2.0.1" }, "typed-ast": { "hashes": [ - "sha256:0eb77764ea470f14fcbb89d51bc6bbf5e7623446ac4ed06cbd9ca9495b62e36e", - "sha256:1098df9a0592dd4c8c0ccfc2e98931278a6c6c53cb3a3e2cf7e9ee3b06153344", - "sha256:183b183b7771a508395d2cbffd6db67d6ad52958a5fdc99f450d954003900266", - "sha256:18fe320f354d6f9ad3147859b6e16649a0781425268c4dde596093177660e71a", - "sha256:26a432dc219c6b6f38be20a958cbe1abffcc5492821d7e27f08606ef99e0dffd", - "sha256:294a6903a4d087db805a7656989f613371915fc45c8cc0ddc5c5a0a8ad9bea4d", - "sha256:31d8c6b2df19a777bc8826770b872a45a1f30cfefcfd729491baa5237faae837", - "sha256:33b4a19ddc9fc551ebabca9765d54d04600c4a50eda13893dadf67ed81d9a098", - "sha256:42c47c3b43fe3a39ddf8de1d40dbbfca60ac8530a36c9b198ea5b9efac75c09e", - "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27", - "sha256:58ae097a325e9bb7a684572d20eb3e1809802c5c9ec7108e85da1eb6c1a3331b", - "sha256:676d051b1da67a852c0447621fdd11c4e104827417bf216092ec3e286f7da596", - "sha256:74cac86cc586db8dfda0ce65d8bcd2bf17b58668dfcc3652762f3ef0e6677e76", - "sha256:8c08d6625bb258179b6e512f55ad20f9dfef019bbfbe3095247401e053a3ea30", - "sha256:90904d889ab8e81a956f2c0935a523cc4e077c7847a836abee832f868d5c26a4", - "sha256:963a0ccc9a4188524e6e6d39b12c9ca24cc2d45a71cfdd04a26d883c922b4b78", - "sha256:bbebc31bf11762b63bf61aaae232becb41c5bf6b3461b80a4df7e791fabb3aca", - "sha256:bc2542e83ac8399752bc16e0b35e038bdb659ba237f4222616b4e83fb9654985", - "sha256:c29dd9a3a9d259c9fa19d19738d021632d673f6ed9b35a739f48e5f807f264fb", - "sha256:c7407cfcad702f0b6c0e0f3e7ab876cd1d2c13b14ce770e412c0c4b9728a0f88", - "sha256:da0a98d458010bf4fe535f2d1e367a2e2060e105978873c04c04212fb20543f7", - "sha256:df05aa5b241e2e8045f5f4367a9f6187b09c4cdf8578bb219861c4e27c443db5", - "sha256:f290617f74a610849bd8f5514e34ae3d09eafd521dceaa6cf68b3f4414266d4e", - "sha256:f30ddd110634c2d7534b2d4e0e22967e88366b0d356b24de87419cc4410c41b7" - ], - "markers": "python_version < '3.8'", - "version": "==1.5.2" + "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2", + "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1", + "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6", + "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62", + "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac", + "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d", + "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc", + "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2", + "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97", + "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35", + "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6", + "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1", + "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4", + "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c", + "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e", + "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec", + "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f", + "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72", + "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47", + "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72", + "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe", + "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6", + "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3", + "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66" + ], + "index": "pypi", + "version": "==1.5.4" + }, + "typepy": { + "extras": [ + "datetime" + ], + "hashes": [ + "sha256:96788530614083164993d1443959f6c58e6bb8e2da839812ddf462c203e4b84c", + "sha256:cf1913982969cf6348152c4a5feec08e324addd99670999e57cdb3ad87a61e9a" + ], + "markers": "python_version >= '3.6'", + "version": "==1.3.0" }, "types-pyyaml": { "hashes": [ @@ -508,73 +590,72 @@ }, "types-requests": { "hashes": [ - "sha256:5d6f77f3c7565659bdb7b7bce1d33d1abb7d0b056138cac714860e13da2f19df", - "sha256:cf0646031dd6307113b37814f743c04f0707a3357378c2bb1326f848412f5ba9" + "sha256:36385618d4bd2ee3211d4d2e78b44f067ceb5984865c0f253f3c9ecb964526cf", + "sha256:38015d310d13cf7d4d712d2507178349e13fd5dab85259dab7d9a9884c2c9c2a" ], "index": "pypi", - "version": "==2.27.13" + "version": "==2.28.7" }, "types-urllib3": { "hashes": [ - "sha256:24d64e441168851eb05f1d022de18ae31558f5649c8f1117e384c2e85e31315b", - "sha256:bd0abc01e9fb963e4fddd561a56d21cc371b988d1245662195c90379077139cd" + "sha256:1fb6e2af519a7216a19dd6be8cd2ee787b402a754ccb4a13ca1c0e5b202aea5a", + "sha256:6249b6223226cb2012db3b4ff6945c9cb0e12ece9b24f5e29787c4f05028a979" ], - "version": "==1.26.11" + "version": "==1.26.20" }, "typing-extensions": { "hashes": [ - "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", - "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" + "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", + "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], - "markers": "python_version >= '3.6'", - "version": "==4.1.1" + "markers": "python_version >= '3.7'", + "version": "==4.3.0" }, "urllib3": { "hashes": [ - "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", - "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.9" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.11" }, "validators": { "hashes": [ - "sha256:0143dcca8a386498edaf5780cbd5960da1a4c85e0719f3ee5c9b41249c4fefbd", - "sha256:37cd9a9213278538ad09b5b9f9134266e7c226ab1fede1d500e29e0a8fbb9ea6" + "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a" ], "index": "pypi", - "version": "==0.18.2" + "version": "==0.20.0" }, "virtualenv": { "hashes": [ - "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021", - "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134" + "sha256:0ef5be6d07181946891f5abc8047fda8bc2f0b4b9bf222c64e6e8963baee76db", + "sha256:635b272a8e2f77cb051946f46c60a54ace3cb5e25568228bd6b57fc70eca9ff3" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.3" + "markers": "python_version >= '3.6'", + "version": "==20.16.2" }, "yamlfix": { "hashes": [ - "sha256:ddcc14d6330c211d92a93d5662d3ef1cde61ab9b27d012f87f716a285425ccc8", - "sha256:ddf097c99fd572ccf6fb444227c0575308d16531cba7d231a689ccd6005e18a0" + "sha256:2d6752010d8c087cc0142840f77d3f7aeb092762d79514d0433e9e60b3c06569", + "sha256:a7e34d1bbd4d29a2b715c7b8b9ba62a766862f3b42489062d520569dd2289e48" ], "index": "pypi", - "version": "==0.9.0" + "version": "==1.0.1" }, "yamllint": { "hashes": [ - "sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e" + "sha256:e688324b58560ab68a1a3cff2c0a474e3fed371dfe8da5d1b9817b7df55039ce" ], "index": "pypi", - "version": "==1.26.3" + "version": "==1.27.1" }, "zipp": { "hashes": [ - "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", - "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" + "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2", + "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009" ], "markers": "python_version >= '3.7'", - "version": "==3.7.0" + "version": "==3.8.1" } }, "develop": {} diff --git a/README.md b/README.md index 647194b70f..b5a34e67c6 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - [CI/CD Environment](#cicd-environment) - [Latest Distribution Url](#latest-distribution-url) - [Testing the Distribution](#testing-the-distribution) + - [Components Release Notes Check](#components-release-notes-check) - [Signing Artifacts](#signing-artifacts) - [Making a Release](#making-a-release) - [Releasing for Linux](#releasing-for-linux) @@ -123,14 +124,61 @@ Tests the OpenSearch distribution, including integration, backwards-compatibilit See [src/test_workflow](./src/test_workflow) for more information. +#### Components Release Notes Check + +Pulls the latest CommitID based on user passed environment value of `GIT_LOG_DATE` and checks if the release notes exists or not. Outputs a formated markdown table and adds a comment to the github issue using user passed `GIT_ISSUE_NUMBER`, set `ADD_COMMENT` to false to just output the table. +``` +# OpenSearch CommitID(after 2022-07-26) & Release Notes info +| Repo |Branch|CommitID|Release Notes| +|-------------|------|--------|-------------| +|OpenSearch | 2.2|4035bf7 |YES | +|common-utils |2.x |7d53102 |NO | +|job-scheduler| 2.2|a501307 |YES | +|security |main |f7b6fe5 |YES | +|geospatial | 2.2|a71475a |YES | +|k-NN |2.x |53185a0 |YES | +``` + +The workflow uses the following environment values: +* `GIT_LOG_DATE`: To check if commit exists after a specific date (in format yyyy-mm-dd, example 2022-07-26). +* `ADD_COMMENT`: true/false (Default is true). +* `GIT_ISSUE_NUMBER`: Build repo GitHub issue number to add the generated table as comment. +* `GITHUB_TOKEN`: Token used to add the comment on the github issue. + +To run: +```bash +./run_releasenotes_check.sh manifests/2.2.0/opensearch-2.2.0.yml +``` #### Signing Artifacts -The signing step takes the manifest file created from the build step and signs all its component artifacts using a tool called `opensearch-signer-client` (in progress of being open-sourced). The input requires a path to the build manifest and is expected to be inside the artifacts directory with the same directories mentioned in the build step. +For all types of signing within OpenSearch project we use `opensearch-signer-client` (in progress of being open-sourced) which is a wrapper around internal signing system and is only available for authenticated users. The input requires a path to the build manifest or directory containing all the artifacts or a single artifact. +Usage: ```bash ./sign.sh builds/opensearch/manifest.yml ``` +The tool currently supports following platforms for signing. +#### PGP + +Anything can be signed using PGP signing eg: tarball, any type of file, etc. A .sig file will be returned containing the signature. OpenSearch and OpenSearch dashboards distributions, components such as data prepper, etc as well as maven artifacts are signed using PGP signing. See [this page](https://opensearch.org/verify-signatures.html) for how to verify signatures. + + +#### WINDOWS + +Windows signing can be used to sign windows executables such as .msi, .msp, .msm, .cab, .dll, .exe, .appx, .appxbundle, .msix, .msixbundle, .sys, .vxd, .ps1, .psm1, and any PE file that is supported by [Signtool.exe](https://docs.microsoft.com/en-us/dotnet/framework/tools/signtool-exe). Various windows artifacts such as SQL OBDC, opensearch-cli, etc are signed using this method. +Windows code signing uses EV (Extended Validated) code signing certificates. + +| Types of signing/Details | Digest | Cipher | Key Size| +| ------------- |:-------------| :-----| :-----| +| PGP | SHA1 | AES-128 | 2048 | +| Windows | SHA256 | RSA | | +| RPM | SHA512 | RSA | 4096 | + + +### Signing RPM artifacts +RPM artifacts are signed via shell script which uses a [macros template](https://github.com/opensearch-project/opensearch-build/blob/main/scripts/pkg/sign_templates/rpmmacros). More details in this [commit](https://github.com/opensearch-project/opensearch-build/commit/950d55c1ed3f82e98120541fa40ff506338c1059). See [issue](https://github.com/opensearch-project/opensearch-build/issues/1547) to add RPM artifact signing functionality to the above signing system. Currently we are only signing OpenSearch and OpenSearch dashboards RPM distributions using this method. + See [src/sign_workflow](./src/sign_workflow) for more information. ### Making a Release diff --git a/build.gradle b/build.gradle index 866c181b04..992f59583c 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ repositories { } dependencies { - compile group: 'junit', name: 'junit', version: '4.13.2' + testImplementation group: 'junit', name: 'junit', version: '4.13.2' implementation group: 'org.codehaus.groovy', name: 'groovy-all', version: '3.0.9', ext: 'pom' implementation group: 'com.cloudbees', name: 'groovy-cps', version: '1.31' testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.29' @@ -69,19 +69,19 @@ sourceSets { } sharedLibrary { - coreVersion = '2.355' // https://mvnrepository.com/artifact/org.jenkins-ci.main/jenkins-core?repo=jenkins-releases + coreVersion = '2.358' // https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-core/ testHarnessVersion = '1736.vc72c458c5103' // https://mvnrepository.com/artifact/org.jenkins-ci.main/jenkins-test-harness?repo=jenkins-releases pluginDependencies { - workflowCpsGlobalLibraryPluginVersion = '2.21.3' // https://mvnrepository.com/artifact/org.jenkins-ci.plugins.workflow/workflow-cps-global-lib?repo=jenkins-releases + workflowCpsGlobalLibraryPluginVersion = '570.v21311f4951f8' // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/workflow/workflow-cps-global-lib/ // see https://mvnrepository.com/artifact/org.jenkins-ci.plugins/?repo=jenkins-releases for latest dependency('org.jenkins-ci.plugins.workflow', 'workflow-cps', '2.94.1') dependency('org.jenkins-ci.plugins.workflow', 'workflow-multibranch', '2.26.1') - dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '2.12') + dependency('org.jenkins-ci.plugins', 'pipeline-input-step', '449.v77f0e8b_845c4') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/pipeline-input-step/ dependency('org.jenkins-ci.plugins', 'script-security', '1172.v35f6a_0b_8207e') dependency('org.jenkins-ci.plugins', 'credentials', '1112.vc87b_7a_3597f6') dependency('org.jenkins-ci.plugins', 'git-client', '3.10.1') dependency('org.jenkins-ci.plugins', 'junit', '1.55') - dependency('org.jenkins-ci.plugins', 'mailer', '1.34.2') + dependency('org.jenkins-ci.plugins', 'mailer', '408.vd726a_1130320') // https://repo.jenkins-ci.org/public/org/jenkins-ci/plugins/mailer/ } } diff --git a/docker/ci/config/jdk-setup.sh b/docker/ci/config/jdk-setup.sh index ddc8238797..20aa3297f0 100755 --- a/docker/ci/config/jdk-setup.sh +++ b/docker/ci/config/jdk-setup.sh @@ -19,15 +19,15 @@ JDKS="" case "${ARCH}" in aarch64|arm64) # Use "@" format to collect all JDK platform specific distributions - JDKS+="f287cdc2a688c2df247ea0d8bfe2863645b73848e4e5c35b02a8a3d2d6b69551@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u302-b08/OpenJDK8U-jdk_aarch64_linux_hotspot_8u302b08.tar.gz " - JDKS+="0ba188a2a739733163cd0049344429d2284867e04ca452879be24f3b54320c9a@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.14%2B9/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.14_9.tar.gz " - JDKS+="302caf29f73481b2b914ba2b89705036010c65eb9bc8d7712b27d6e9bedf6200@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.2%2B8/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.2_8.tar.gz " + JDKS+="d10efb2afad3ed3d7bac9d3249cea77928aca6acb973cac0f90a2dd3606a3533@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u332-b09/OpenJDK8U-jdk_aarch64_linux_hotspot_8u332b09.tar.gz " + JDKS+="999fbd90b070f9896142f0eb28354abbeb367cbe49fd86885c626e2999189e0a@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.15_10.tar.gz " + JDKS+="2e3c19c1707205c6b90cc04b416e8d83078ed98417d5a69dce3cf7dc0d7cfbca@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.3%2B7/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.3_7.tar.gz " ;; amd64|x86_64) # Use "@" format to collect all JDK platform specific distributions - JDKS+="cc13f274becf9dd5517b6be583632819dfd4dd81e524b5c1b4f406bdaf0e063a@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u302-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u302b08.tar.gz " - JDKS+="1189bee178d11402a690edf3fbba0c9f2ada1d3a36ff78929d81935842ef24a9@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.14%2B9/OpenJDK11U-jdk_x64_linux_hotspot_11.0.14_9.tar.gz " - JDKS+="288f34e3ba8a4838605636485d0365ce23e57d5f2f68997ac4c2e4c01967cd48@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.2%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.2_8.tar.gz " + JDKS+="adc13a0a0540d77f0a3481b48f10d61eb203e5ad4914507d489c2de3bd3d83da@https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u332-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u332b09.tar.gz " + JDKS+="5fdb4d5a1662f0cca73fec30f99e67662350b1fa61460fa72e91eb9f66b54d0b@https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.15_10.tar.gz " + JDKS+="81f5bed21077f9fbb04909b50391620c78b9a3c376593c0992934719c0de6b73@https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.3%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.3_7.tar.gz " ;; *) echo "Unsupported arch: ${ARCH}" diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile index 490e902886..c4cefd03f0 100644 --- a/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.centos7.opensearch-dashboards.x64.arm64.dockerfile @@ -57,9 +57,9 @@ SHELL ["/bin/bash", "-lc"] CMD ["/bin/bash", "-l"] # Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.4.0 && rvm --default use 2.4.0 && yum install -y rpm-build createrepo && yum clean all +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all -ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.4.0/bin +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin ENV GEM_HOME=/usr/share/opensearch/.gem ENV GEM_PATH=$GEM_HOME diff --git a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile index 2d7258b3a2..30e56dc62e 100644 --- a/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.centos7.opensearch.x64.arm64.dockerfile @@ -69,9 +69,9 @@ SHELL ["/bin/bash", "-lc"] CMD ["/bin/bash", "-l"] # Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.4.0 && rvm --default use 2.4.0 && yum install -y rpm-build createrepo && yum clean all +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all -ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.4.0/bin +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin ENV GEM_HOME=/usr/share/opensearch/.gem ENV GEM_PATH=$GEM_HOME diff --git a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile index 295a3117ef..71413bd17b 100644 --- a/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile +++ b/docker/ci/dockerfiles/current/build.rockylinux8.opensearch.x64.arm64.dockerfile @@ -69,9 +69,9 @@ SHELL ["/bin/bash", "-lc"] CMD ["/bin/bash", "-l"] # Install ruby / rpm / fpm related dependencies -RUN . /etc/profile.d/rvm.sh && rvm install 2.4.0 && rvm --default use 2.4.0 && dnf install -y rpm-build rpm-sign createrepo pinentry && dnf clean all +RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build rpm-sign createrepo pinentry && dnf clean all -ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.4.0/bin +ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin ENV RVM_HOME=/usr/local/rvm/bin ENV GEM_HOME=/usr/share/opensearch/.gem ENV GEM_PATH=$GEM_HOME diff --git a/docker/ci/dockerfiles/current/yq.alpine3.x64.arm64.dockerfile b/docker/ci/dockerfiles/current/yq.alpine3.x64.arm64.dockerfile new file mode 100644 index 0000000000..5994d2b821 --- /dev/null +++ b/docker/ci/dockerfiles/current/yq.alpine3.x64.arm64.dockerfile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This docker image is based on yq docker image, but with additional pkgs for specific uses. +# We didnt add yq to other build image because: +# 1. We want to keep the image as slim as possible for specific use case. +# 2. It is not easy to install yq as it is not part of default package manager repositories on RHEL/Ubuntu +# 3. Alpine can install yq with apk from the get go, and does not have the official yq image entrypoint +# which will cause jenkins to not able to `cat` and hold the container. +# Thanks. + +FROM alpine:3 + +USER 0 + +# Install pkgs +RUN apk update && apk upgrade && apk add git yq bash + +# User opensearch +RUN set -eux; \ + addgroup -g 1000 opensearch; \ + adduser -u 1000 -G opensearch -s /bin/bash -h /home/opensearch -D opensearch + +USER 1000 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c023..41d9927a4d 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d89a20d212..aa991fceae 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,18 +1,5 @@ - -# -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. -# - distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=11657af6356b7587bfb37287b5992e94a9686d5c8a0a1b60b87b9928a2decde5 \ No newline at end of file diff --git a/gradlew b/gradlew index 3a163de711..1b6c787337 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' -exec "$JAVACMD" "$@" \ No newline at end of file +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index 28690fe08a..ac1b06f938 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -86,4 +86,4 @@ exit /b 1 :mainEnd if "%OS%"=="Windows_NT" endlocal -:omega \ No newline at end of file +:omega diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index 386e3d1ba1..66edc4e6e1 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -7,19 +7,16 @@ pipeline { } agent none environment { - AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=2.2.0/opensearch-2.2.0.yml;TARGET_JOB_NAME=distribution-build-opensearch - H/10 * * * * %INPUT_MANIFEST=2.1.0/opensearch-2.1.0.yml;TARGET_JOB_NAME=distribution-build-opensearch - H/10 * * * * %INPUT_MANIFEST=2.1.0/opensearch-dashboards-2.1.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards + H/60 * * * * %INPUT_MANIFEST=2.2.0/opensearch-2.2.0.yml;TEST_MANIFEST=2.2.0/opensearch-2.2.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch + H/60 * * * * %INPUT_MANIFEST=2.2.0/opensearch-dashboards-2.2.0.yml;TEST_MANIFEST=2.2.0/opensearch-dashboards-2.2.0-test.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards + H 1 * * * %INPUT_MANIFEST=1.3.5/opensearch-1.3.5.yml;TARGET_JOB_NAME=distribution-build-opensearch + H 1 * * * %INPUT_MANIFEST=1.3.5/opensearch-dashboards-1.3.5.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H 1 * * * %INPUT_MANIFEST=2.0.2/opensearch-dashboards-2.0.2.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H 1 * * * %INPUT_MANIFEST=2.0.2/opensearch-2.0.2.yml;TARGET_JOB_NAME=distribution-build-opensearch - H 1 * * * %INPUT_MANIFEST=1.3.4/opensearch-dashboards-1.3.4.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards - H 1 * * * %INPUT_MANIFEST=1.3.4/opensearch-1.3.4.yml;TARGET_JOB_NAME=distribution-build-opensearch ''' } parameters { @@ -105,7 +102,7 @@ pipeline { publishNotification( icon: ':warning:', message: 'Failed checking for build to trigger', - credentialsId: 'BUILD_NOTICE_WEBHOOK', + credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}", target_job_name: "${TARGET_JOB_NAME}" ) diff --git a/jenkins/cross-cluster-replication/perf-test.jenkinsfile b/jenkins/cross-cluster-replication/perf-test.jenkinsfile index 7a7b1d5ebd..7821d9580a 100644 --- a/jenkins/cross-cluster-replication/perf-test.jenkinsfile +++ b/jenkins/cross-cluster-replication/perf-test.jenkinsfile @@ -6,7 +6,7 @@ pipeline { timeout(time: 10, unit: 'HOURS') } environment { - AGENT_LABEL = 'Jenkins-Agent-al2-x64-m52xlarge-Docker-Host-Perf-Test' + AGENT_LABEL = 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v1' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'ccr-perf-test' @@ -116,7 +116,7 @@ pipeline { icon: ':white_check_mark:', message: 'CCR Performance Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', ) postCleanup() } @@ -130,7 +130,7 @@ pipeline { icon: ':warning:', message: 'Failed CCR Performance Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', ) postCleanup() } diff --git a/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile b/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile index 5db941a297..a41cdc6e97 100644 --- a/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile +++ b/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile @@ -9,11 +9,18 @@ pipeline { string(name: 'VERSION', description: 'The version of Data Prepper', trim: true) string(name: 'DATA_PREPPER_BUILD_NUMBER', description: 'The build number of the Data Prepper build from GitHub to release.', trim: true) } + environment { + DATA_PREPPER_ARTIFACT_STAGING_SITE = credentials('jenkins-data-prepper-artifact-staging-site') + DATA_PREPPER_STAGING_CONTAINER_REPOSITORY = credentials('jenkins-data-prepper-staging-container-repository') + ARTIFACT_PROMOTION_ROLE_NAME = credentials('jenkins-artifact-promotion-role') + AWS_ACCOUNT_ARTIFACT = credentials('jenkins-aws-production-account') + ARTIFACT_PRODUCTION_BUCKET_NAME = credentials('jenkins-artifact-production-bucket-name') + } stages { stage('Promote Archives') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2' args '-e JAVA_HOME=/opt/java/openjdk-11' alwaysPull true @@ -65,7 +72,7 @@ pipeline { stage('Promote Docker') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'alpine:3' alwaysPull true } @@ -111,7 +118,7 @@ pipeline { stage('Promote Maven') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2' args '-e JAVA_HOME=/opt/java/openjdk-11' alwaysPull true @@ -151,7 +158,7 @@ pipeline { steps { script { // stage artifacts for release with Sonatype - withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + withCredentials([usernamePassword(credentialsId: 'jenkins-sonatype-creds', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { sh("$WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/maven") } } diff --git a/jenkins/docker/docker-build.jenkinsfile b/jenkins/docker/docker-build.jenkinsfile index 37f288025b..6f3b1b5fe7 100644 --- a/jenkins/docker/docker-build.jenkinsfile +++ b/jenkins/docker/docker-build.jenkinsfile @@ -27,7 +27,7 @@ pipeline { stage('docker-build') { agent { docker { - label 'Jenkins-Agent-Ubuntu2004-X64-m52xlarge-Docker-Builder' + label 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder' image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk14' args '-u root -v /var/run/docker.sock:/var/run/docker.sock' alwaysPull true diff --git a/jenkins/docker/docker-copy.jenkinsfile b/jenkins/docker/docker-copy.jenkinsfile index 5df1b64059..ea4c373f1e 100644 --- a/jenkins/docker/docker-copy.jenkinsfile +++ b/jenkins/docker/docker-copy.jenkinsfile @@ -8,7 +8,7 @@ pipeline { parameters { choice( name: 'SOURCE_IMAGE_REGISTRY', - choices: ['opensearchstaging', 'public.ecr.aws/opensearchstaging', 'opensearchproject', 'public.ecr.aws/opensearchproject', "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}"], + choices: ['opensearchstaging', 'public.ecr.aws/opensearchstaging', 'opensearchproject', 'public.ecr.aws/opensearchproject', 'public.ecr.aws/t2m2d0w1'], description: 'Choose the source image registry' ) string( @@ -31,7 +31,7 @@ pipeline { stage("Image Copy") { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2' args '-u root -v /var/run/docker.sock:/var/run/docker.sock' } diff --git a/jenkins/gradle/gradle-check.jenkinsfile b/jenkins/gradle/gradle-check.jenkinsfile index e33a84fc0c..cb39414516 100644 --- a/jenkins/gradle/gradle-check.jenkinsfile +++ b/jenkins/gradle/gradle-check.jenkinsfile @@ -2,14 +2,25 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') + throttleJobProperty( + categories: [], + limitOneJobWithMatchingParams: false, + maxConcurrentPerNode: 0, + maxConcurrentTotal: 15, + paramsToUseForLimit: '', + throttleEnabled: true, + throttleOption: 'project', + ) } // gradle check have a lot of issues running on containers // Therefore, we directly run it on the agent node agent { node { // Must use Ubuntu agent with 1 executor or gradle check will show a lot of java-related errors - label 'Jenkins-Agent-Ubuntu2004-X64-c518xlarge-Single-Host' + // The c524xlarge is the instance type that has the least amount of errors during gradle check + // https://github.com/opensearch-project/OpenSearch/issues/1975 + label 'Jenkins-Agent-Ubuntu2004-X64-C524xlarge-Single-Host' } } parameters { @@ -42,49 +53,63 @@ pipeline { ) } environment { - JAVA11_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-11" - JAVA17_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-17" - JAVA8_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-8" - JAVA_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-17" + JAVA8_HOME="/var/jenkins/tools/hudson.model.JDK/openjdk-8" + JAVA11_HOME="/var/jenkins/tools/hudson.model.JDK/openjdk-11" + JAVA14_HOME="/var/jenkins/tools/hudson.model.JDK/openjdk-14" + JAVA17_HOME="/var/jenkins/tools/hudson.model.JDK/openjdk-17" USER_BUILD_CAUSE = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause') TIMER_BUILD_CAUSE = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause') } stages { - stage('Install jdk-8') { + stage('Install openjdk-8') { tools { - jdk 'jdk-8' + jdk 'openjdk-8' + } + steps { + echo "Install openjdk-8" + sh("ls ${JAVA8_HOME}/.. | grep openjdk-8") } + } + stage('Install openjdk-11') { + tools { + jdk 'openjdk-11' + } steps { - echo "Install jdk-8" - sh("ls ${JAVA8_HOME}/.. | grep jdk-8") + echo "Install openjdk-11" + sh("ls ${JAVA11_HOME}/.. | grep openjdk-11") } } - stage('Install jdk-11') { + stage('Install openjdk-14') { tools { - jdk 'jdk-11' + jdk 'openjdk-14' } steps { - echo "Install jdk-11" - sh("ls ${JAVA11_HOME}/.. | grep jdk-11") + echo "Install openjdk-14" + sh("ls ${JAVA14_HOME}/.. | grep openjdk-14") } } - stage('Install jdk-17') { + stage('Install openjdk-17') { tools { - jdk 'jdk-17' + jdk 'openjdk-17' } steps { - echo "Install jdk-17" - sh("ls ${JAVA17_HOME}/.. | grep jdk-17") + echo "Install openjdk-17" + sh("ls ${JAVA17_HOME}/.. | grep openjdk-17") } } stage('Run Gradle Check') { steps { script { - sh("ls /var/jenkins/tools/hudson.model.JDK/ && env | grep JAVA") + sh("ls /var/jenkins/tools/hudson.model.JDK/") if (env.USER_BUILD_CAUSE.equals('[]') && env.TIMER_BUILD_CAUSE.equals('[]')) { def pr_url = "${pr_to_clone_url}".replace(".git", "/pull/${pr_number}") - println("Triggered by PR: ${pr_url}") - currentBuild.description = """PR #${pr_number}: ${pr_title}""" + println("Triggered by GitHub: ${pr_to_clone_url}") + if ("$pr_number" == "Null") { + currentBuild.description = """Others: ${pr_title}""" + } + else { + currentBuild.description = """PR #${pr_number}: ${pr_title}""" + } runGradleCheck( gitRepoUrl: "${pr_from_clone_url}", @@ -105,6 +130,9 @@ pipeline { } post() { always { + sh ("cp -v `find search/build/reports/jacoco/ -name '*.xml' | head -n 1` codeCoverage.xml || echo") + junit allowEmptyResults: true, testResults: '**/build/test-results/**/*.xml' + archiveArtifacts artifacts: 'codeCoverage.xml', onlyIfSuccessful: true script { sh("rm -rf *") postCleanup() diff --git a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile index dc407267c2..66583bf30c 100644 --- a/jenkins/opensearch-dashboards/bwc-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/bwc-test.jenkinsfile @@ -22,7 +22,7 @@ pipeline { ) string( name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', trim: true ) } @@ -40,9 +40,9 @@ pipeline { currentBuild.result = 'ABORTED' error("BWC Tests failed to start. Missing parameter: AGENT_LABEL.") } - if (!fileExists("manifests/${TEST_MANIFEST}")) { + if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) { currentBuild.result = 'ABORTED' - error("BWC Tests failed to start. Test manifest not found in manifests/${TEST_MANIFEST}.") + error("BWC Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.") } /* Rebuilding of this job will result in considering the upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug @@ -57,7 +57,7 @@ pipeline { stage('detect docker image + args') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' alwaysPull true } @@ -120,7 +120,7 @@ pipeline { icon: ':white_check_mark:', message: 'BWC Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) @@ -136,7 +136,7 @@ pipeline { icon: ':warning:', message: 'Failed BWC Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index 343017fb99..6635c871e2 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -6,8 +6,8 @@ pipeline { } agent none environment { - AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ } parameters { @@ -65,6 +65,30 @@ pipeline { } } } + stage('Get Repository Commits') { + agent { + docker { + label AGENT_X64 + image 'opensearchstaging/ci-runner:ci-runner-alpine3-yq-v1' + alwaysPull true + } + } + steps { + script { + getRepositoryCommit( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + outputFile: "commits.yml" + ) + } + } + post { + always { + archiveArtifacts artifacts: 'commits.yml' + postCleanup() + } + } + } stage('build') { parallel { stage('build-and-test-linux-x64-tar') { @@ -518,7 +542,7 @@ pipeline { icon: ':white_check_mark:', message: 'Successful Build', extra: stashed, - credentialsId: 'BUILD_NOTICE_WEBHOOK', + credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } @@ -534,7 +558,7 @@ pipeline { publishNotification( icon: ':warning:', message: 'Failed Build', - credentialsId: 'BUILD_NOTICE_WEBHOOK', + credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } diff --git a/jenkins/opensearch-dashboards/integ-test.jenkinsfile b/jenkins/opensearch-dashboards/integ-test.jenkinsfile index 7924430b98..d09c4461c0 100644 --- a/jenkins/opensearch-dashboards/integ-test.jenkinsfile +++ b/jenkins/opensearch-dashboards/integ-test.jenkinsfile @@ -22,7 +22,7 @@ pipeline { ) string( name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-al2-x64-c54xlarge-Docker-Host,for arm64 use Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host', + description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) } @@ -36,9 +36,9 @@ pipeline { currentBuild.result = 'ABORTED' error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.") } - if (!fileExists("manifests/${TEST_MANIFEST}")) { + if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) { currentBuild.result = 'ABORTED' - error("Integration Tests failed to start. Test manifest not found in manifests/${TEST_MANIFEST}.") + error("Integration Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.") } /* Rebuilding of this job will result in considering upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug @@ -115,7 +115,7 @@ pipeline { icon: ':white_check_mark:', message: 'Integration Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) @@ -131,7 +131,7 @@ pipeline { icon: ':warning:', message: 'Failed Integration Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) diff --git a/jenkins/opensearch-maven-release/maven-sign-release.jenkinsfile b/jenkins/opensearch-maven-release/maven-sign-release.jenkinsfile index cc189e65ca..b709d1a17a 100644 --- a/jenkins/opensearch-maven-release/maven-sign-release.jenkinsfile +++ b/jenkins/opensearch-maven-release/maven-sign-release.jenkinsfile @@ -3,7 +3,7 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130' alwaysPull true } @@ -22,6 +22,7 @@ pipeline { } environment { ARTIFACT_PATH = "distribution-build-opensearch/${VERSION}/${BUILD_ID}/linux/x64/tar/builds" + ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') } stages { stage('sign') { @@ -44,9 +45,6 @@ pipeline { } } stage('stage maven artifacts') { - tools { - maven "maven-3.8.2" - } environment { REPO_URL = "https://aws.oss.sonatype.org/" STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}" @@ -55,7 +53,7 @@ pipeline { steps { script { // stage artifacts for release with Sonatype - withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + withCredentials([usernamePassword(credentialsId: 'jenkins-sonatype-creds', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { sh('$WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/opensearch/maven') } } diff --git a/jenkins/opensearch-ruby/Jenkinsfile b/jenkins/opensearch-ruby/opensearch-ruby.jenkinsfile similarity index 81% rename from jenkins/opensearch-ruby/Jenkinsfile rename to jenkins/opensearch-ruby/opensearch-ruby.jenkinsfile index b927bc4831..7b5b4f0b30 100644 --- a/jenkins/opensearch-ruby/Jenkinsfile +++ b/jenkins/opensearch-ruby/opensearch-ruby.jenkinsfile @@ -12,12 +12,15 @@ pipeline { trim: true ) } + environment { + ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') + } stages { stage('ruby-build-sign-upload') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2' alwaysPull true } } @@ -28,7 +31,7 @@ pipeline { error('RELEASE_BRANCH parameter is not set') } git url: 'https://github.com/opensearch-project/opensearch-ruby.git', branch: "$RELEASE_BRANCH" - withCredentials([string(credentialsId: 'opensearchproject-rubygems-private-key', variable: 'RUBYGEMS_PRIVATE_KEY')]) { + withCredentials([string(credentialsId: 'jenkins-opensearchproject-rubygems-private-key', variable: 'RUBYGEMS_PRIVATE_KEY')]) { writeFile file: 'gem-private_key.pem', text: RUBYGEMS_PRIVATE_KEY } sh''' @@ -50,7 +53,7 @@ pipeline { } post { always { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { postCleanup() } diff --git a/jenkins/opensearch/bwc-test.jenkinsfile b/jenkins/opensearch/bwc-test.jenkinsfile index 4bd0d2c4bf..105dc0ec4f 100644 --- a/jenkins/opensearch/bwc-test.jenkinsfile +++ b/jenkins/opensearch/bwc-test.jenkinsfile @@ -9,9 +9,6 @@ pipeline { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" } - tools { - maven "maven-3.8.2" - } parameters { string( name: 'TEST_MANIFEST', @@ -25,7 +22,7 @@ pipeline { ) string( name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host.', trim: true ) } @@ -43,9 +40,9 @@ pipeline { currentBuild.result = 'ABORTED' error("BWC Tests failed to start. Missing parameter: AGENT_LABEL.") } - if (!fileExists("manifests/${TEST_MANIFEST}")) { + if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) { currentBuild.result = 'ABORTED' - error("BWC Tests failed to start. Test manifest not found in manifests/${TEST_MANIFEST}.") + error("BWC Tests failed to start. Test manifest not was provided or not found in manifests/${TEST_MANIFEST}.") } /* Rebuilding of this job will result in considering upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug @@ -60,7 +57,7 @@ pipeline { stage('detect docker image + args') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' alwaysPull true } @@ -123,7 +120,7 @@ pipeline { icon: ':white_check_mark:', message: 'BWC Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) @@ -139,7 +136,7 @@ pipeline { icon: ':warning:', message: 'Failed BWC Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 546579b48e..742f5211cc 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -6,8 +6,8 @@ pipeline { } agent none environment { - AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + AGENT_ARM64 = 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' IMAGE_RPM = 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' // required for rpm to create digest sha256 correctly with rpm 4.12+ } parameters { @@ -65,6 +65,30 @@ pipeline { } } } + stage('Get Repository Commits') { + agent { + docker { + label AGENT_X64 + image 'opensearchstaging/ci-runner:ci-runner-alpine3-yq-v1' + alwaysPull true + } + } + steps { + script { + getRepositoryCommit( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + outputFile: "commits.yml" + ) + } + } + post { + always { + archiveArtifacts artifacts: 'commits.yml' + postCleanup() + } + } + } stage('build') { parallel { stage('build-snapshot-x64-tar') { @@ -91,7 +115,7 @@ pipeline { String mavenPath = "$WORKSPACE/tar/builds/opensearch/maven" if (fileExists(mavenPath)) { - withCredentials([usernamePassword(credentialsId: 'Sonatype', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { + withCredentials([usernamePassword(credentialsId: 'jenkins-sonatype-creds', usernameVariable: 'SONATYPE_USERNAME', passwordVariable: 'SONATYPE_PASSWORD')]) { sh("$WORKSPACE/publish/publish-snapshot.sh ${mavenPath}") } } else { @@ -140,6 +164,36 @@ pipeline { } } } + stage('build-snapshot-macos-x64-tar') { + agent { + node { + label 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host' + } + } + tools { + jdk dockerAgent.javaVersion + } + steps { + script { + buildManifest( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + distribution: 'tar', + snapshot: true + ) + echo("Uploading darwin min snapshots to S3") + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()], + distribution: 'tar' + ) + } + } + post { + always { + postCleanup() + } + } + } stage('build-and-test-x64-tar') { agent { docker { @@ -555,7 +609,7 @@ pipeline { icon: ':white_check_mark:', message: 'Successful Build', extra: stashed, - credentialsId: 'BUILD_NOTICE_WEBHOOK', + credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } @@ -571,7 +625,7 @@ pipeline { publishNotification( icon: ':warning:', message: buildFailureMessage(), - credentialsId: 'BUILD_NOTICE_WEBHOOK', + credentialsId: 'jenkins-build-notice-webhook', manifest: "${INPUT_MANIFEST}" ) } diff --git a/jenkins/opensearch/integ-test.jenkinsfile b/jenkins/opensearch/integ-test.jenkinsfile index 986cd7b8e1..d3d1ac713f 100644 --- a/jenkins/opensearch/integ-test.jenkinsfile +++ b/jenkins/opensearch/integ-test.jenkinsfile @@ -9,9 +9,6 @@ pipeline { BUILD_MANIFEST = "build-manifest.yml" DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch" } - tools { - maven "maven-3.8.2" - } parameters { string( name: 'TEST_MANIFEST', @@ -25,7 +22,7 @@ pipeline { ) string( name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-al2-x64-c54xlarge-Docker-Host,for arm64 use Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host', + description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) } @@ -39,9 +36,9 @@ pipeline { currentBuild.result = 'ABORTED' error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.") } - if (!fileExists("manifests/${TEST_MANIFEST}")) { + if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) { currentBuild.result = 'ABORTED' - error("Integration Tests failed to start. Test manifest not found in manifests/${TEST_MANIFEST}.") + error("Integration Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.") } /* Rebuilding of this job will result in considering upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug @@ -152,7 +149,7 @@ pipeline { icon: ':white_check_mark:', message: 'Integration Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) @@ -168,7 +165,7 @@ pipeline { icon: ':warning:', message: 'Failed Integration Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', manifest: TEST_MANIFEST, ) diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 0ca6706e8e..d35d5dcc6c 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -6,11 +6,23 @@ pipeline { timeout(time: 15, unit: 'HOURS') } environment { - AGENT_LABEL = 'Jenkins-Agent-al2-x64-m52xlarge-Docker-Host-Perf-Test' + AGENT_LABEL = 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-performance-test-v1' BUNDLE_MANIFEST = 'bundle-manifest.yml' JOB_NAME = 'perf-test' } + triggers { + parameterizedCron ''' + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + ''' + } parameters { string( name: 'BUNDLE_MANIFEST_URL', @@ -194,7 +206,7 @@ pipeline { icon: ':white_check_mark:', message: 'Performance Tests Successful', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', ) postCleanup() } @@ -208,7 +220,7 @@ pipeline { icon: ':warning:', message: 'Failed Performance Tests', extra: stashed, - credentialsId: 'INTEG_TEST_WEBHOOK', + credentialsId: 'jenkins-integ-test-webhook', ) postCleanup() } diff --git a/jenkins/promotion/promote-artifacts.jenkinsfile b/jenkins/promotion/promote-artifacts.jenkinsfile index 85626eed8c..e2bd3fd10b 100644 --- a/jenkins/promotion/promote-artifacts.jenkinsfile +++ b/jenkins/promotion/promote-artifacts.jenkinsfile @@ -38,7 +38,7 @@ pipeline { stage('promote artifacts') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' alwaysPull true } diff --git a/jenkins/promotion/promote-docker-ecr.jenkinsfile b/jenkins/promotion/promote-docker-ecr.jenkinsfile new file mode 100644 index 0000000000..e3463f0711 --- /dev/null +++ b/jenkins/promotion/promote-docker-ecr.jenkinsfile @@ -0,0 +1,86 @@ +lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) + +pipeline { + options { + timeout(time: 1, unit: 'HOURS') + } + agent { + docker { + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2' + args '-u root -v /var/run/docker.sock:/var/run/docker.sock' + } +} + parameters { + string( + name: 'SOURCE_IMAGES', + description: 'Comma separated list of product with its image tag that we want to promote from staging to production. E.g.: opensearch:2.0.1.3910, opensearch-dashboards:2.0.1, data-prepper:1.5.0-19.', + trim: true + ) + string( + name: 'RELEASE_VERSION', + description: 'Official release version on production repository. This is uniform for all source images. E.g.: 2.0.1, 1.5.0.', + trim: true + ) + booleanParam( + name: 'DOCKER_HUB_PROMOTE', + defaultValue: true, + description: 'Promote SOURCE_IMAGES to `opensearchproject` production Docker Hub.' + ) + booleanParam( + name: 'ECR_PROMOTE', + defaultValue: true, + description: 'Promote SOURCE_IMAGES to `gallery.ecr.aws/opensearchproject` production ECR repository.' + ) + booleanParam( + name: 'TAG_LATEST', + defaultValue: true, + description: 'Tag the copied image as latest' + ) + booleanParam( + name: 'TAG_MAJOR_VERSION', + defaultValue: true, + description: 'Tag the copied image with its major version. E.g.: 1.3.2 image will be tagged with 1 in the hub.' + ) + } + + stages { + stage('Parameters Check') { + steps { + script { + currentBuild.description = "Promoting ${SOURCE_IMAGES} to production hub." + if(SOURCE_IMAGES.isEmpty() || RELEASE_VERSION.isEmpty()) { + currentBuild.result = 'ABORTED' + error('Make sure at lease one product is added to be promoted with proper release version.') + } + } + } + } + stage('image-promote-to-prod') { + steps { + script { + for (product in SOURCE_IMAGES.split(',')) { + def productRepo = product.trim() + println("Promoting \"$productRepo\" from staging to production.") + promoteContainer( + imageRepository: productRepo, + version: RELEASE_VERSION, + dockerPromote: DOCKER_HUB_PROMOTE, + ecrPromote: ECR_PROMOTE, + latestTag: TAG_LATEST, + majorVersionTag: TAG_MAJOR_VERSION + ) + } + } + } + } + } + post() { + always { + script { + postCleanup() + sh "docker logout" + } + } + } +} diff --git a/jenkins/promotion/promote-yum-repos.jenkinsfile b/jenkins/promotion/promote-yum-repos.jenkinsfile index b2de0c7b04..fbe39be251 100644 --- a/jenkins/promotion/promote-yum-repos.jenkinsfile +++ b/jenkins/promotion/promote-yum-repos.jenkinsfile @@ -28,7 +28,7 @@ pipeline { stage('promote yum repo') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' alwaysPull true } diff --git a/jenkins/release-notes-check/release-notes-check.jenkinsfile b/jenkins/release-notes-check/release-notes-check.jenkinsfile new file mode 100644 index 0000000000..dcc7dca601 --- /dev/null +++ b/jenkins/release-notes-check/release-notes-check.jenkinsfile @@ -0,0 +1,79 @@ +lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) + +pipeline { + options { + timeout(time: 2, unit: 'HOURS') + } + agent none + parameters { + string( + name: 'INPUT_MANIFEST', + description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.', + trim: true + ) + string( + name: 'GIT_LOG_DATE', + description: 'in format yyyy-mm-dd, example 2022-07-26.', + trim: true + ) + choice( + name: 'ADD_COMMENT', + choices: ['true', 'false'], + description: 'Adds MardownTable output as comment on GIT_ISSUE_NUMBER.', + ) + string( + name: 'GIT_ISSUE_NUMBER', + description: 'The release issue number from opensearch-build repo.', + trim: true + ) + } + environment { + AGENT_X64 = 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' + GIT_LOG_DATE = "$GIT_LOG_DATE" + GIT_ISSUE_NUMBER = "$GIT_ISSUE_NUMBER" + ADD_COMMENT = "$ADD_COMMENT" + } + stages { + stage('detect docker image + args') { + agent { + docker { + label AGENT_X64 + image 'alpine:3' + alwaysPull true + } + } + steps { + script { + dockerAgent = detectDockerAgent() + currentBuild.description = INPUT_MANIFEST + } + } + post { + always { + postCleanup() + } + } + } + stage('Generate MarkDown table') { + agent { + docker { + label AGENT_X64 + image dockerAgent.image + alwaysPull true + } + } + steps { + script { + withCredentials([usernamePassword(credentialsId: "jenkins-github-bot-token", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { + sh "./run_releasenotes_check.sh manifests/${INPUT_MANIFEST}" + } + } + } + post { + always { + postCleanup() + } + } + } + } +} diff --git a/jenkins/release-tag/release-tag.jenkinsfile b/jenkins/release-tag/release-tag.jenkinsfile index 3742021fc7..a621543580 100644 --- a/jenkins/release-tag/release-tag.jenkinsfile +++ b/jenkins/release-tag/release-tag.jenkinsfile @@ -6,7 +6,7 @@ pipeline { } agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130' alwaysPull true } diff --git a/jenkins/rpm-validation/rpm-validation.jenkinsfile b/jenkins/rpm-validation/rpm-validation.jenkinsfile index 8585023fcc..bd3755a427 100644 --- a/jenkins/rpm-validation/rpm-validation.jenkinsfile +++ b/jenkins/rpm-validation/rpm-validation.jenkinsfile @@ -13,7 +13,7 @@ pipeline { ) string( name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host or Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host or Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', trim: true ) diff --git a/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile b/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile index 464373ef4a..80db04f055 100644 --- a/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile +++ b/jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile @@ -3,7 +3,7 @@ lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) pipeline { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2' alwaysPull true } @@ -20,16 +20,19 @@ pipeline { description: 'Path to upload to artifacts and signatures on s3. Eg: dummy_project/1.0' ) choice( - choices: ['linux'], + choices: ['linux', 'windows'], name: 'DISTRIBUTION_PLATFORM', description: 'What platform is this distribution build for?' ) choice( choices: ['.sig', '.rpm'], name: 'SIGNATURE_TYPE', - description: 'What is signature file type?' + description: 'What is signature file type? Required only for linux signing.' ) } + environment { + ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name') + } stages { stage('sign') { steps { @@ -52,11 +55,13 @@ pipeline { println("Note: only supported file types will be signed") for(filename in downloadedFiles){ - if (SIGNATURE_TYPE.equals('.sig')) { + if (DISTRIBUTION_PLATFORM == 'windows') { + filenamesForUrls.add(filename) + filenamesForUrls.add('signed/' + filename) + } else if (SIGNATURE_TYPE.equals('.sig')) { filenamesForUrls.add(filename) filenamesForUrls.add(filename + SIGNATURE_TYPE) - } - else { + } else { filenamesForUrls.add(filename) } } @@ -80,7 +85,6 @@ pipeline { artifactFileNames: filenamesForUrls, uploadPath: finalUploadPath ) - } } post() { diff --git a/jenkins/vulnerability-scan/whitesource-scan.jenkinsfile b/jenkins/vulnerability-scan/whitesource-scan.jenkinsfile index ca9033c768..25a34346f1 100644 --- a/jenkins/vulnerability-scan/whitesource-scan.jenkinsfile +++ b/jenkins/vulnerability-scan/whitesource-scan.jenkinsfile @@ -11,7 +11,7 @@ pipeline { stage('Run the scan') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' args '-e LANG=en_US.UTF-8' alwaysPull true @@ -19,7 +19,7 @@ pipeline { } steps { script { - withCredentials([string(credentialsId: 'whitesource-scan-apikey', variable: 'wss_apikey')]) { + withCredentials([string(credentialsId: 'jenkins-whitesource-scan-apikey', variable: 'wss_apikey')]) { echo 'Executing WhiteSource Scan script' scan() } @@ -44,9 +44,9 @@ pipeline { void scan() { sh """ - curl -SLO https://services.gradle.org/distributions/gradle-6.6.1-bin.zip - unzip gradle-6.6.1-bin.zip - GRADLE_HOME=`realpath gradle-6.6.1` + curl -SLO https://services.gradle.org/distributions/gradle-7.4.2-bin.zip + unzip gradle-7.4.2-bin.zip + GRADLE_HOME=`realpath gradle-7.4.2` export PATH=\${GRADLE_HOME}/bin:\$PATH gradle --version cd ./tools/vulnerability-scan/ diff --git a/manifests/1.0.0/opensearch-1.0.0-maven.yml b/legacy-manifests/1.0.0/opensearch-1.0.0-maven.yml similarity index 100% rename from manifests/1.0.0/opensearch-1.0.0-maven.yml rename to legacy-manifests/1.0.0/opensearch-1.0.0-maven.yml diff --git a/manifests/1.0.0/opensearch-1.0.0-test.yml b/legacy-manifests/1.0.0/opensearch-1.0.0-test.yml similarity index 100% rename from manifests/1.0.0/opensearch-1.0.0-test.yml rename to legacy-manifests/1.0.0/opensearch-1.0.0-test.yml diff --git a/manifests/1.0.0/opensearch-1.0.0.yml b/legacy-manifests/1.0.0/opensearch-1.0.0.yml similarity index 100% rename from manifests/1.0.0/opensearch-1.0.0.yml rename to legacy-manifests/1.0.0/opensearch-1.0.0.yml diff --git a/manifests/1.0.1/opensearch-1.0.1-test.yml b/legacy-manifests/1.0.1/opensearch-1.0.1-test.yml similarity index 100% rename from manifests/1.0.1/opensearch-1.0.1-test.yml rename to legacy-manifests/1.0.1/opensearch-1.0.1-test.yml diff --git a/manifests/1.0.1/opensearch-1.0.1.yml b/legacy-manifests/1.0.1/opensearch-1.0.1.yml similarity index 100% rename from manifests/1.0.1/opensearch-1.0.1.yml rename to legacy-manifests/1.0.1/opensearch-1.0.1.yml diff --git a/manifests/1.0.1/opensearch-dashboards-1.0.1.yml b/legacy-manifests/1.0.1/opensearch-dashboards-1.0.1.yml similarity index 100% rename from manifests/1.0.1/opensearch-dashboards-1.0.1.yml rename to legacy-manifests/1.0.1/opensearch-dashboards-1.0.1.yml diff --git a/manifests/1.1.0/opensearch-1.1.0-test.yml b/legacy-manifests/1.1.0/opensearch-1.1.0-test.yml similarity index 100% rename from manifests/1.1.0/opensearch-1.1.0-test.yml rename to legacy-manifests/1.1.0/opensearch-1.1.0-test.yml diff --git a/manifests/1.1.0/opensearch-1.1.0.yml b/legacy-manifests/1.1.0/opensearch-1.1.0.yml similarity index 100% rename from manifests/1.1.0/opensearch-1.1.0.yml rename to legacy-manifests/1.1.0/opensearch-1.1.0.yml diff --git a/manifests/1.1.0/opensearch-dashboards-1.1.0-test.yml b/legacy-manifests/1.1.0/opensearch-dashboards-1.1.0-test.yml similarity index 100% rename from manifests/1.1.0/opensearch-dashboards-1.1.0-test.yml rename to legacy-manifests/1.1.0/opensearch-dashboards-1.1.0-test.yml diff --git a/manifests/1.1.0/opensearch-dashboards-1.1.0.yml b/legacy-manifests/1.1.0/opensearch-dashboards-1.1.0.yml similarity index 100% rename from manifests/1.1.0/opensearch-dashboards-1.1.0.yml rename to legacy-manifests/1.1.0/opensearch-dashboards-1.1.0.yml diff --git a/manifests/1.1.1/opensearch-1.1.1-test.yml b/legacy-manifests/1.1.1/opensearch-1.1.1-test.yml similarity index 100% rename from manifests/1.1.1/opensearch-1.1.1-test.yml rename to legacy-manifests/1.1.1/opensearch-1.1.1-test.yml diff --git a/manifests/1.1.1/opensearch-1.1.1.yml b/legacy-manifests/1.1.1/opensearch-1.1.1.yml similarity index 100% rename from manifests/1.1.1/opensearch-1.1.1.yml rename to legacy-manifests/1.1.1/opensearch-1.1.1.yml diff --git a/manifests/1.1.1/opensearch-dashboards-1.1.1-test.yml b/legacy-manifests/1.1.1/opensearch-dashboards-1.1.1-test.yml similarity index 100% rename from manifests/1.1.1/opensearch-dashboards-1.1.1-test.yml rename to legacy-manifests/1.1.1/opensearch-dashboards-1.1.1-test.yml diff --git a/manifests/1.1.1/opensearch-dashboards-1.1.1.yml b/legacy-manifests/1.1.1/opensearch-dashboards-1.1.1.yml similarity index 100% rename from manifests/1.1.1/opensearch-dashboards-1.1.1.yml rename to legacy-manifests/1.1.1/opensearch-dashboards-1.1.1.yml diff --git a/manifests/1.2.0/opensearch-1.2.0-test.yml b/legacy-manifests/1.2.0/opensearch-1.2.0-test.yml similarity index 100% rename from manifests/1.2.0/opensearch-1.2.0-test.yml rename to legacy-manifests/1.2.0/opensearch-1.2.0-test.yml diff --git a/manifests/1.2.0/opensearch-1.2.0.yml b/legacy-manifests/1.2.0/opensearch-1.2.0.yml similarity index 100% rename from manifests/1.2.0/opensearch-1.2.0.yml rename to legacy-manifests/1.2.0/opensearch-1.2.0.yml diff --git a/manifests/1.2.0/opensearch-dashboards-1.2.0-test.yml b/legacy-manifests/1.2.0/opensearch-dashboards-1.2.0-test.yml similarity index 100% rename from manifests/1.2.0/opensearch-dashboards-1.2.0-test.yml rename to legacy-manifests/1.2.0/opensearch-dashboards-1.2.0-test.yml diff --git a/manifests/1.2.0/opensearch-dashboards-1.2.0.yml b/legacy-manifests/1.2.0/opensearch-dashboards-1.2.0.yml similarity index 100% rename from manifests/1.2.0/opensearch-dashboards-1.2.0.yml rename to legacy-manifests/1.2.0/opensearch-dashboards-1.2.0.yml diff --git a/manifests/1.2.1/opensearch-1.2.1-test.yml b/legacy-manifests/1.2.1/opensearch-1.2.1-test.yml similarity index 100% rename from manifests/1.2.1/opensearch-1.2.1-test.yml rename to legacy-manifests/1.2.1/opensearch-1.2.1-test.yml diff --git a/manifests/1.2.1/opensearch-1.2.1.yml b/legacy-manifests/1.2.1/opensearch-1.2.1.yml similarity index 100% rename from manifests/1.2.1/opensearch-1.2.1.yml rename to legacy-manifests/1.2.1/opensearch-1.2.1.yml diff --git a/manifests/1.2.1/opensearch-dashboards-1.2.1.yml b/legacy-manifests/1.2.1/opensearch-dashboards-1.2.1.yml similarity index 100% rename from manifests/1.2.1/opensearch-dashboards-1.2.1.yml rename to legacy-manifests/1.2.1/opensearch-dashboards-1.2.1.yml diff --git a/manifests/1.2.2/opensearch-1.2.2-test.yml b/legacy-manifests/1.2.2/opensearch-1.2.2-test.yml similarity index 100% rename from manifests/1.2.2/opensearch-1.2.2-test.yml rename to legacy-manifests/1.2.2/opensearch-1.2.2-test.yml diff --git a/manifests/1.2.2/opensearch-1.2.2.yml b/legacy-manifests/1.2.2/opensearch-1.2.2.yml similarity index 100% rename from manifests/1.2.2/opensearch-1.2.2.yml rename to legacy-manifests/1.2.2/opensearch-1.2.2.yml diff --git a/manifests/1.2.3/opensearch-1.2.3-test.yml b/legacy-manifests/1.2.3/opensearch-1.2.3-test.yml similarity index 100% rename from manifests/1.2.3/opensearch-1.2.3-test.yml rename to legacy-manifests/1.2.3/opensearch-1.2.3-test.yml diff --git a/manifests/1.2.3/opensearch-1.2.3.yml b/legacy-manifests/1.2.3/opensearch-1.2.3.yml similarity index 100% rename from manifests/1.2.3/opensearch-1.2.3.yml rename to legacy-manifests/1.2.3/opensearch-1.2.3.yml diff --git a/manifests/1.2.4/opensearch-1.2.4-test.yml b/legacy-manifests/1.2.4/opensearch-1.2.4-test.yml similarity index 100% rename from manifests/1.2.4/opensearch-1.2.4-test.yml rename to legacy-manifests/1.2.4/opensearch-1.2.4-test.yml diff --git a/manifests/1.2.4/opensearch-1.2.4.yml b/legacy-manifests/1.2.4/opensearch-1.2.4.yml similarity index 100% rename from manifests/1.2.4/opensearch-1.2.4.yml rename to legacy-manifests/1.2.4/opensearch-1.2.4.yml diff --git a/manifests/1.2.5/opensearch-1.2.5.yml b/legacy-manifests/1.2.5/opensearch-1.2.5.yml similarity index 100% rename from manifests/1.2.5/opensearch-1.2.5.yml rename to legacy-manifests/1.2.5/opensearch-1.2.5.yml diff --git a/manifests/1.3.4/opensearch-1.3.4-test.yml b/manifests/1.3.4/opensearch-1.3.4-test.yml new file mode 100644 index 0000000000..f50fd7fc1f --- /dev/null +++ b/manifests/1.3.4/opensearch-1.3.4-test.yml @@ -0,0 +1,71 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: dashboards-reports + working-directory: reports-scheduler + integ-test: + test-configs: + - without-security + + - name: observability + working-directory: opensearch-observability + integ-test: + test-configs: + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/1.3.4/opensearch-1.3.4.yml b/manifests/1.3.4/opensearch-1.3.4.yml index 9b7b550c63..442a1393b5 100644 --- a/manifests/1.3.4/opensearch-1.3.4.yml +++ b/manifests/1.3.4/opensearch-1.3.4.yml @@ -6,10 +6,103 @@ build: ci: image: name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: '1.3' + ref: tags/1.3.4 checks: - gradle:publish - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.3.4.0 + checks: + - gradle:publish + - gradle:properties:version + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: ml-commons + repository: https://github.com/opensearch-project/ml-commons.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: opensearch-ml-plugin + - name: security + repository: https://github.com/opensearch-project/security.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: tags/1.3.4.0 + platforms: + - darwin + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - darwin + - linux + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: dashboards-reports + repository: https://github.com/opensearch-project/dashboards-reports.git + ref: tags/1.3.4.0 + working_directory: reports-scheduler + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: opensearch-observability + repository: https://github.com/opensearch-project/observability + ref: tags/1.3.4.0 + working_directory: opensearch-observability + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.3.4.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: plugin diff --git a/manifests/1.3.4/opensearch-dashboards-1.3.4-test.yml b/manifests/1.3.4/opensearch-dashboards-1.3.4-test.yml new file mode 100644 index 0000000000..3ad90cc802 --- /dev/null +++ b/manifests/1.3.4/opensearch-dashboards-1.3.4-test.yml @@ -0,0 +1,17 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 +components: + - name: OpenSearch-Dashboards + bwc-test: + test-configs: + - with-security + - without-security + - name: functionalTestDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/1.3.4/opensearch-dashboards-1.3.4.yml b/manifests/1.3.4/opensearch-dashboards-1.3.4.yml index 58d1f700aa..fd5c44f75a 100644 --- a/manifests/1.3.4/opensearch-dashboards-1.3.4.yml +++ b/manifests/1.3.4/opensearch-dashboards-1.3.4.yml @@ -9,4 +9,35 @@ ci: components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git - ref: '1.3' + ref: tags/1.3.4 + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: tags/1.3.4 + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin.git + ref: tags/1.3.4.0 + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: tags/1.3.4.0 + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin + ref: tags/1.3.4.0 + - name: queryWorkbenchDashboards + repository: https://github.com/opensearch-project/sql.git + working_directory: workbench + ref: tags/1.3.4.0 + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reports.git + working_directory: dashboards-reports + ref: tags/1.3.4.0 + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations.git + working_directory: gantt-chart + ref: tags/1.3.4.0 + - name: observabilityDashboards + repository: https://github.com/opensearch-project/observability.git + working_directory: dashboards-observability + ref: tags/1.3.4.0 + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git + ref: tags/1.3.4.0 diff --git a/manifests/1.3.5/opensearch-1.3.5.yml b/manifests/1.3.5/opensearch-1.3.5.yml new file mode 100644 index 0000000000..d849245127 --- /dev/null +++ b/manifests/1.3.5/opensearch-1.3.5.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 1.3.5 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: '1.3' + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/1.3.5/opensearch-dashboards-1.3.5.yml b/manifests/1.3.5/opensearch-dashboards-1.3.5.yml new file mode 100644 index 0000000000..42b7aed637 --- /dev/null +++ b/manifests/1.3.5/opensearch-dashboards-1.3.5.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 1.3.5 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: '1.3' diff --git a/manifests/2.1.0/opensearch-2.1.0.yml b/manifests/2.1.0/opensearch-2.1.0.yml index 142353d7d1..39535db408 100644 --- a/manifests/2.1.0/opensearch-2.1.0.yml +++ b/manifests/2.1.0/opensearch-2.1.0.yml @@ -10,85 +10,85 @@ ci: components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: '2.1' + ref: tags/2.1.0 checks: - gradle:publish - gradle:properties:version - name: common-utils repository: https://github.com/opensearch-project/common-utils.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:publish - gradle:properties:version - name: job-scheduler repository: https://github.com/opensearch-project/job-scheduler.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: ml-commons repository: https://github.com/opensearch-project/ml-commons.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: security repository: https://github.com/opensearch-project/security.git - ref: '2.1' + ref: tags/2.1.0.0 - name: notifications-core repository: https://github.com/opensearch-project/notifications.git - ref: '2.1' + ref: tags/2.1.0.0 working_directory: notifications checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-notifications-core - name: notifications repository: https://github.com/opensearch-project/notifications.git - ref: '2.1' + ref: tags/2.1.0.0 working_directory: notifications checks: - gradle:properties:version - gradle:dependencies:opensearch.version: notifications - name: index-management repository: https://github.com/opensearch-project/index-management.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - name: sql repository: https://github.com/opensearch-project/sql.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - - gradle:dependencies:opensearch.version: plugin + - gradle:dependencies:opensearch.version: opensearch-sql-plugin - name: opensearch-observability repository: https://github.com/opensearch-project/observability - ref: '2.1' + ref: tags/2.1.0.0 working_directory: opensearch-observability checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: dashboards-reports repository: https://github.com/opensearch-project/dashboards-reports.git - ref: '2.1' + ref: tags/2.1.0.0 working_directory: reports-scheduler checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: alerting repository: https://github.com/opensearch-project/alerting.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version: alerting - name: k-NN repository: https://github.com/opensearch-project/k-NN.git - ref: '2.1' + ref: tags/2.1.0.0 platforms: - darwin - linux @@ -97,19 +97,19 @@ components: - gradle:dependencies:opensearch.version - name: anomaly-detection repository: https://github.com/opensearch-project/anomaly-detection.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: asynchronous-search repository: https://github.com/opensearch-project/asynchronous-search.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: cross-cluster-replication repository: https://github.com/opensearch-project/cross-cluster-replication.git - ref: '2.1' + ref: tags/2.1.0.0 checks: - gradle:properties:version - gradle:dependencies:opensearch.version diff --git a/manifests/2.1.0/opensearch-dashboards-2.1.0.yml b/manifests/2.1.0/opensearch-dashboards-2.1.0.yml index a4b39a2b00..8cb61dd2c6 100644 --- a/manifests/2.1.0/opensearch-dashboards-2.1.0.yml +++ b/manifests/2.1.0/opensearch-dashboards-2.1.0.yml @@ -9,36 +9,41 @@ ci: components: - name: OpenSearch-Dashboards repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git - ref: 2.x + ref: tags/2.1.0 + checks: + - npm:package:version + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: tags/2.1.0 - name: notificationsDashboards repository: https://github.com/opensearch-project/notifications.git working_directory: dashboards-notifications - ref: main + ref: tags/2.1.0.0 - name: queryWorkbenchDashboards repository: https://github.com/opensearch-project/sql.git working_directory: workbench - ref: main + ref: tags/2.1.0.0 - name: observabilityDashboards repository: https://github.com/opensearch-project/observability.git working_directory: dashboards-observability - ref: main + ref: tags/2.1.0.0 - name: ganttChartDashboards repository: https://github.com/opensearch-project/dashboards-visualizations.git working_directory: gantt-chart - ref: main + ref: tags/2.1.0.0 - name: reportsDashboards repository: https://github.com/opensearch-project/dashboards-reports.git working_directory: dashboards-reports - ref: main + ref: tags/2.1.0.0 - name: securityDashboards repository: https://github.com/opensearch-project/security-dashboards-plugin.git - ref: main + ref: tags/2.1.0.0 - name: alertingDashboards repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git - ref: main + ref: tags/2.1.0.0 - name: anomalyDetectionDashboards repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin - ref: main + ref: tags/2.1.0.0 - name: indexManagementDashboards repository: https://github.com/opensearch-project/index-management-dashboards-plugin.git - ref: main + ref: tags/2.1.0.0 diff --git a/manifests/2.1.1/opensearch-2.1.1.yml b/manifests/2.1.1/opensearch-2.1.1.yml new file mode 100644 index 0000000000..289a9e6531 --- /dev/null +++ b/manifests/2.1.1/opensearch-2.1.1.yml @@ -0,0 +1,40 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 2.1.1 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: '2.1' + checks: + - gradle:publish + - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: '2.1' + checks: + - gradle:publish + - gradle:properties:version + platforms: + - darwin + - linux + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: '2.1' + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - darwin + - linux + - name: security + repository: https://github.com/opensearch-project/security.git + ref: '2.1' + platforms: + - darwin + - linux diff --git a/manifests/2.2.0/opensearch-2.2.0-test.yml b/manifests/2.2.0/opensearch-2.2.0-test.yml new file mode 100644 index 0000000000..c6491921d1 --- /dev/null +++ b/manifests/2.2.0/opensearch-2.2.0-test.yml @@ -0,0 +1,84 @@ +--- +schema-version: '1.0' +name: OpenSearch +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: index-management + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + additional-cluster-configs: + path.repo: [/tmp] + + - name: anomaly-detection + integ-test: + build-dependencies: + - job-scheduler + test-configs: + - with-security + - without-security + + - name: asynchronous-search + integ-test: + test-configs: + - with-security + - without-security + + - name: alerting + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1] + + - name: notifications + working-directory: notifications + integ-test: + test-configs: + - with-security + - without-security + + - name: sql + integ-test: + test-configs: + - with-security + - without-security + additional-cluster-configs: + script.context.field.max_compilations_rate: 1000/1m + + - name: k-NN + integ-test: + test-configs: + - with-security + - without-security + + - name: dashboards-reports + working-directory: reports-scheduler + integ-test: + test-configs: + - without-security + + - name: opensearch-observability + working-directory: opensearch-observability + integ-test: + test-configs: + - without-security + + - name: ml-commons + integ-test: + test-configs: + - with-security + - without-security + + - name: cross-cluster-replication + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/2.2.0/opensearch-2.2.0.yml b/manifests/2.2.0/opensearch-2.2.0.yml index f5383b5af7..0612e71a3a 100644 --- a/manifests/2.2.0/opensearch-2.2.0.yml +++ b/manifests/2.2.0/opensearch-2.2.0.yml @@ -6,10 +6,41 @@ build: ci: image: name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 components: - name: OpenSearch repository: https://github.com/opensearch-project/OpenSearch.git - ref: 2.x + ref: '2.2' checks: - gradle:publish - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: '2.x' + checks: + - gradle:publish + - gradle:properties:version + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: '2.2' + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: security + repository: https://github.com/opensearch-project/security.git + ref: main + - name: geospatial + repository: https://github.com/opensearch-project/geospatial.git + ref: '2.2' + checks: + - gradle:publish + - gradle:properties:version + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: '2.x' + platforms: + - darwin + - linux + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version diff --git a/manifests/2.2.0/opensearch-dashboards-2.2.0-test.yml b/manifests/2.2.0/opensearch-dashboards-2.2.0-test.yml new file mode 100644 index 0000000000..3ad90cc802 --- /dev/null +++ b/manifests/2.2.0/opensearch-dashboards-2.2.0-test.yml @@ -0,0 +1,17 @@ +--- +schema-version: '1.0' +name: OpenSearch Dashboards +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2 +components: + - name: OpenSearch-Dashboards + bwc-test: + test-configs: + - with-security + - without-security + - name: functionalTestDashboards + integ-test: + test-configs: + - with-security + - without-security diff --git a/manifests/2.2.0/opensearch-dashboards-2.2.0.yml b/manifests/2.2.0/opensearch-dashboards-2.2.0.yml new file mode 100644 index 0000000000..74adaaf8ea --- /dev/null +++ b/manifests/2.2.0/opensearch-dashboards-2.2.0.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 2.2.0 +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: 2.x diff --git a/manifests/3.0.0/opensearch-3.0.0.yml b/manifests/3.0.0/opensearch-3.0.0.yml index 07d8434d7d..b47c18ef9f 100644 --- a/manifests/3.0.0/opensearch-3.0.0.yml +++ b/manifests/3.0.0/opensearch-3.0.0.yml @@ -14,3 +14,9 @@ components: checks: - gradle:publish - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch-dashboards/1.x/manifest.yml b/manifests/templates/opensearch-dashboards/1.x/manifest.yml new file mode 100644 index 0000000000..ef01447725 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/1.x/manifest.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main diff --git a/manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml b/manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml new file mode 100644 index 0000000000..118957593a --- /dev/null +++ b/manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml @@ -0,0 +1,55 @@ +--- +schema-version: '1.0' +ci: + image: + name: opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 +build: + name: OpenSearch Dashboards + version: 1.1.0 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: tags/1.1.0 + - name: alertingDashboards + repository: https://github.com/opensearch-project/alerting-dashboards-plugin + ref: tags/1.1.0.0 + # hybrid repo (workbench) + - name: queryWorkbenchDashboards + repository: https://github.com/opensearch-project/sql.git + working_directory: workbench + ref: tags/1.1.0.0 + # hybrid repo (dashboards-notifications) + # - name: notificationsDashboards + # repository: https://github.com/opensearch-project/notifications.git + # working_directory: dashboards-notifications + # ref: main + - name: securityDashboards + repository: https://github.com/opensearch-project/security-dashboards-plugin + ref: tags/1.1.0.0 + - name: indexManagementDashboards + repository: https://github.com/opensearch-project/index-management-dashboards-plugin + ref: tags/1.1.0.0 + - name: anomalyDetectionDashboards + repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin + ref: tags/1.1.0.0 + # hybrid repo (dashboards-reports) + - name: reportsDashboards + repository: https://github.com/opensearch-project/dashboards-reports.git + working_directory: dashboards-reports + ref: tags/1.1.0.0 + # hybrid repo (dashboards-notebooks) + - name: notebooksDashboards + repository: https://github.com/opensearch-project/dashboards-notebooks.git + working_directory: dashboards-notebooks + ref: tags/1.1.0.0 + - name: traceAnalyticsDashboards + repository: https://github.com/opensearch-project/trace-analytics.git + ref: tags/1.1.0.0 + # hybrid repo (gantt-chart) + - name: ganttChartDashboards + repository: https://github.com/opensearch-project/dashboards-visualizations + working_directory: gantt-chart + ref: tags/1.1.0.0 + - name: functionalTestDashboards + repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git + ref: main diff --git a/manifests/templates/opensearch-dashboards/2.x/manifest.yml b/manifests/templates/opensearch-dashboards/2.x/manifest.yml new file mode 100644 index 0000000000..9074cf5d0b --- /dev/null +++ b/manifests/templates/opensearch-dashboards/2.x/manifest.yml @@ -0,0 +1,14 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main + checks: + - npm:package:version diff --git a/manifests/templates/opensearch-dashboards/3.x/manifest.yml b/manifests/templates/opensearch-dashboards/3.x/manifest.yml new file mode 100644 index 0000000000..ef01447725 --- /dev/null +++ b/manifests/templates/opensearch-dashboards/3.x/manifest.yml @@ -0,0 +1,12 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main diff --git a/manifests/templates/opensearch-dashboards/default/manifest.yml b/manifests/templates/opensearch-dashboards/default/manifest.yml new file mode 100644 index 0000000000..9074cf5d0b --- /dev/null +++ b/manifests/templates/opensearch-dashboards/default/manifest.yml @@ -0,0 +1,14 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch Dashboards + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2 +components: + - name: OpenSearch-Dashboards + repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git + ref: main + checks: + - npm:package:version diff --git a/manifests/templates/opensearch/1.x/manifest.yml b/manifests/templates/opensearch/1.x/manifest.yml new file mode 100644 index 0000000000..a1934229d4 --- /dev/null +++ b/manifests/templates/opensearch/1.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-11 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/1.x/os-template-1.0.0.yml b/manifests/templates/opensearch/1.x/os-template-1.0.0.yml new file mode 100644 index 0000000000..e3840c8d0c --- /dev/null +++ b/manifests/templates/opensearch/1.x/os-template-1.0.0.yml @@ -0,0 +1,50 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 1.0.0 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: tags/1.0.0 + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.0.0.0 + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.0.0.0 + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.0.0.0 + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.0.0.0 + - name: security + repository: https://github.com/opensearch-project/security.git + ref: tags/1.0.0.0 + # - name: performance-analyzer-rca + # repository: https://github.com/opensearch-project/performance-analyzer-rca.git + # ref: tags/1.0.0.0 + # - name: performance-analyzer + # repository: https://github.com/opensearch-project/performance-analyzer.git + # ref: tags/1.0.0.0 + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/1.0.0.0 + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: tags/1.0.0.0 + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: tags/1.0.0.0 + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: tags/1.0.0.0 + - name: dashboards-reports + repository: https://github.com/opensearch-project/dashboards-reports.git + ref: tags/1.0.0.0 + working_directory: reports-scheduler + - name: dashboards-notebooks + repository: https://github.com/opensearch-project/dashboards-notebooks.git + ref: tags/1.0.0.0 + working_directory: opensearch-notebooks diff --git a/manifests/templates/opensearch/1.x/os-template-1.1.0.yml b/manifests/templates/opensearch/1.x/os-template-1.1.0.yml new file mode 100644 index 0000000000..99f773301b --- /dev/null +++ b/manifests/templates/opensearch/1.x/os-template-1.1.0.yml @@ -0,0 +1,100 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 1.1.0 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: tags/1.1.0 + checks: + - gradle:publish + - gradle:properties:version + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.1.0.0 + checks: + - gradle:publish + - gradle:properties:version + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: plugin + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version: alerting + - name: security + repository: https://github.com/opensearch-project/security.git + ref: tags/1.1.0.0 + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: performance-analyzer-rca + repository: https://github.com/opensearch-project/performance-analyzer-rca.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - darwin + - linux + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + platforms: + - darwin + - linux + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: tags/1.1.0.0 + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: dashboards-reports + repository: https://github.com/opensearch-project/dashboards-reports.git + ref: tags/1.1.0.0 + working_directory: reports-scheduler + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version + - name: dashboards-notebooks + repository: https://github.com/opensearch-project/dashboards-notebooks.git + ref: tags/1.1.0.0 + working_directory: opensearch-notebooks + checks: + - gradle:properties:version + - gradle:dependencies:opensearch.version diff --git a/manifests/templates/opensearch/1.x/os-template-1.2.0.yml b/manifests/templates/opensearch/1.x/os-template-1.2.0.yml new file mode 100644 index 0000000000..0768b73c66 --- /dev/null +++ b/manifests/templates/opensearch/1.x/os-template-1.2.0.yml @@ -0,0 +1,60 @@ +--- +schema-version: '1.0' +ci: + image: + name: opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 + args: -e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot +build: + name: OpenSearch + version: 1.2.0 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: tags/1.2.0 + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: tags/1.2.0.0 + - name: job-scheduler + repository: https://github.com/opensearch-project/job-scheduler.git + ref: tags/1.2.0.0 + - name: alerting + repository: https://github.com/opensearch-project/alerting.git + ref: tags/1.2.0.0 + - name: asynchronous-search + repository: https://github.com/opensearch-project/asynchronous-search.git + ref: tags/1.2.0.0 + - name: index-management + repository: https://github.com/opensearch-project/index-management.git + ref: tags/1.2.0.0 + - name: k-NN + repository: https://github.com/opensearch-project/k-NN.git + ref: tags/1.2.0.0 + platforms: + - darwin + - linux + - name: security + repository: https://github.com/opensearch-project/security.git + ref: tags/1.2.0.0 + - name: performance-analyzer + repository: https://github.com/opensearch-project/performance-analyzer.git + ref: tags/1.2.0.0 + platforms: + - darwin + - linux + - name: anomaly-detection + repository: https://github.com/opensearch-project/anomaly-detection.git + ref: tags/1.2.0.0 + - name: cross-cluster-replication + repository: https://github.com/opensearch-project/cross-cluster-replication.git + ref: tags/1.2.0.0 + - name: sql + repository: https://github.com/opensearch-project/sql.git + ref: tags/1.2.0.0 + - name: dashboards-reports + repository: https://github.com/opensearch-project/dashboards-reports.git + ref: tags/1.2.0.0 + working_directory: reports-scheduler + - name: opensearch-observability + repository: https://github.com/opensearch-project/trace-analytics.git + ref: tags/1.2.0.0 + working_directory: opensearch-observability diff --git a/manifests/templates/opensearch/2.x/manifest.yml b/manifests/templates/opensearch/2.x/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/2.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/3.x/manifest.yml b/manifests/templates/opensearch/3.x/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/3.x/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/manifests/templates/opensearch/default/manifest.yml b/manifests/templates/opensearch/default/manifest.yml new file mode 100644 index 0000000000..d798342efe --- /dev/null +++ b/manifests/templates/opensearch/default/manifest.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.0' +build: + name: OpenSearch + version: 'replace' +ci: + image: + name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2 + args: -e JAVA_HOME=/opt/java/openjdk-17 +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: main + checks: + - gradle:publish + - gradle:properties:version diff --git a/meta/.gitignore b/meta/.gitignore index 595aa7c9df..ce10ea2a50 100644 --- a/meta/.gitignore +++ b/meta/.gitignore @@ -23,3 +23,4 @@ /opensearch-build/ /ml-commons/ /geospatial/ +/dashboards-maps/ diff --git a/meta/.meta b/meta/.meta index 5ab4a0ef91..e2d14c08fb 100644 --- a/meta/.meta +++ b/meta/.meta @@ -23,6 +23,7 @@ "common-utils": "git@github.com:opensearch-project/common-utils.git", "opensearch-build": "git@github.com:opensearch-project/opensearch-build.git", "ml-commons": "git@github.com:opensearch-project/ml-commons.git", - "geospatial": "git@github.com:opensearch-project/geospatial.git" + "geospatial": "git@github.com:opensearch-project/geospatial.git", + "dashboards-maps": "git@github.com:opensearch-project/dashboards-maps.git" } } diff --git a/release-notes/opensearch-release-notes-1.3.4.md b/release-notes/opensearch-release-notes-1.3.4.md new file mode 100644 index 0000000000..32188369c7 --- /dev/null +++ b/release-notes/opensearch-release-notes-1.3.4.md @@ -0,0 +1,14 @@ +# OpenSearch and OpenSearch Dashboards 1.3.4 Release Notes + +## Release Details + +OpenSearch and OpenSearch Dashboards 1.3.4 includes the following bug fixes, and maintenance updates. + +OpenSearch [Release Notes](https://github.com/opensearch-project/OpenSearch/blob/main/release-notes/opensearch.release-notes-1.3.4.md). + + +## BUG FIXES + +### OpenSearch Dashboards Reports +* [1.3] Move HTML sanitization from client to server side (#382) ([#386](https://github.com/opensearch-project/dashboards-reports/pull/386)) + diff --git a/release-notes/opensearch-release-notes-2.1.0.md b/release-notes/opensearch-release-notes-2.1.0.md new file mode 100644 index 0000000000..a94b3324a5 --- /dev/null +++ b/release-notes/opensearch-release-notes-2.1.0.md @@ -0,0 +1,245 @@ +# OpenSearch and Dashboards 2.1.0 Release Notes + +## Release Highlights + + +* OpenSearch 2.1.0 supports version spoofing against 2.0.0 where it was [removed](https://github.com/opensearch-project/opensearch/pull/3530). +* You can now configure nodes with [dynamic nodes roles](https://github.com/opensearch-project/OpenSearch/pull/3436), which allows for custom node roles that won't affect node start processes. +* The [ML node role](https://github.com/opensearch-project/ml-commons/pull/346) can be configured for ML functions and tasks. +* SQL and PPL queries now supports [relevance-based search](https://github.com/opensearch-project/sql/issues/182), including [match_function](https://github.com/opensearch-project/sql/pull/204), [match_phrase](https://github.com/opensearch-project/sql/pull/604), and [match_bool_prefix](https://github.com/opensearch-project/sql/pull/634). +* OpenSearch now supports [multi-term aggregation](https://github.com/opensearch-project/OpenSearch/pull/2687). + +## Release Details + +OpenSearch and OpenSearch Dashboards 2.1.0 includes the following features, enhancements, bug fixes, infrastructure, documentation, maintenance, and refactoring updates. + +OpenSearch [Release Notes](https://github.com/opensearch-project/OpenSearch/blob/main/release-notes/opensearch.release-notes-2.1.0.md). + +OpenSearch Dashboards [Release Notes](https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/release-notes/opensearch-dashboards.release-notes-2.1.0.md). + +## FEATURES + +### Opensearch Index Management +* Merge snapshot management into main branch ([#390](https://github.com/opensearch-project/index-management/pull/390)) +* Adds snapshot management notification implementation ([#387](https://github.com/opensearch-project/index-management/pull/387)) +* Snapshot management default date format in snapshot name ([#392](https://github.com/opensearch-project/index-management/pull/392)) + + +### Opensearch Index Management Dashboards Plugin +* Merge snapshot management into main branch ([#205](https://github.com/opensearch-project/index-management-dashboards-plugin/pull/205)) + + +### Opensearch Ml Common +* Dispatch ML task to ML node first ([#346](https://github.com/opensearch-project/ml-commons/pull/346)) + + +### Opensearch Performance Analyzer +* Thread Metrics RCA ([#180](https://github.com/opensearch-project/performance-analyzer/pull/180)) + + +### Opensearch SQL +* Support match_phrase filter function in SQL and PPL ([#604](https://github.com/opensearch-project/sql/pull/604)) +* Add implementation for `simple_query_string` relevance search function in SQL and PPL ([#635](https://github.com/opensearch-project/sql/pull/635)) +* Add multi_match to SQL plugin ([#649](https://github.com/opensearch-project/sql/pull/649)) +* Integ match bool prefix #187 ([#634](https://github.com/opensearch-project/sql/pull/634)) +* PPL describe command ([#646](https://github.com/opensearch-project/sql/pull/646)) + + +## ENHANCEMENT + +### Opensearch Index Management Dashboards Plugin +* Snapshot management small fixes ([#208](https://github.com/opensearch-project/index-management-dashboards-plugin/pull/208)) +* Tune the column width, fix the problem of showing snapshot failures ([#210](https://github.com/opensearch-project/index-management-dashboards-plugin/pull/210)) + + +### Opensearch Security Dashboards Plugin +* Dynamically compute OpenID redirectUri from proxy HTTP headers ([#929](https://github.com/opensearch-project/security-dashboards-plugin/pull/929)) +* Clear the sessionStorage when logging out ([#1003](https://github.com/opensearch-project/security-dashboards-plugin/pull/1003)) +S + +## BUG FIX + +### Opensearch Security +* Cluster permissions evaluation logic will now include `index_template` type action ([#1885](https://github.com/opensearch-project/security/pull/1885)) +* Add missing settings to plugin allowed list ([#1814](https://github.com/opensearch-project/security/pull/1814)) +* Updates license headers ([#1829](https://github.com/opensearch-project/security/pull/1829)) +* Prevent recursive action groups ([#1868](https://github.com/opensearch-project/security/pull/1868)) +* Update `org.springframework:spring-core` to `5.3.20` ([#1850](https://github.com/opensearch-project/security/pull/1850)) + + +### Opensearch Security Dashboards Plugin +* Disable private tenant for read only users ([#868](https://github.com/opensearch-project/security-dashboards-plugin/pull/868)) +* Replace _opendistro route with _plugins ([#895](https://github.com/opensearch-project/security-dashboards-plugin/pull/895)) +ES + +### Opensearch SQL +* Integ replace junit assertthat with hamcrest import ([#616](https://github.com/opensearch-project/sql/pull/616)) +* Integ relevance function it fix ([#608](https://github.com/opensearch-project/sql/pull/608)) +* Fix merge conflict on function name ([#664](https://github.com/opensearch-project/sql/pull/664)) +* Fix `fuzziness` parsing in `multi_match` function. Update tests. ([#668](https://github.com/opensearch-project/sql/pull/668)) +* ODBC SSL Compliance Fix ([#653](https://github.com/opensearch-project/sql/pull/653)) + + +## INFRASTRUCTURE + +### Opensearch Anomaly Detection +* Cluster manager revert fix ([#584](https://github.com/opensearch-project/anomaly-detection/pull/584)) +* Adding HCAD data ingestion script to AD ([#585](https://github.com/opensearch-project/anomaly-detection/pull/585)) +* Update ingestion ([#592](https://github.com/opensearch-project/anomaly-detection/pull/592)) +* Adding custom plugin to publish zip to maven ([#594](https://github.com/opensearch-project/anomaly-detection/pull/594)) + + +### Opensearch Anomaly Detection Dashboards +* Added UT for validation API related components ([#252](https://github.com/opensearch-project/anomaly-detection-dashboards-plugin/pull/252)) +* Run UT/IT on all branches ([#228](https://github.com/opensearch-project/anomaly-detection-dashboards-plugin/pull/228)) + + +### Opensearch Index Management +* Uses custom plugin to publish zips to maven ([#366](https://github.com/opensearch-project/index-management/pull/366)) + + +### Opensearch k-NN + +* Update opensearch version in BWCWorkflow ([#402](https://github.com/opensearch-project/k-NN/pull/402)) +* Adding workflow for creating documentation issues ([#403](https://github.com/opensearch-project/k-NN/pull/403)) +* Add Querying Functionality to OSB ([#409](https://github.com/opensearch-project/k-NN/pull/409)) +* Add OpenSearch Benchmark index workload for k-NN ([#364](https://github.com/opensearch-project/k-NN/pull/364)) +* Set tests.security.manager flag to false in integTestRemote task +([#410](https://github.com/opensearch-project/k-NN/pull/410)) + + +### Opensearch Ml Common +* Bump RCF version to 3.0-rc3 ([#340](https://github.com/opensearch-project/ml-commons/pull/340)) + + +### Opensearch Observability +* Uses custom plugin to publish zips to maven ([#786](https://github.com/opensearch-project/observability/pull/786)) + + +### Opensearch SQL +* Match Query Unit Tests ([#614](https://github.com/opensearch-project/sql/pull/614)) +* Uses custom plugin to publish zips to maven ([#638](https://github.com/opensearch-project/sql/pull/638)) + + +## DOCUMENTATION + +### Opensearch Alerting +* Added 2.1 release notes. ([#485](https://github.com/opensearch-project/alerting/pull/485)) + + +### Opensearch Alerting Dashboards Plugin +* Added 2.1 release notes. ([#284](https://github.com/opensearch-project/alerting-dashboards-plugin/pull/284)) + + +### Opensearch Common Utils +* Added 2.1 release notes. ([#194](https://github.com/opensearch-project/common-utils/pull/194)) + + +### Opensearch Index Management +* Updated issue templates from .github. ([#324](https://github.com/opensearch-project/index-management/pull/324)) + + +### Opensearch Index Management Dashboards Plugin +* Adding workflow for creating documentation issues. ([#197](https://github.com/opensearch-project/index-management-dashboards-plugin/pull/197)) + + +### OpenSearch Job Scheduler +* Added 2.1 release notes. ([#198](https://github.com/opensearch-project/job-scheduler/pull/198)) + + +## MAINTENANCE + +### Opensearch Anomaly Detection +* 2.1 version bump and Gradle bump ([#582](https://github.com/opensearch-project/anomaly-detection/pull/582)) + + +### Opensearch Anomaly Detection Dashboards +* Bump to 2.1.0 compatibility ([#282](https://github.com/opensearch-project/anomaly-detection-dashboards-plugin/pull/282)) + +### Opensearch Alerting +* Bumped version to 2.1.0, and gradle to 7.4.2. ([#475](https://github.com/opensearch-project/alerting/pull/475])) + + +### Opensearch Alerting Dashboards Plugin +* Bumped version from 2.0.1 to 2.1.0. ([#277](https://github.com/opensearch-project/alerting-dashboards-plugin/pull/277)) +* Bumped OpenSearch-Dashboards branch used by by the unit-tests-workflow. ([#278](https://github.com/opensearch-project/alerting-dashboards-plugin/pull/278)) + + +### Opensearch Common Utils +* Upgrade gradle to 7.4.2. ([#191](https://github.com/opensearch-project/common-utils/pull/191)) +* Bump up the version to 2.1. ([#190](https://github.com/opensearch-project/common-utils/pull/190)) + + +### Opensearch Dashboards Visualizations +* Version bump to 2.1.0 ([#89](https://github.com/opensearch-project/dashboards-visualizations/pull/89)) + + +### Opensearch Index Management +* Version upgrade to 2.1.0 ([#389](https://github.com/opensearch-project/index-management/pull/389)) + + +### Opensearch k-NN +* Bumping main version to opensearch core 2.1.0 ([#411](https://github.com/opensearch-project/k-NN/pull/411)) + + +### Opensearch Index Management Dashboards Plugin +* Version bump 2.1.0 ([#206](https://github.com/opensearch-project/index-management-dashboards-plugin/pull/206)) + + +### OpenSearch Job Scheduler +* Bump up the OS version to 2.1. ([#195](https://github.com/opensearch-project/job-scheduler/pull/195)) + + +### Opensearch Notifications +* Upgrade Notifications and Notifications Dashboards to 2.1 ([#468](https://github.com/opensearch-project/notifications/pull/468)) +* Fix Email test for security integration test ([#462](https://github.com/opensearch-project/notifications/pull/462)) + + +### Opensearch Observability +* Bump version to 2.1.0 and gradle version to 7.4.2 ([#817](https://github.com/opensearch-project/observability/pull/817)) + + +### Opensearch Performance Analyzer +* Update 2.1 release version ([#192](https://github.com/opensearch-project/performance-analyzer-rca/pull/192)) +* Update 2.1 release version ([#232](https://github.com/opensearch-project/performance-analyzer/pull/232)) + + +### Opensearch Security +* Revert "Bump version to 2.1.0.0 (#1865)" ([#1882](https://github.com/opensearch-project/security/pull/1882)) +* Bump version to 2.1.0.0 ([#1865](https://github.com/opensearch-project/security/pull/1865)) +* Revert "Bump version to 2.1.0.0 (#1855)" ([#1864](https://github.com/opensearch-project/security/pull/1864)) +* Bump version to 2.1.0.0 ([#1855](https://github.com/opensearch-project/security/pull/1855)) +* Add suppression for all removal warnings ([#1828](https://github.com/opensearch-project/security/pull/1828)) +* Update support link ([#1851](https://github.com/opensearch-project/security/pull/1851)) +* Create 2.0.0 release notes ([#1854](https://github.com/opensearch-project/security/pull/1854)) +* Switch to standard OpenSearch gradle build ([#1888](https://github.com/opensearch-project/security/pull/1888)) +* Fix build break from cluster manager changes ([#1911](https://github.com/opensearch-project/security/pull/1911)) +* Update org.apache.zookeeper:zookeeper to 3.7.1 ([#1912](https://github.com/opensearch-project/security/pull/1912)) + + +### Opensearch Security Dashboards Plugin +* Bump version to 2.1.0.0 ([#1004](https://github.com/opensearch-project/security-dashboards-plugin/pull/1004)) +* Adds 1.3.1.0 release notes ([#988](https://github.com/opensearch-project/security-dashboards-plugin/pull/988)) +* Create release notes 2.0.0 ([#996](https://github.com/opensearch-project/security-dashboards-plugin/pull/996)) + + +### Opensearch SQL +* Change plugin folder name to opensearch-sql-plugin ([#670](https://github.com/opensearch-project/sql/pull/670)) +* Version bump to 2.1.0 and gradle version bump ([#655](https://github.com/opensearch-project/sql/pull/655)) + + +## REFACTORING + +### Opensearch k-NN +* Adding support for Lombok ([#393](https://github.com/opensearch-project/k-NN/pull/393)) + + +### Opensearch Observability +* Make common delete modal for components ([#766](https://github.com/opensearch-project/observability/pull/766)) +* Sync app and app list types ([#763](https://github.com/opensearch-project/observability/pull/763)) + + +### Opensearch Security +* Remove master keywords ([#1886](https://github.com/opensearch-project/security/pull/1886)) + + diff --git a/run_releasenotes_check.sh b/run_releasenotes_check.sh new file mode 100755 index 0000000000..aa961e7d58 --- /dev/null +++ b/run_releasenotes_check.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +set -e + +DIR="$(dirname "$0")" +"$DIR/run.sh" "$DIR/src/run_releasenotes_check.py" $@ diff --git a/scripts/components/notifications-core/build.sh b/scripts/components/notifications-core/build.sh index 95cc23128b..79449c432a 100644 --- a/scripts/components/notifications-core/build.sh +++ b/scripts/components/notifications-core/build.sh @@ -74,3 +74,7 @@ fi mkdir -p ./$OUTPUT/plugins notifCoreZipPath=$(ls core/build/distributions/ | grep .zip) cp -v core/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins + +./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +mkdir -p $OUTPUT/maven/org/opensearch/plugin +cp -r ./build/local-staging-repo/org/opensearch/plugin/opensearch-notifications-core $OUTPUT/maven/org/opensearch/plugin/ diff --git a/scripts/components/notifications/build.sh b/scripts/components/notifications/build.sh index 5ddaf06657..d715bd40de 100644 --- a/scripts/components/notifications/build.sh +++ b/scripts/components/notifications/build.sh @@ -75,3 +75,6 @@ mkdir -p ./$OUTPUT/plugins notifCoreZipPath=$(ls notifications/build/distributions/ | grep .zip) cp -v notifications/build/distributions/$notifCoreZipPath ./$OUTPUT/plugins +./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +mkdir -p $OUTPUT/maven/org/opensearch/plugin +cp -r ./build/local-staging-repo/org/opensearch/plugin/notifications $OUTPUT/maven/org/opensearch/plugin/ diff --git a/scripts/default/opensearch/build.sh b/scripts/default/opensearch/build.sh index 5cf8540650..3fbe1755b9 100755 --- a/scripts/default/opensearch/build.sh +++ b/scripts/default/opensearch/build.sh @@ -77,3 +77,8 @@ distributions="$(dirname "${zipPath}")" echo "COPY ${distributions}/*.zip" mkdir -p $OUTPUT/plugins cp ${distributions}/*.zip ./$OUTPUT/plugins + +# Publish plugin zips to maven +./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER +mkdir -p $OUTPUT/maven/org/opensearch +cp -r ./build/local-staging-repo/org/opensearch/. $OUTPUT/maven/org/opensearch diff --git a/scripts/gradle/gradle-check.sh b/scripts/gradle/gradle-check.sh new file mode 100644 index 0000000000..2d0dc522c5 --- /dev/null +++ b/scripts/gradle/gradle-check.sh @@ -0,0 +1,69 @@ +#!/bin/bash + + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +# This script is used in OpenSearch Core repo github actions +# To trigger Jenkins Gradle Check from a PR + + +JENKINS_URL="https://build.ci.opensearch.org" +TIMEPASS=0 +TIMEOUT=7200 +RESULT="null" +TRIGGER_TOKEN=$1 +PAYLOAD_JSON="{\"pr_from_sha\": \"$pr_from_sha\", \"pr_from_clone_url\": \"$pr_from_clone_url\", \"pr_to_clone_url\": \"$pr_to_clone_url\", \"pr_title\": \"$pr_title\", \"pr_number\": \"$pr_number\"}" + +echo "Trigger Jenkins workflows" +JENKINS_REQ=`curl -s -XPOST \ + -H "Authorization: Bearer $TRIGGER_TOKEN" \ + -H "Content-Type: application/json" \ + "$JENKINS_URL/generic-webhook-trigger/invoke" \ + --data "$(echo $PAYLOAD_JSON)"` + +echo $PAYLOAD_JSON | jq +echo $JENKINS_REQ + +QUEUE_URL=$(echo $JENKINS_REQ | jq --raw-output '.jobs."gradle-check".url') +echo QUEUE_URL $QUEUE_URL +echo "wait for jenkins to start workflow" && sleep 15 + +echo "Check if queue exist in Jenkins after triggering" +if [ -z "$QUEUE_URL" ] || [ "$QUEUE_URL" != "null" ]; then + WORKFLOW_URL=$(curl -s -XGET ${JENKINS_URL}/${QUEUE_URL}api/json | jq --raw-output .executable.url) + echo WORKFLOW_URL $WORKFLOW_URL + + echo "Use queue information to find build number in Jenkins if available" + if [ -z "$WORKFLOW_URL" ] || [ "$WORKFLOW_URL" != "null" ]; then + + RUNNING="true" + + echo "Waiting for Jenkins to complete the run" + while [ "$RUNNING" = "true" ] && [ "$TIMEPASS" -le "$TIMEOUT" ]; do + echo "Still running, wait for another 30 seconds before checking again, max timeout $TIMEOUT" + echo "Jenkins Workflow Url: $WORKFLOW_URL" + TIMEPASS=$(( TIMEPASS + 30 )) && echo time pass: $TIMEPASS + sleep 30 + RUNNING=$(curl -s -XGET ${WORKFLOW_URL}api/json | jq --raw-output .building) + done + + echo "Complete the run, checking results now......" + RESULT=$(curl -s -XGET ${WORKFLOW_URL}api/json | jq --raw-output .result) + + fi +fi + +echo "Please check jenkins url for logs: $WORKFLOW_URL" + +if [ "$RESULT" != "SUCCESS" ]; then + echo "Result: $RESULT" + exit 1 +else + echo "Result: $RESULT" + echo "Get codeCoverage.xml" && curl -SLO ${WORKFLOW_URL}artifact/codeCoverage.xml + echo 0 +fi diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index e40e3a9e67..f1dee86fd3 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -11,7 +11,7 @@ from abc import abstractmethod from typing import Dict, List, Type, Union -from manifests.input_manifest import InputManifest +from manifests.input_manifest import InputComponents, InputManifest from manifests.manifests import Manifests from manifests_workflow.component_opensearch import ComponentOpenSearch from manifests_workflow.component_opensearch_dashboards_min import ComponentOpenSearchDashboardsMin @@ -48,7 +48,12 @@ def files(self, name: str) -> List: return results @abstractmethod - def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], component_klass: Type[ComponentOpenSearch], keep: bool = False) -> None: + def update( + self, + min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOpenSearchDashboardsMin]], + component_klass: Type[ComponentOpenSearch], + keep: bool = False, + ) -> None: known_versions = self.versions logging.info(f"Known versions: {known_versions}") main_versions: Dict = {} @@ -106,30 +111,23 @@ def update(self, min_klass: Union[Type[ComponentOpenSearchMin], Type[ComponentOp self.add_to_cron(release_version) def create_manifest(self, version: str, components: List = []) -> InputManifest: - image_map = { - "opensearch": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", - "opensearch-dashboards": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2" - } - - data: Dict = { - "schema-version": "1.0", - "build": { - "name": self.name, - "version": version - }, - "ci": { - "image": { - "name": image_map[self.prefix] - } - }, - "components": [], - } + templates_base_path = os.path.join(self.manifests_path(), "templates") + template_version_folder = version.split(".")[0] + ".x" + template_full_path = os.path.join(templates_base_path, self.prefix, template_version_folder, "manifest.yml") + if not os.path.exists(template_full_path): + template_full_path = os.path.join(templates_base_path, self.prefix, "default", "manifest.yml") + + manifest = InputManifest.from_file(open(template_full_path)) + + manifest.build.version = version + manifests_components = [] for component in components: logging.info(f" Adding {component.name}") - data["components"].append(component.to_dict()) + manifests_components.append(component.to_dict()) - return InputManifest(data) + manifest.components = InputComponents(manifests_components) # type: ignore + return manifest def write_manifest(self, version: str, components: List = []) -> None: logging.info(f"Creating new version: {version}") @@ -151,10 +149,7 @@ def add_to_cron(self, version: str) -> None: if cron_entry in data: raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}") - data = data.replace( - "parameterizedCron '''\n", - f"parameterizedCron '''\n{' ' * 12}{cron_entry}" - ) + data = data.replace("parameterizedCron '''\n", f"parameterizedCron '''\n{' ' * 12}{cron_entry}") with open(jenkinsfile, "w") as f: f.write(data) diff --git a/src/run_releasenotes_check.py b/src/run_releasenotes_check.py new file mode 100644 index 0000000000..6443f6ae65 --- /dev/null +++ b/src/run_releasenotes_check.py @@ -0,0 +1,98 @@ +import logging +import os +import subprocess +import sys +from typing import Any, List + +from pytablewriter import MarkdownTableWriter + +from checkout_workflow.checkout_args import CheckoutArgs +from git.git_repository import GitRepository +from manifests.input_manifest import InputComponentFromSource, InputManifest +from system import console +from system.temporary_directory import TemporaryDirectory + +# in format yyyy-mm-dd, example 2022-07-26 +gitLogDate = os.getenv('GIT_LOG_DATE') +# Github issue to add the generated markdown table as a new comment +gitIssueNumber = os.getenv('GIT_ISSUE_NUMBER') +# Boolen if set to null, just prints on the console. +addComment = os.getenv('ADD_COMMENT', default="true") +# Token used to add comment on an issue +userToken = os.getenv('GITHUB_TOKEN') + + +def main(gitLogDate: str, addComment: str) -> int: + if gitLogDate is None: + print("No GIT_LOG_DATE environmental value passed") + sys.exit() + value_matrix = [] # type: List[Any] + args = CheckoutArgs() + console.configure(level=args.logging_level) + manifest = InputManifest.from_file(args.manifest) + writer = MarkdownTableWriter( + table_name=f"{manifest.build.name} CommitID(after {gitLogDate}) & Release Notes info", + headers=["Repo", "Branch", "CommitID", "Release Notes"], + value_matrix=value_matrix + ) + with TemporaryDirectory(chdir=True) as work_dir: + logging.info(f"Checking out into {work_dir.name}") + for component in manifest.components.select(): + logging.info(f"Checking out {component.name}") + repo_list = [] # type: List[Any] + if type(component) is InputComponentFromSource: + with GitRepository( + component.repository, + component.ref, + os.path.join(work_dir.name, component.name), + component.working_directory, + ) as repo: + logging.debug(f"Checked out {component.name} into {repo.dir}") + gitLogCmd = f'git log --after={gitLogDate} --pretty=format:"%h"' + gitLog = subprocess.check_output(gitLogCmd.split(), cwd=repo.dir).decode() + repo_list.append(component.name) + repo_list.append(component.ref) + repo_list.append(gitLog) + release_notes = "NO" + if component.name == 'OpenSearch' or component.name == 'OpenSearch-Dashboards': + release_notes_check = f'.release-notes-{manifest.build.version}.md' + else: + release_notes_check = f'.release-notes-{manifest.build.version}.0.md' + try: + if any(fname.endswith(release_notes_check) for fname in os.listdir(f'{repo.dir}/release-notes/')): + release_notes = "YES" + except FileNotFoundError: + print("No such release-notes file or directory") + release_notes = "NULL" + repo_list.append(release_notes) + value_matrix.append(repo_list) + writer.write_table() + writer.dump("{}/table.txt".format(work_dir.name)) + if addComment == "true": + if gitIssueNumber is not None: + comment_under_issue(gitIssueNumber, f'{work_dir.name}/table.txt', userToken) + else: + print("No GIT_ISSUE_NUMBER environmental value passed") + sys.exit() + return 0 + + +def subprocess_cmd(command: str) -> int: + process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) + proc_stdout = process.communicate()[0].strip() + print(proc_stdout) + return 0 + + +def comment_under_issue(issue_number: str, comment_text: str, userToken: str) -> int: + # GITHUB_TOKEN used to add comment, to passed in GH issue, using GIT_ISSUE_NUMBER + os.environ['GITHUB_TOKEN'] = userToken + subprocess_cmd('gh auth login') + subprocess_cmd('gh repo clone https://github.com/opensearch-project/opensearch-build.git; cd opensearch-build') + subprocess_cmd('gh issue comment {} --body-file {} --repo opensearch-project/opensearch-build'.format(issue_number, comment_text)) + subprocess_cmd('gh auth logout --hostname github.com; cd ..; rm -rf opensearch-build') + return 0 + + +if __name__ == "__main__": + main(gitLogDate, addComment) diff --git a/src/run_sign.py b/src/run_sign.py index e9716c4d7a..89edc6b4e9 100755 --- a/src/run_sign.py +++ b/src/run_sign.py @@ -10,7 +10,6 @@ from sign_workflow.sign_args import SignArgs from sign_workflow.sign_artifacts import SignArtifacts -from sign_workflow.signer import Signer from system import console @@ -24,7 +23,7 @@ def main() -> int: components=args.components, artifact_type=args.type, signature_type=args.sigtype, - signer=Signer() + platform=args.platform ) sign.sign() diff --git a/src/sign_workflow/sign_args.py b/src/sign_workflow/sign_args.py index 742a1e0408..cefa5155f6 100644 --- a/src/sign_workflow/sign_args.py +++ b/src/sign_workflow/sign_args.py @@ -11,7 +11,8 @@ class SignArgs: - ACCEPTED_SIGNATURE_FILE_TYPES = [".sig"] + ACCEPTED_SIGNATURE_FILE_TYPES = [".sig", ".asc"] + ACCEPTED_PLATFORM = ["linux", "windows"] target: Path components: List[str] @@ -25,7 +26,7 @@ def __init__(self) -> None: parser.add_argument("-c", "--component", type=str, nargs='*', dest="components", help="Component or components to sign") parser.add_argument("--type", help="Artifact type") parser.add_argument("--sigtype", choices=self.ACCEPTED_SIGNATURE_FILE_TYPES, help="Type of signature file.", default=".asc") - parser.add_argument("--platform", nargs="?", help="Distribution platform.", default="linux") + parser.add_argument("--platform", choices=self.ACCEPTED_PLATFORM, help="Distribution platform.", default="linux") parser.add_argument( "-v", "--verbose", diff --git a/src/sign_workflow/sign_artifacts.py b/src/sign_workflow/sign_artifacts.py index 7355926d8b..3dbe3ee1c8 100644 --- a/src/sign_workflow/sign_artifacts.py +++ b/src/sign_workflow/sign_artifacts.py @@ -14,6 +14,7 @@ from manifests.build_manifest import BuildManifest from sign_workflow.signer import Signer +from sign_workflow.signers import Signers class SignArtifacts: @@ -21,14 +22,16 @@ class SignArtifacts: component: str artifact_type: str signature_type: str + platform: str signer: Signer - def __init__(self, target: Path, components: List[str], artifact_type: str, signature_type: str, signer: Signer) -> None: + def __init__(self, target: Path, components: List[str], artifact_type: str, signature_type: str, platform: str) -> None: self.target = target self.components = components self.artifact_type = artifact_type self.signature_type = signature_type - self.signer = signer + self.platform = platform + self.signer = Signers.create(platform) @abstractmethod def __sign__(self) -> None: @@ -54,9 +57,9 @@ def __signer_class__(self, path: Path) -> Type[Any]: return SignArtifactsExistingArtifactFile @classmethod - def from_path(self, path: Path, components: List[str], artifact_type: str, signature_type: str, signer: Signer) -> Any: + def from_path(self, path: Path, components: List[str], artifact_type: str, signature_type: str, platform: str) -> Any: klass = self.__signer_class__(path) - return klass(path, components, artifact_type, signature_type, signer) + return klass(path, components, artifact_type, signature_type, platform) class SignWithBuildManifest(SignArtifacts): diff --git a/src/sign_workflow/signer.py b/src/sign_workflow/signer.py index ab9f6e06f0..f68badb5fe 100644 --- a/src/sign_workflow/signer.py +++ b/src/sign_workflow/signer.py @@ -8,22 +8,16 @@ import logging import os +from abc import ABC, abstractmethod from pathlib import Path from typing import List from git.git_repository import GitRepository -""" -This class is responsible for signing an artifact using the OpenSearch-signer-client and verifying its signature. -The signed artifacts will be found in the same location as the original artifacts. -""" - -class Signer: +class Signer(ABC): git_repo: GitRepository - ACCEPTED_FILE_TYPES = [".zip", ".jar", ".war", ".pom", ".module", ".tar.gz", ".whl", ".crate", ".rpm"] - def __init__(self) -> None: self.git_repo = GitRepository(self.get_repo_url(), "HEAD", working_subdirectory="src") self.git_repo.execute("./bootstrap") @@ -42,15 +36,13 @@ def sign_artifacts(self, artifacts: List[str], basepath: Path, signature_type: s continue self.generate_signature_and_verify(artifact, basepath, signature_type) + @abstractmethod def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None: - location = os.path.join(basepath, artifact) - self.sign(location, signature_type) - self.verify(location + signature_type) + pass + @abstractmethod def is_valid_file_type(self, file_name: str) -> bool: - return any( - file_name.endswith(x) for x in Signer.ACCEPTED_FILE_TYPES - ) + pass def get_repo_url(self) -> str: if "GITHUB_TOKEN" in os.environ: @@ -62,20 +54,6 @@ def __remove_existing_signature__(self, signature_file: str) -> None: logging.warning(f"Removing existing signature file {signature_file}") os.remove(signature_file) - def sign(self, filename: str, signature_type: str) -> None: - signature_file = filename + signature_type - self.__remove_existing_signature__(signature_file) - signing_cmd = [ - "./opensearch-signer-client", - "-i", - filename, - "-o", - signature_file, - "-p", - "pgp", - ] - self.git_repo.execute(" ".join(signing_cmd)) - - def verify(self, filename: str) -> None: - verify_cmd = ["gpg", "--verify-files", filename] - self.git_repo.execute(" ".join(verify_cmd)) + @abstractmethod + def sign(self, artifact: str, basepath: Path, signature_type: str) -> None: + pass diff --git a/src/sign_workflow/signer_pgp.py b/src/sign_workflow/signer_pgp.py new file mode 100644 index 0000000000..18675ddeac --- /dev/null +++ b/src/sign_workflow/signer_pgp.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +from pathlib import Path + +from sign_workflow.signer import Signer + +""" +This class is responsible for signing an artifact using the OpenSearch-signer-client and verifying its signature. +The signed artifacts will be found in the same location as the original artifacts. +""" + + +class SignerPGP(Signer): + + ACCEPTED_FILE_TYPES = [".zip", ".jar", ".war", ".pom", ".module", ".tar.gz", ".whl", ".crate", ".rpm"] + + def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None: + location = os.path.join(basepath, artifact) + self.sign(artifact, basepath, signature_type) + self.verify(location + signature_type) + + def is_valid_file_type(self, file_name: str) -> bool: + return any( + file_name.endswith(x) for x in SignerPGP.ACCEPTED_FILE_TYPES + ) + + def sign(self, artifact: str, basepath: Path, signature_type: str) -> None: + filename = os.path.join(basepath, artifact) + signature_file = filename + signature_type + self.__remove_existing_signature__(signature_file) + signing_cmd = [ + "./opensearch-signer-client", + "-i", + filename, + "-o", + signature_file, + "-p", + "pgp", + ] + self.git_repo.execute(" ".join(signing_cmd)) + + def verify(self, filename: str) -> None: + verify_cmd = ["gpg", "--verify-files", filename] + self.git_repo.execute(" ".join(verify_cmd)) diff --git a/src/sign_workflow/signer_windows.py b/src/sign_workflow/signer_windows.py new file mode 100644 index 0000000000..30ece8eb9b --- /dev/null +++ b/src/sign_workflow/signer_windows.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +from pathlib import Path + +from sign_workflow.signer import Signer + +""" +This class is responsible for signing an artifact using the OpenSearch-signer-client and verifying its signature. +The signed artifacts will be found in the subfolder called signed under the origin location as the original artifacts. +""" + + +class SignerWindows(Signer): + + ACCEPTED_FILE_TYPES = [".msi", ".exe", ".dll", ".sys", ".ps1", ".psm1", ".psd1", ".cat", ".zip"] + + def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None: + self.sign(artifact, basepath, signature_type) + + def is_valid_file_type(self, file_name: str) -> bool: + return any( + file_name.endswith(x) for x in SignerWindows.ACCEPTED_FILE_TYPES + ) + + def sign(self, artifact: str, basepath: Path, signature_type: str) -> None: + filename = os.path.join(basepath, artifact) + signed_prefix = "signed_" + signature_file = os.path.join(basepath, signed_prefix + artifact) + self.__remove_existing_signature__(signature_file) + signing_cmd = [ + "./opensearch-signer-client", + "-i", + filename, + "-o", + signature_file, + "-p", + "windows", + ] + self.git_repo.execute(" ".join(signing_cmd)) + signed_folder = os.path.join(basepath, "signed") + if not os.path.exists(signed_folder): + os.mkdir(signed_folder) + signed_location = os.path.join(signed_folder, artifact) + os.rename(signature_file, signed_location) diff --git a/src/sign_workflow/signers.py b/src/sign_workflow/signers.py new file mode 100644 index 0000000000..7d27fb57a9 --- /dev/null +++ b/src/sign_workflow/signers.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + + +from sign_workflow.signer import Signer +from sign_workflow.signer_pgp import SignerPGP +from sign_workflow.signer_windows import SignerWindows + + +class Signers: + TYPES = { + "windows": SignerWindows, + "linux": SignerPGP, + } + + @classmethod + def from_platform(cls, platform: str) -> Signer: + klass = cls.TYPES.get(platform, None) + if not klass: + raise ValueError(f"Unsupported type of platform for signing: {platform}") + return klass # type: ignore[return-value] + + @classmethod + def create(cls, platform: str) -> Signer: + klass = cls.from_platform(platform) + return klass() # type: ignore[no-any-return, operator] diff --git a/tests/jenkins/TestCCRPerfTest.groovy b/tests/jenkins/TestCCRPerfTest.groovy index 9711e3804b..16946aff34 100644 --- a/tests/jenkins/TestCCRPerfTest.groovy +++ b/tests/jenkins/TestCCRPerfTest.groovy @@ -53,7 +53,7 @@ class TestCCRPerfTest extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(1)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true}".toString() )) } diff --git a/tests/jenkins/TestCopyContainer.groovy b/tests/jenkins/TestCopyContainer.groovy index 345fa5a7ec..4ffe56af7c 100644 --- a/tests/jenkins/TestCopyContainer.groovy +++ b/tests/jenkins/TestCopyContainer.groovy @@ -13,7 +13,7 @@ class TestCopyContainer extends BuildPipelineTest { binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'sample-agent-AssumeRole') binding.setVariable('AWS_ACCOUNT_ARTIFACT', '1234567890') binding.setVariable('DATA_PREPPER_STAGING_CONTAINER_REPOSITORY', 'sample_dataprepper_ecr_url') - helper.registerAllowedMethod('withAWS', [Map, Closure], null) + helper.registerAllowedMethod('withAWS', [Map, Closure], null) super.setUp() } diff --git a/tests/jenkins/TestGetRepositoryCommit.groovy b/tests/jenkins/TestGetRepositoryCommit.groovy new file mode 100644 index 0000000000..8d44c89735 --- /dev/null +++ b/tests/jenkins/TestGetRepositoryCommit.groovy @@ -0,0 +1,31 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + + +class TestGetRepositoryCommit extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new GetRepositoryCommitLibTester( + '', + 'tests/jenkins/data/opensearch-2.0.0.yml', + 'commits.yml' + ) + ) + super.setUp() + } + + @Test + void testGetRepositoryCommit() { + super.testPipeline("tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile") + } +} diff --git a/tests/jenkins/TestOpenSearchBwcTest.groovy b/tests/jenkins/TestOpenSearchBwcTest.groovy index b574e62d82..1df9ab2fb9 100644 --- a/tests/jenkins/TestOpenSearchBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchBwcTest.groovy @@ -18,7 +18,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { def buildId = 717 def buildManifest = "tests/jenkins/data/opensearch-1.3.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/${buildId}/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz" - def agentLabel = "Jenkins-Agent-al2-x64-c54xlarge-Docker-Host" + def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" this.registerLibTester(new DetectTestDockerAgentLibTester()) this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest)) @@ -29,7 +29,7 @@ class TestOpenSearchBwcTest extends BuildPipelineTest { 'BWC Tests Successful', '', testManifest, - 'INTEG_TEST_WEBHOOK')) + 'jenkins-integ-test-webhook')) super.setUp() // Variables diff --git a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy index cce6013c9e..db1ca6b5be 100644 --- a/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsBwcTest.groovy @@ -18,7 +18,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { def buildId = 215 def buildManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz" - def agentLabel = "Jenkins-Agent-al2-x64-c54xlarge-Docker-Host" + def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" this.registerLibTester(new DetectTestDockerAgentLibTester()) this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest)) @@ -29,7 +29,7 @@ class TestOpenSearchDashboardsBwcTest extends BuildPipelineTest { 'BWC Tests Successful', '', testManifest, - 'INTEG_TEST_WEBHOOK')) + 'jenkins-integ-test-webhook')) super.setUp() // Variables diff --git a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy index ca24389c1b..c54d929467 100644 --- a/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy @@ -18,7 +18,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { def buildId = 215 def buildManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz" - def agentLabel = "Jenkins-Agent-al2-x64-c54xlarge-Docker-Host" + def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" this.registerLibTester(new DetectTestDockerAgentLibTester()) this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest)) @@ -29,7 +29,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest { 'Integration Tests Successful', '', testManifest, - 'INTEG_TEST_WEBHOOK')) + 'jenkins-integ-test-webhook')) super.setUp() // Variables diff --git a/tests/jenkins/TestOpenSearchIntegTest.groovy b/tests/jenkins/TestOpenSearchIntegTest.groovy index 3b84bb24fb..2a1f919fbf 100644 --- a/tests/jenkins/TestOpenSearchIntegTest.groovy +++ b/tests/jenkins/TestOpenSearchIntegTest.groovy @@ -18,7 +18,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { def buildId = 717 def buildManifest = "tests/jenkins/data/opensearch-1.3.0-build.yml" def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/${buildId}/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz" - def agentLabel = "Jenkins-Agent-al2-x64-c54xlarge-Docker-Host" + def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host" this.registerLibTester(new DetectTestDockerAgentLibTester()) this.registerLibTester(new DownloadBuildManifestLibTester(buildManifestUrl, buildManifest)) @@ -29,7 +29,7 @@ class TestOpenSearchIntegTest extends BuildPipelineTest { 'Integration Tests Successful', '', testManifest, - 'INTEG_TEST_WEBHOOK')) + 'jenkins-integ-test-webhook')) super.setUp() // Variables diff --git a/tests/jenkins/TestPromoteArtifacts.groovy b/tests/jenkins/TestPromoteArtifacts.groovy index c2a430d8a5..591b50a463 100644 --- a/tests/jenkins/TestPromoteArtifacts.groovy +++ b/tests/jenkins/TestPromoteArtifacts.groovy @@ -29,28 +29,27 @@ class TestPromoteArtifacts extends BuildPipelineTest { binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc') binding.setVariable('DISTRIBUTION_JOB_NAME', 'vars-build') - binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket') - binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account') binding.setVariable('STAGE_NAME', 'stage') binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42') binding.setVariable('DISTRIBUTION_BUILD_NUMBER', '33') binding.setVariable('DISTRIBUTION_PLATFORM', 'linux') binding.setVariable('DISTRIBUTION_ARCHITECTURE', 'x64') - binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'downloadRoleName') - binding.setVariable('AWS_ACCOUNT_PUBLIC', 'publicAccount') - binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'artifactPromotionRole') - binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'artifactsAccount') - binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'prod-bucket-name') binding.setVariable('WORKSPACE', 'tests/jenkins') binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') - binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role') - binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id') - binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket') - binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket') + def configs = ["role": "dummy_role", + "external_id": "dummy_ID", + "unsigned_bucket": "dummy_unsigned_bucket", + "signed_bucket": "dummy_signed_bucket"] + binding.setVariable('configs', configs) + helper.registerAllowedMethod("readJSON", [Map.class], {c -> configs}) helper.registerAllowedMethod("git", [Map]) helper.registerAllowedMethod("s3Download", [Map]) helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> closure.delegate = delegate return helper.callClosure(closure) diff --git a/tests/jenkins/TestPromoteContainer.groovy b/tests/jenkins/TestPromoteContainer.groovy new file mode 100644 index 0000000000..b5dcb3f260 --- /dev/null +++ b/tests/jenkins/TestPromoteContainer.groovy @@ -0,0 +1,116 @@ +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +class TestPromoteContainer extends BuildPipelineTest { + + String PROMOTE_PRODUCT = 'opensearch:2.0.1.2901, opensearch-dashboards:2.0.1-2345, data-prepper:2.0.1.123' + String RELEASE_VERSION = '2.0.1' + + @Before + void setUp() { + binding.setVariable('SOURCE_IMAGES', PROMOTE_PRODUCT) + binding.setVariable('RELEASE_VERSION', RELEASE_VERSION) + binding.setVariable('DOCKER_USERNAME', 'dummy_docker_username') + binding.setVariable('DOCKER_PASSWORD', 'dummy_docker_password') + binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'dummy-agent-AssumeRole') + binding.setVariable('AWS_ACCOUNT_ARTIFACT', '1234567890') + binding.setVariable('DATA_PREPPER_STAGING_CONTAINER_REPOSITORY', 'dummy_dataprepper_ecr_url') + + + helper.registerAllowedMethod('withAWS', [Map, Closure], null) + super.setUp() + + } + + @Test + public void testPromoteContainerToDocker() { + String dockerPromote = true + String ecrPromote = false + String latestBoolean = false + String majorVersionBoolean = false + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDocker.jenkinsfile") + } + + @Test + public void testPromoteContainerToDockerLatest() { + String dockerPromote = true + String ecrPromote = false + String latestBoolean = true + String majorVersionBoolean = false + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatest.jenkinsfile") + } + + @Test + public void testPromoteContainerToDockerMajor() { + String dockerPromote = true + String ecrPromote = false + String latestBoolean = false + String majorVersionBoolean = true + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerMajor.jenkinsfile") + } + + @Test + public void testPromoteContainerToDockerLatestMajor() { + String dockerPromote = true + String ecrPromote = false + String latestBoolean = true + String majorVersionBoolean = true + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatestMajor.jenkinsfile") + } + + @Test + public void testPromoteContainerToECRLatestMajor() { + String dockerPromote = false + String ecrPromote = true + String latestBoolean = true + String majorVersionBoolean = true + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToECRLatestMajor.jenkinsfile") + } + + @Test + public void testPromoteContainerToDockerECRLatestMajor() { + String dockerPromote = true + String ecrPromote = true + String latestBoolean = true + String majorVersionBoolean = true + binding.setVariable('DOCKER_HUB_PROMOTE', dockerPromote) + binding.setVariable('ECR_PROMOTE', ecrPromote) + binding.setVariable('TAG_LATEST', latestBoolean) + binding.setVariable('TAG_MAJOR_VERSION', majorVersionBoolean) + + super.testPipeline("jenkins/promotion/promote-docker-ecr.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerECRLatestMajor.jenkinsfile") + } + +} diff --git a/tests/jenkins/TestPromoteYumRepos.groovy b/tests/jenkins/TestPromoteYumRepos.groovy index 17307e7744..dc6ff59055 100644 --- a/tests/jenkins/TestPromoteYumRepos.groovy +++ b/tests/jenkins/TestPromoteYumRepos.groovy @@ -20,15 +20,18 @@ class TestPromoteYumRepos extends BuildPipelineTest { super.setUp() binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc') - binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'artifactPromotionRole') - binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'artifactsAccount') - binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'prod-bucket-name') binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') - binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role') - binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id') - binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket') - binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket') + def configs = ["role": "dummy_role", + "external_id": "dummy_ID", + "unsigned_bucket": "dummy_unsigned_bucket", + "signed_bucket": "dummy_signed_bucket"] + binding.setVariable('configs', configs) + helper.registerAllowedMethod("readJSON", [Map.class], {c -> configs}) helper.registerAllowedMethod("git", [Map]) + helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> closure.delegate = delegate return helper.callClosure(closure) diff --git a/tests/jenkins/TestPublishNotification.groovy b/tests/jenkins/TestPublishNotification.groovy index 7b30d2db5f..1da7357b04 100644 --- a/tests/jenkins/TestPublishNotification.groovy +++ b/tests/jenkins/TestPublishNotification.groovy @@ -17,7 +17,7 @@ class TestPublishNotification extends BuildPipelineTest { void setUp() { this.registerLibTester(new PublishNotificationLibTester( - ':white_check_mark:', 'Successful Build' , 'extra', '1.2.0/opensearch-1.2.0.yml', 'BUILD_NOTICE_WEBHOOK')) + ':white_check_mark:', 'Successful Build' , 'extra', '1.2.0/opensearch-1.2.0.yml', 'jenkins-build-notice-webhook')) super.setUp() } diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy index 36bf2a0495..0457aafdfc 100644 --- a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -56,7 +56,7 @@ class TestRunNonSecurityPerfTestScript extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(1)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString() )) } diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index 6bd1445a8b..3aaa792fb8 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -56,7 +56,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { assertThat(s3DownloadCommands.size(), equalTo(2)) assertThat(s3DownloadCommands, hasItem( - "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + "{file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}".toString() )) } diff --git a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt index 7a5d5d356f..164e9f7d92 100644 --- a/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/cross-cluster-replication/perf-test.jenkinsfile.txt @@ -5,7 +5,7 @@ perf-test.timeout({time=10, unit=HOURS}) perf-test.echo(Executing on agent [label:none]) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -21,7 +21,7 @@ perf-test.writeFile({file=messages/perf-test.msg, text=CCR Performance tests for #1236}) perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.stage(perf-test, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -42,8 +42,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config-ccr.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config-ccr.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --component cross-cluster-replication) @@ -80,7 +83,7 @@ CCR Performance tests for 1236 completed}) uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) @@ -88,8 +91,8 @@ CCR Performance tests for 1236 completed}) perf-test.findFiles({excludes=, glob=messages/*}) perf-test.dir(messages, groovy.lang.Closure) perf-test.deleteDir() - perf-test.publishNotification({icon=:white_check_mark:, message=CCR Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + perf-test.publishNotification({icon=:white_check_mark:, message=CCR Performance Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=perf-test diff --git a/tests/jenkins/jenkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt index cc78959a78..45c74efa21 100644 --- a/tests/jenkins/jenkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt @@ -2,6 +2,11 @@ release-data-prepper-all-artifacts.legacySCM(groovy.lang.Closure) release-data-prepper-all-artifacts.library({identifier=jenkins@20211123, retriever=null}) release-data-prepper-all-artifacts.pipeline(groovy.lang.Closure) + release-data-prepper-all-artifacts.credentials(jenkins-data-prepper-artifact-staging-site) + release-data-prepper-all-artifacts.credentials(jenkins-data-prepper-staging-container-repository) + release-data-prepper-all-artifacts.credentials(jenkins-artifact-promotion-role) + release-data-prepper-all-artifacts.credentials(jenkins-aws-production-account) + release-data-prepper-all-artifacts.credentials(jenkins-artifact-production-bucket-name) release-data-prepper-all-artifacts.timeout({time=1, unit=HOURS}) release-data-prepper-all-artifacts.echo(Executing on agent [label:none]) release-data-prepper-all-artifacts.stage(Download Archives, groovy.lang.Closure) @@ -12,22 +17,60 @@ release-data-prepper-all-artifacts.stage(Sign Archives, groovy.lang.Closure) release-data-prepper-all-artifacts.script(groovy.lang.Closure) release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/archive, sigtype=.sig, platform=linux}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - /tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux - ) +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux + ) release-data-prepper-all-artifacts.stage(Release Archives to Production Distribution Bucket, groovy.lang.Closure) release-data-prepper-all-artifacts.script(groovy.lang.Closure) release-data-prepper-all-artifacts.withAWS({role=production-role-name, roleAccount=aws-account-artifact, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) @@ -84,25 +127,63 @@ release-data-prepper-all-artifacts.stage(Sign Maven Artifacts, groovy.lang.Closure) release-data-prepper-all-artifacts.script(groovy.lang.Closure) release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/maven, type=maven, platform=linux}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - /tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux - ) +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux + ) release-data-prepper-all-artifacts.stage(Upload Artifacts to Sonatype, groovy.lang.Closure) release-data-prepper-all-artifacts.script(groovy.lang.Closure) - release-data-prepper-all-artifacts.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD}) + release-data-prepper-all-artifacts.usernamePassword({credentialsId=jenkins-sonatype-creds, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD}) release-data-prepper-all-artifacts.withCredentials([[SONATYPE_USERNAME, SONATYPE_PASSWORD]], groovy.lang.Closure) release-data-prepper-all-artifacts.sh(/tmp/workspace/publish/stage-maven-release.sh /tmp/workspace/maven) release-data-prepper-all-artifacts.script(groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt index 60f91148c8..3a5d3d6f31 100644 --- a/tests/jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/maven-sign-release/maven-sign-release.jenkinsfile.txt @@ -2,34 +2,83 @@ maven-sign-release.legacySCM(groovy.lang.Closure) maven-sign-release.library({identifier=jenkins@20211123, retriever=null}) maven-sign-release.pipeline(groovy.lang.Closure) + maven-sign-release.credentials(jenkins-artifact-bucket-name) +<<<<<<< HEAD +<<<<<<< HEAD + maven-sign-release.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) +======= maven-sign-release.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) +>>>>>>> 8517abc (Move artifact bucket name from env to creds (#2327)) +======= + maven-sign-release.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) +>>>>>>> 09366a4 (Make agent label have consistent capital naming (#2353)) maven-sign-release.stage(sign, groovy.lang.Closure) maven-sign-release.script(groovy.lang.Closure) maven-sign-release.echo(Downloading from S3.) maven-sign-release.downloadFromS3({destPath=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.0.0/123/linux/x64/builds/, force=true}) - downloadFromS3.withAWS({role=Dummy_Download_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.0.0/123/linux/x64/builds/, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + downloadFromS3.withCredentials([AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/workspace/artifacts, bucket=job-s3-bucket-name, path=distribution-build-opensearch/1.0.0/123/linux/x64/builds/, force=true}) maven-sign-release.echo(Signing Maven artifacts.) maven-sign-release.signArtifacts({artifactPath=/tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml, type=maven, platform=linux}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - /tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux - ) +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/artifacts/distribution-build-opensearch/1.0.0/123/linux/x64/builds/opensearch/manifest.yml --type=maven --platform=linux + ) maven-sign-release.stage(stage maven artifacts, groovy.lang.Closure) maven-sign-release.script(groovy.lang.Closure) - maven-sign-release.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD}) + maven-sign-release.usernamePassword({credentialsId=jenkins-sonatype-creds, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD}) maven-sign-release.withCredentials([[SONATYPE_USERNAME, SONATYPE_PASSWORD]], groovy.lang.Closure) maven-sign-release.sh($WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/artifacts/$ARTIFACT_PATH/opensearch/maven) maven-sign-release.script(groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt index ade86a86b2..76767aea5d 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/bwc-test.jenkinsfile.txt @@ -5,11 +5,11 @@ bwc-test.timeout({time=3, unit=HOURS}) bwc-test.echo(Executing on agent [label:none]) bwc-test.stage(verify-parameters, groovy.lang.Closure) - bwc-test.echo(Executing on agent [label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]) + bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) - bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -18,7 +18,7 @@ TestManifest.asBoolean() detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) - bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -59,7 +59,7 @@ uploadTestResults.stash({includes=messages/*, name=messages-DUMMY_STAGE_NAME}) bwc-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - bwc-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) bwc-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([bwc-test]) @@ -67,8 +67,8 @@ bwc-test.findFiles({excludes=, glob=messages/*}) bwc-test.dir(messages, groovy.lang.Closure) bwc-test.deleteDir() - bwc-test.publishNotification({icon=:white_check_mark:, message=BWC Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK, manifest=tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + bwc-test.publishNotification({icon=:white_check_mark:, message=BWC Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=dummy_job diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt index 58e851f954..403028d319 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile.txt @@ -5,11 +5,11 @@ integ-test.timeout({time=3, unit=HOURS}) integ-test.echo(Executing on agent [label:none]) integ-test.stage(verify-parameters, groovy.lang.Closure) - integ-test.echo(Executing on agent [label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]) + integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) - integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -18,7 +18,7 @@ TestManifest.asBoolean() detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) - integ-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + integ-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/215/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -61,7 +61,7 @@ uploadTestResults.stash({includes=messages/*, name=messages-DUMMY_STAGE_NAME}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - integ-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([integ-test]) @@ -69,8 +69,8 @@ integ-test.findFiles({excludes=, glob=messages/*}) integ-test.dir(messages, groovy.lang.Closure) integ-test.deleteDir() - integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK, manifest=tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=dummy_job diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt index b60d26a3fe..aec9576e71 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/bwc-test.jenkinsfile.txt @@ -5,11 +5,11 @@ bwc-test.timeout({time=3, unit=HOURS}) bwc-test.echo(Executing on agent [label:none]) bwc-test.stage(verify-parameters, groovy.lang.Closure) - bwc-test.echo(Executing on agent [label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]) + bwc-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) bwc-test.script(groovy.lang.Closure) bwc-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) bwc-test.stage(detect docker image + args, groovy.lang.Closure) - bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) bwc-test.script(groovy.lang.Closure) bwc-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -18,7 +18,7 @@ TestManifest.asBoolean() detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) bwc-test.stage(bwc-test, groovy.lang.Closure) - bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + bwc-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:-e JAVA_HOME=/opt/java/openjdk-11, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) bwc-test.script(groovy.lang.Closure) bwc-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -59,7 +59,7 @@ uploadTestResults.stash({includes=messages/*, name=messages-DUMMY_STAGE_NAME}) bwc-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - bwc-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + bwc-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) bwc-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([bwc-test]) @@ -67,8 +67,8 @@ bwc-test.findFiles({excludes=, glob=messages/*}) bwc-test.dir(messages, groovy.lang.Closure) bwc-test.deleteDir() - bwc-test.publishNotification({icon=:white_check_mark:, message=BWC Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK, manifest=tests/jenkins/data/opensearch-1.3.0-test.yml}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + bwc-test.publishNotification({icon=:white_check_mark:, message=BWC Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-1.3.0-test.yml}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=dummy_job diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt index 6a60804c17..7e45ef1ccd 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/integ-test.jenkinsfile.txt @@ -5,11 +5,11 @@ integ-test.timeout({time=3, unit=HOURS}) integ-test.echo(Executing on agent [label:none]) integ-test.stage(verify-parameters, groovy.lang.Closure) - integ-test.echo(Executing on agent [label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]) + integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) integ-test.fileExists(manifests/tests/jenkins/data/opensearch-1.3.0-test.yml) integ-test.stage(detect docker image + args, groovy.lang.Closure) - integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + integ-test.echo(Executing on agent [docker:[image:alpine:3, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) integ-test.script(groovy.lang.Closure) integ-test.detectTestDockerAgent() detectTestDockerAgent.legacySCM(groovy.lang.Closure) @@ -18,7 +18,7 @@ TestManifest.asBoolean() detectTestDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) integ-test.stage(integ-test, groovy.lang.Closure) - integ-test.echo(Executing on agent [label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]) + integ-test.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) integ-test.script(groovy.lang.Closure) integ-test.downloadBuildManifest({url=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/717/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, path=tests/jenkins/data/opensearch-1.3.0-build.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -34,7 +34,7 @@ integ-test.echo(Add Component: OpenSearch) integ-test.parallel({Run Integtest OpenSearch=groovy.lang.Closure}) integ-test.timeout({time=2, unit=HOURS}, groovy.lang.Closure) - integ-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.echo(Component Name: OpenSearch) integ-test.unstash(opensearch-build-repo) integ-test.sh(rm -rf test-results) @@ -71,7 +71,7 @@ postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) integ-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - integ-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + integ-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) integ-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([integ-test]) @@ -79,8 +79,8 @@ integ-test.findFiles({excludes=, glob=messages/*}) integ-test.dir(messages, groovy.lang.Closure) integ-test.deleteDir() - integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK, manifest=tests/jenkins/data/opensearch-1.3.0-test.yml}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + integ-test.publishNotification({icon=:white_check_mark:, message=Integration Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook, manifest=tests/jenkins/data/opensearch-1.3.0-test.yml}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=dummy_job diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 83754a284b..ea228457a7 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -4,8 +4,18 @@ perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) perf-test.echo(Executing on agent [label:none]) + perf-test.parameterizedCron( + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + ) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -21,7 +31,7 @@ perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.stage(test-with-security, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -42,8 +52,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) @@ -81,7 +94,7 @@ Performance tests with security for 1236 completed}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) perf-test.stage(test-without-security, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -102,8 +115,11 @@ Performance tests with security for 1236 completed}) pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) @@ -140,7 +156,7 @@ Performance tests without security for 1236 completed}) uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) @@ -148,8 +164,8 @@ Performance tests without security for 1236 completed}) perf-test.findFiles({excludes=, glob=messages/*}) perf-test.dir(messages, groovy.lang.Closure) perf-test.deleteDir() - perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=perf-test diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 514d265b0b..5aaa8bccd1 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -4,8 +4,18 @@ perf-test.pipeline(groovy.lang.Closure) perf-test.timeout({time=15, unit=HOURS}) perf-test.echo(Executing on agent [label:none]) + perf-test.parameterizedCron( + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 1 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 17 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.4/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=3;WARMUP_ITERATIONS=2 + H 9 * * 3 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + H 9 * * 5 %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.1.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_ITERATIONS=250;WARMUP_ITERATIONS=0 + ) perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -22,7 +32,7 @@ perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.echo(Skipping stage test-with-security) perf-test.stage(test-without-security, groovy.lang.Closure) - perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) @@ -43,8 +53,11 @@ pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runPerfTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runPerfTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=ARTIFACT_BUCKET_NAME, path=test_config/config.yml, force=true}) runPerfTestScript.usernamePassword({credentialsId=bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) runPerfTestScript.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64-perf-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1 ) @@ -81,7 +94,7 @@ Performance tests without security for 1236 completed}) uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.node(Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host, groovy.lang.Closure) perf-test.script(groovy.lang.Closure) Messages.asBoolean() Messages.get([perf-test]) @@ -89,8 +102,8 @@ Performance tests without security for 1236 completed}) perf-test.findFiles({excludes=, glob=messages/*}) perf-test.dir(messages, groovy.lang.Closure) perf-test.deleteDir() - perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) - publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=jenkins-integ-test-webhook}) + publishNotification.string({credentialsId=jenkins-integ-test-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=perf-test diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDocker.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDocker.jenkinsfile.txt new file mode 100644 index 0000000000..27bed9474a --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDocker.jenkinsfile.txt @@ -0,0 +1,32 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerECRLatestMajor.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerECRLatestMajor.jenkinsfile.txt new file mode 100644 index 0000000000..20188ec551 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerECRLatestMajor.jenkinsfile.txt @@ -0,0 +1,107 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=true, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=true, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=true, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatest.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatest.jenkinsfile.txt new file mode 100644 index 0000000000..72461fb19c --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatest.jenkinsfile.txt @@ -0,0 +1,47 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=false}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatestMajor.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatestMajor.jenkinsfile.txt new file mode 100644 index 0000000000..b0aff85140 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerLatestMajor.jenkinsfile.txt @@ -0,0 +1,62 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerMajor.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerMajor.jenkinsfile.txt new file mode 100644 index 0000000000..fa3dbdb931 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToDockerMajor.jenkinsfile.txt @@ -0,0 +1,47 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=true, ecrPromote=false, latestTag=false, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToECRLatestMajor.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToECRLatestMajor.jenkinsfile.txt new file mode 100644 index 0000000000..3e740131c8 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/promotion/promote-container/promote-container-testPromoteContainerToECRLatestMajor.jenkinsfile.txt @@ -0,0 +1,62 @@ + promote-docker-ecr.run() + promote-docker-ecr.legacySCM(groovy.lang.Closure) + promote-docker-ecr.library({identifier=jenkins@20211123, retriever=null}) + promote-docker-ecr.pipeline(groovy.lang.Closure) + promote-docker-ecr.timeout({time=1, unit=HOURS}) + promote-docker-ecr.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ubuntu2004-x64-docker-buildx0.6.3-qemu5.0-awscli1.22-jdk11-v2, reuseNode:false, stages:[:], args:-u root -v /var/run/docker.sock:/var/run/docker.sock, alwaysPull:false, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) + promote-docker-ecr.stage(Parameters Check, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.stage(image-promote-to-prod, groovy.lang.Closure) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.promoteContainer({imageRepository=opensearch:2.0.1.2901, version=2.0.1, dockerPromote=false, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch:2.0.1.2901}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=opensearch-dashboards:2.0.1-2345, version=2.0.1, dockerPromote=false, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=opensearchstaging}) + promoteContainer.string({name=SOURCE_IMAGE, value=opensearch-dashboards:2.0.1-2345}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=opensearch-dashboards:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.promoteContainer({imageRepository=data-prepper:2.0.1.123, version=2.0.1, dockerPromote=false, ecrPromote=true, latestTag=true, majorVersionTag=true}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2.0.1}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:2}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promoteContainer.string({name=SOURCE_IMAGE_REGISTRY, value=dummy_dataprepper_ecr_url}) + promoteContainer.string({name=SOURCE_IMAGE, value=data-prepper:2.0.1.123}) + promoteContainer.string({name=DESTINATION_IMAGE_REGISTRY, value=public.ecr.aws/opensearchproject}) + promoteContainer.string({name=DESTINATION_IMAGE, value=data-prepper:latest}) + promoteContainer.build({job=docker-copy, parameters=[null, null, null, null]}) + promote-docker-ecr.script(groovy.lang.Closure) + promote-docker-ecr.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + promote-docker-ecr.sh(docker logout) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag-dashboards.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag-dashboards.jenkinsfile.txt index 3c3235566b..22d55cfe99 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag-dashboards.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag-dashboards.jenkinsfile.txt @@ -3,7 +3,7 @@ release-tag.library({identifier=jenkins@20211123, retriever=null}) release-tag.pipeline(groovy.lang.Closure) release-tag.timeout({time=2, unit=HOURS}) - release-tag.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + release-tag.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) release-tag.stage(Create Release Tag, groovy.lang.Closure) release-tag.script(groovy.lang.Closure) release-tag.sh(curl -SLO https://artifacts.opensearch.org/releases/bundle/opensearch-dashboards/2.0.0-rc1/opensearch-dashboards-2.0.0-rc1-linux-x64.tar.gz && tar -xf opensearch-dashboards-2.0.0-rc1-linux-x64.tar.gz opensearch-dashboards-2.0.0-rc1/manifest.yml) @@ -12,20 +12,20 @@ createReleaseTag.legacySCM(groovy.lang.Closure) createReleaseTag.library({identifier=jenkins@20211123, retriever=null}) createReleaseTag.readYaml({file=tests/jenkins/data/opensearch-dashboards-bundle-2.0.0-rc1.yml}) - BuildManifest.asBoolean() - BuildManifest.getNames() + BundleManifest.asBoolean() + BundleManifest.getNames() createReleaseTag.echo(Creating 2.0.0-rc1 release tag for 11 components in the manifest) createReleaseTag.usernamePassword({credentialsId=dummy_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) createReleaseTag.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - BuildManifest.getCommitId(ganttChartDashboards) - BuildManifest.getRepo(ganttChartDashboards) + BundleManifest.getCommitId(ganttChartDashboards) + BundleManifest.getRepo(ganttChartDashboards) createReleaseTag.echo(Tagging ganttChartDashboards at 69c54344eebb48d56deca823732954c27a10211c ...) createReleaseTag.dir(ganttChartDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=69c54344eebb48d56deca823732954c27a10211c}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-visualizations.git}]}) createReleaseTag.sh({script=git ls-remote --tags https://github.com/opensearch-project/dashboards-visualizations.git 2.0.0.0-rc1 | awk 'NR==1{print $1}', returnStdout=true}) createReleaseTag.echo(Tag 2.0.0.0-rc1 has been created with identical commit ID. Skipping creating new tag for ganttChartDashboards.) - BuildManifest.getCommitId(indexManagementDashboards) - BuildManifest.getRepo(indexManagementDashboards) + BundleManifest.getCommitId(indexManagementDashboards) + BundleManifest.getRepo(indexManagementDashboards) createReleaseTag.echo(Tagging indexManagementDashboards at a39d9d307e9fbbaf6f23aefe794aa23022a68fa0 ...) createReleaseTag.dir(indexManagementDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=a39d9d307e9fbbaf6f23aefe794aa23022a68fa0}], userRemoteConfigs=[{url=https://github.com/opensearch-project/index-management-dashboards-plugin.git}]}) @@ -34,8 +34,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/index-management-dashboards-plugin.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/index-management-dashboards-plugin.git 2.0.0.0-rc1) - BuildManifest.getCommitId(anomalyDetectionDashboards) - BuildManifest.getRepo(anomalyDetectionDashboards) + BundleManifest.getCommitId(anomalyDetectionDashboards) + BundleManifest.getRepo(anomalyDetectionDashboards) createReleaseTag.echo(Tagging anomalyDetectionDashboards at 3324c01f66ec9919cc6d69420c37f5687312f9c4 ...) createReleaseTag.dir(anomalyDetectionDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=3324c01f66ec9919cc6d69420c37f5687312f9c4}], userRemoteConfigs=[{url=https://github.com/opensearch-project/anomaly-detection-dashboards-plugin}]}) @@ -44,8 +44,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection-dashboards-plugin 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection-dashboards-plugin 2.0.0.0-rc1) - BuildManifest.getCommitId(OpenSearch-Dashboards) - BuildManifest.getRepo(OpenSearch-Dashboards) + BundleManifest.getCommitId(OpenSearch-Dashboards) + BundleManifest.getRepo(OpenSearch-Dashboards) createReleaseTag.echo(Tagging OpenSearch-Dashboards at 23ee797c0542271a36746e0beea33833cd7396ba ...) createReleaseTag.dir(OpenSearch-Dashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=23ee797c0542271a36746e0beea33833cd7396ba}], userRemoteConfigs=[{url=https://github.com/opensearch-project/OpenSearch-Dashboards.git}]}) @@ -54,8 +54,8 @@ createReleaseTag.sh(git tag 2.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch-Dashboards.git 2.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch-Dashboards.git 2.0.0-rc1) - BuildManifest.getCommitId(notificationsDashboards) - BuildManifest.getRepo(notificationsDashboards) + BundleManifest.getCommitId(notificationsDashboards) + BundleManifest.getRepo(notificationsDashboards) createReleaseTag.echo(Tagging notificationsDashboards at eb3af31759668a94727950d081e8a3a161f22918 ...) createReleaseTag.dir(notificationsDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=eb3af31759668a94727950d081e8a3a161f22918}], userRemoteConfigs=[{url=https://github.com/opensearch-project/notifications.git}]}) @@ -64,8 +64,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 2.0.0.0-rc1) - BuildManifest.getCommitId(securityDashboards) - BuildManifest.getRepo(securityDashboards) + BundleManifest.getCommitId(securityDashboards) + BundleManifest.getRepo(securityDashboards) createReleaseTag.echo(Tagging securityDashboards at 7420eb2b5115ad29c40a49de9b5acb42dd26bdc7 ...) createReleaseTag.dir(securityDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=7420eb2b5115ad29c40a49de9b5acb42dd26bdc7}], userRemoteConfigs=[{url=https://github.com/opensearch-project/security-dashboards-plugin.git}]}) @@ -74,8 +74,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/security-dashboards-plugin.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/security-dashboards-plugin.git 2.0.0.0-rc1) - BuildManifest.getCommitId(functionalTestDashboards) - BuildManifest.getRepo(functionalTestDashboards) + BundleManifest.getCommitId(functionalTestDashboards) + BundleManifest.getRepo(functionalTestDashboards) createReleaseTag.echo(Tagging functionalTestDashboards at ec2da5c44634f40dc97c3fdac4e85ff7139091b3 ...) createReleaseTag.dir(functionalTestDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=ec2da5c44634f40dc97c3fdac4e85ff7139091b3}], userRemoteConfigs=[{url=https://github.com/opensearch-project/opensearch-dashboards-functional-test.git}]}) @@ -84,8 +84,8 @@ createReleaseTag.sh(git tag 2.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/opensearch-dashboards-functional-test.git 2.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/opensearch-dashboards-functional-test.git 2.0.0-rc1) - BuildManifest.getCommitId(alertingDashboards) - BuildManifest.getRepo(alertingDashboards) + BundleManifest.getCommitId(alertingDashboards) + BundleManifest.getRepo(alertingDashboards) createReleaseTag.echo(Tagging alertingDashboards at de11dd89cbdd431f5073901a74907125f1133baf ...) createReleaseTag.dir(alertingDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=de11dd89cbdd431f5073901a74907125f1133baf}], userRemoteConfigs=[{url=https://github.com/opensearch-project/alerting-dashboards-plugin.git}]}) @@ -94,8 +94,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting-dashboards-plugin.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting-dashboards-plugin.git 2.0.0.0-rc1) - BuildManifest.getCommitId(queryWorkbenchDashboards) - BuildManifest.getRepo(queryWorkbenchDashboards) + BundleManifest.getCommitId(queryWorkbenchDashboards) + BundleManifest.getRepo(queryWorkbenchDashboards) createReleaseTag.echo(Tagging queryWorkbenchDashboards at eb65d2d956872cb13c7cca340fff679468b86074 ...) createReleaseTag.dir(queryWorkbenchDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=eb65d2d956872cb13c7cca340fff679468b86074}], userRemoteConfigs=[{url=https://github.com/opensearch-project/sql.git}]}) @@ -104,8 +104,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 2.0.0.0-rc1) - BuildManifest.getCommitId(reportsDashboards) - BuildManifest.getRepo(reportsDashboards) + BundleManifest.getCommitId(reportsDashboards) + BundleManifest.getRepo(reportsDashboards) createReleaseTag.echo(Tagging reportsDashboards at 478ea41b5d962bcf40964ffc7e3f1131f7e1fc34 ...) createReleaseTag.dir(reportsDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=478ea41b5d962bcf40964ffc7e3f1131f7e1fc34}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-reports.git}]}) @@ -114,8 +114,8 @@ createReleaseTag.sh(git tag 2.0.0.0-rc1) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 2.0.0.0-rc1, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 2.0.0.0-rc1) - BuildManifest.getCommitId(observabilityDashboards) - BuildManifest.getRepo(observabilityDashboards) + BundleManifest.getCommitId(observabilityDashboards) + BundleManifest.getRepo(observabilityDashboards) createReleaseTag.echo(Tagging observabilityDashboards at 52ae188b9a38e96f1d666f51e56ed92998c1d745 ...) createReleaseTag.dir(observabilityDashboards, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=52ae188b9a38e96f1d666f51e56ed92998c1d745}], userRemoteConfigs=[{url=https://github.com/opensearch-project/observability.git}]}) diff --git a/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag.jenkinsfile.txt index 9041d6d34a..c6d39340e0 100644 --- a/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/release-tag/release-tag.jenkinsfile.txt @@ -3,7 +3,7 @@ release-tag.library({identifier=jenkins@20211123, retriever=null}) release-tag.pipeline(groovy.lang.Closure) release-tag.timeout({time=2, unit=HOURS}) - release-tag.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + release-tag.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211130, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) release-tag.stage(Create Release Tag, groovy.lang.Closure) release-tag.script(groovy.lang.Closure) release-tag.sh(curl -SLO https://artifacts.opensearch.org/releases/bundle/opensearch/1.1.0/opensearch-1.1.0-linux-x64.tar.gz && tar -xf opensearch-1.1.0-linux-x64.tar.gz opensearch-1.1.0/manifest.yml) @@ -12,20 +12,20 @@ createReleaseTag.legacySCM(groovy.lang.Closure) createReleaseTag.library({identifier=jenkins@20211123, retriever=null}) createReleaseTag.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) - BuildManifest.asBoolean() - BuildManifest.getNames() + BundleManifest.asBoolean() + BundleManifest.getNames() createReleaseTag.echo(Creating 1.1.0 release tag for 15 components in the manifest) createReleaseTag.usernamePassword({credentialsId=dummy_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) createReleaseTag.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - BuildManifest.getCommitId(index-management) - BuildManifest.getRepo(index-management) + BundleManifest.getCommitId(index-management) + BundleManifest.getRepo(index-management) createReleaseTag.echo(Tagging index-management at 7897e9ae9cd5b49535e6a8bbf4c2f73cb458af24 ...) createReleaseTag.dir(index-management, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=7897e9ae9cd5b49535e6a8bbf4c2f73cb458af24}], userRemoteConfigs=[{url=https://github.com/opensearch-project/index-management.git}]}) createReleaseTag.sh({script=git ls-remote --tags https://github.com/opensearch-project/index-management.git 1.1.0.0 | awk 'NR==1{print $1}', returnStdout=true}) createReleaseTag.echo(Tag 1.1.0.0 has been created with identical commit ID. Skipping creating new tag for index-management.) - BuildManifest.getCommitId(job-scheduler) - BuildManifest.getRepo(job-scheduler) + BundleManifest.getCommitId(job-scheduler) + BundleManifest.getRepo(job-scheduler) createReleaseTag.echo(Tagging job-scheduler at 4504dabfc67dd5628c1451e91e9a1c3c4ca71525 ...) createReleaseTag.dir(job-scheduler, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=4504dabfc67dd5628c1451e91e9a1c3c4ca71525}], userRemoteConfigs=[{url=https://github.com/opensearch-project/job-scheduler.git}]}) @@ -34,8 +34,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/job-scheduler.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/job-scheduler.git 1.1.0.0) - BuildManifest.getCommitId(anomaly-detection) - BuildManifest.getRepo(anomaly-detection) + BundleManifest.getCommitId(anomaly-detection) + BundleManifest.getRepo(anomaly-detection) createReleaseTag.echo(Tagging anomaly-detection at bedc5b620384163abe272e913705fa23cfd3b3a3 ...) createReleaseTag.dir(anomaly-detection, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=bedc5b620384163abe272e913705fa23cfd3b3a3}], userRemoteConfigs=[{url=https://github.com/opensearch-project/anomaly-detection.git}]}) @@ -44,8 +44,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection.git 1.1.0.0) - BuildManifest.getCommitId(performance-analyzer) - BuildManifest.getRepo(performance-analyzer) + BundleManifest.getCommitId(performance-analyzer) + BundleManifest.getRepo(performance-analyzer) createReleaseTag.echo(Tagging performance-analyzer at f184f0bc39302ac38af2585c663d619048f6cffe ...) createReleaseTag.dir(performance-analyzer, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=f184f0bc39302ac38af2585c663d619048f6cffe}], userRemoteConfigs=[{url=https://github.com/opensearch-project/performance-analyzer.git}]}) @@ -54,8 +54,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/performance-analyzer.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/performance-analyzer.git 1.1.0.0) - BuildManifest.getCommitId(common-utils) - BuildManifest.getRepo(common-utils) + BundleManifest.getCommitId(common-utils) + BundleManifest.getRepo(common-utils) createReleaseTag.echo(Tagging common-utils at 3913d7097934cbfe1fdcf919347f22a597d00b76 ...) createReleaseTag.dir(common-utils, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=3913d7097934cbfe1fdcf919347f22a597d00b76}], userRemoteConfigs=[{url=https://github.com/opensearch-project/common-utils.git}]}) @@ -64,8 +64,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/common-utils.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/common-utils.git 1.1.0.0) - BuildManifest.getCommitId(asynchronous-search) - BuildManifest.getRepo(asynchronous-search) + BundleManifest.getCommitId(asynchronous-search) + BundleManifest.getRepo(asynchronous-search) createReleaseTag.echo(Tagging asynchronous-search at aa344cc1ecdf9ad21d6cb8d9e368361e6a0f3132 ...) createReleaseTag.dir(asynchronous-search, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=aa344cc1ecdf9ad21d6cb8d9e368361e6a0f3132}], userRemoteConfigs=[{url=https://github.com/opensearch-project/asynchronous-search.git}]}) @@ -74,8 +74,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/asynchronous-search.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/asynchronous-search.git 1.1.0.0) - BuildManifest.getCommitId(dashboards-notebooks) - BuildManifest.getRepo(dashboards-notebooks) + BundleManifest.getCommitId(dashboards-notebooks) + BundleManifest.getRepo(dashboards-notebooks) createReleaseTag.echo(Tagging dashboards-notebooks at 5a996eb8ec8c92e7469cd5e5f87b237352f60f61 ...) createReleaseTag.dir(dashboards-notebooks, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=5a996eb8ec8c92e7469cd5e5f87b237352f60f61}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-notebooks.git}]}) @@ -84,8 +84,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-notebooks.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-notebooks.git 1.1.0.0) - BuildManifest.getCommitId(OpenSearch) - BuildManifest.getRepo(OpenSearch) + BundleManifest.getCommitId(OpenSearch) + BundleManifest.getRepo(OpenSearch) createReleaseTag.echo(Tagging OpenSearch at b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583 ...) createReleaseTag.dir(OpenSearch, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583}], userRemoteConfigs=[{url=https://github.com/opensearch-project/OpenSearch.git}]}) @@ -94,8 +94,8 @@ createReleaseTag.sh(git tag 1.1.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 1.1.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 1.1.0) - BuildManifest.getCommitId(sql) - BuildManifest.getRepo(sql) + BundleManifest.getCommitId(sql) + BundleManifest.getRepo(sql) createReleaseTag.echo(Tagging sql at d68547d585092af1e053d01e1b834259723cd304 ...) createReleaseTag.dir(sql, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=d68547d585092af1e053d01e1b834259723cd304}], userRemoteConfigs=[{url=https://github.com/opensearch-project/sql.git}]}) @@ -104,8 +104,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 1.1.0.0) - BuildManifest.getCommitId(alerting) - BuildManifest.getRepo(alerting) + BundleManifest.getCommitId(alerting) + BundleManifest.getRepo(alerting) createReleaseTag.echo(Tagging alerting at 8024b8b9195f837e49e5bebd7f4a31dfc333eb4d ...) createReleaseTag.dir(alerting, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=8024b8b9195f837e49e5bebd7f4a31dfc333eb4d}], userRemoteConfigs=[{url=https://github.com/opensearch-project/alerting.git}]}) @@ -114,8 +114,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting.git 1.1.0.0) - BuildManifest.getCommitId(security) - BuildManifest.getRepo(security) + BundleManifest.getCommitId(security) + BundleManifest.getRepo(security) createReleaseTag.echo(Tagging security at 534fffe0e6cf2b33b9abcbc6508e98fc2d077a3d ...) createReleaseTag.dir(security, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=534fffe0e6cf2b33b9abcbc6508e98fc2d077a3d}], userRemoteConfigs=[{url=https://github.com/opensearch-project/security.git}]}) @@ -124,8 +124,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/security.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/security.git 1.1.0.0) - BuildManifest.getCommitId(k-NN) - BuildManifest.getRepo(k-NN) + BundleManifest.getCommitId(k-NN) + BundleManifest.getRepo(k-NN) createReleaseTag.echo(Tagging k-NN at 6a3fdcafd75e63521bcf7893ce908642e1d9fcb6 ...) createReleaseTag.dir(k-NN, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=6a3fdcafd75e63521bcf7893ce908642e1d9fcb6}], userRemoteConfigs=[{url=https://github.com/opensearch-project/k-NN.git}]}) @@ -134,8 +134,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/k-NN.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/k-NN.git 1.1.0.0) - BuildManifest.getCommitId(dashboards-reports) - BuildManifest.getRepo(dashboards-reports) + BundleManifest.getCommitId(dashboards-reports) + BundleManifest.getRepo(dashboards-reports) createReleaseTag.echo(Tagging dashboards-reports at 622f334b0724e47f7ffd21cf7e7d521a9f6c949e ...) createReleaseTag.dir(dashboards-reports, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=622f334b0724e47f7ffd21cf7e7d521a9f6c949e}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-reports.git}]}) @@ -144,8 +144,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 1.1.0.0) - BuildManifest.getCommitId(notifications) - BuildManifest.getRepo(notifications) + BundleManifest.getCommitId(notifications) + BundleManifest.getRepo(notifications) createReleaseTag.echo(Tagging notifications at d0d3e485c4a850f73652a989eeec795b7347fbb6 ...) createReleaseTag.dir(notifications, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=d0d3e485c4a850f73652a989eeec795b7347fbb6}], userRemoteConfigs=[{url=https://github.com/opensearch-project/notifications.git}]}) @@ -154,8 +154,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 1.1.0.0) - BuildManifest.getCommitId(performance-analyzer-rca) - BuildManifest.getRepo(performance-analyzer-rca) + BundleManifest.getCommitId(performance-analyzer-rca) + BundleManifest.getRepo(performance-analyzer-rca) createReleaseTag.echo(Tagging performance-analyzer-rca at 345a10fd4f4e94d6392c925ad95503ba8addd152 ...) createReleaseTag.dir(performance-analyzer-rca, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=345a10fd4f4e94d6392c925ad95503ba8addd152}], userRemoteConfigs=[{url=https://github.com/opensearch-project/performance-analyzer-rca.git}]}) diff --git a/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt index 1eec75e275..2177fedf77 100644 --- a/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts/sign-standalone-artifacts.jenkinsfile.txt @@ -2,29 +2,76 @@ sign-standalone-artifacts.legacySCM(groovy.lang.Closure) sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null}) sign-standalone-artifacts.pipeline(groovy.lang.Closure) + sign-standalone-artifacts.credentials(jenkins-artifact-bucket-name) +<<<<<<< HEAD +<<<<<<< HEAD + sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) +======= sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) +>>>>>>> 8517abc (Move artifact bucket name from env to creds (#2327)) +======= + sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-build-v2, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) +>>>>>>> 09366a4 (Make agent label have consistent capital naming (#2353)) sign-standalone-artifacts.stage(sign, groovy.lang.Closure) sign-standalone-artifacts.script(groovy.lang.Closure) sign-standalone-artifacts.sh(mkdir /tmp/workspace/artifacts) sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_1_artifact.tar.gz) sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o /tmp/workspace/artifacts/dummy_2_artifact.tar.gz) sign-standalone-artifacts.signArtifacts({artifactPath=/tmp/workspace/artifacts, sigtype=.sig, platform=linux}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux - ) +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux + ) sign-standalone-artifacts.uploadToS3({sourcePath=/tmp/workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed}) uploadToS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) uploadToS3.withCredentials([AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) diff --git a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt index dab364ab42..6909c43d72 100644 --- a/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt @@ -33,7 +33,8 @@ uploadArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) - uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT], groovy.lang.Closure) + uploadArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PROMOTION_ROLE_NAME], groovy.lang.Closure) uploadArtifacts.echo(Uploading to s3://ARTIFACT_BUCKET_NAME/vars-build/1.3.0/33/linux/x64/tar) uploadArtifacts.uploadToS3({sourcePath=tar/builds, bucket=ARTIFACT_BUCKET_NAME, path=vars-build/1.3.0/33/linux/x64/tar/builds}) uploadToS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -46,7 +47,7 @@ uploadToS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadToS3.s3Upload({file=tar/dist, bucket=ARTIFACT_BUCKET_NAME, path=vars-build/1.3.0/33/linux/x64/tar/dist}) uploadArtifacts.echo(Uploading to s3://ARTIFACT_PRODUCTION_BUCKET_NAME/vars-build/1.3.0/33/linux/x64/tar) - uploadArtifacts.withAWS({role=role, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadArtifacts.s3Upload({file=tar/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=release-candidates/core/opensearch/1.3.0/}) uploadArtifacts.s3Upload({file=tar/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=release-candidates/bundle/opensearch/1.3.0/}) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) diff --git a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt index 4490d61a10..d6f1a60de6 100644 --- a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt +++ b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt @@ -13,11 +13,13 @@ BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 123) assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.3.0-rpm.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64) assembleManifest.signArtifacts({artifactPath=rpm/dist/opensearch, sigtype=.rpm, platform=linux}) - signArtifacts.echo(RPM Add Sign) - signArtifacts.withAWS({role=sign_asm_role, roleAccount=sign_asm_account, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) - signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-pass-id, variable=SIGNING_PASS_ID}) - signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-secret-id, variable=SIGNING_SECRET_ID}) - signArtifacts.withCredentials([SIGNING_PASS_ID, SIGNING_SECRET_ID], groovy.lang.Closure) + signArtifacts.string({credentialsId=jenkins-rpm-signing-account-number, variable=RPM_SIGNING_ACCOUNT_NUMBER}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-passphrase-secrets-arn, variable=RPM_SIGNING_PASSPHRASE_SECRETS_ARN}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-secret-key-secrets-arn, variable=RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-key-id, variable=RPM_SIGNING_KEY_ID}) + signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure) + signArtifacts.echo(RPM Add Sign) + signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) signArtifacts.sh( set -e set +x @@ -57,8 +59,8 @@ echo "------------------------------------------------------------------------" echo "Import OpenSearch keys" - aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_PASS_ID" | jq -r .SecretBinary | base64 --decode > passphrase - aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_SECRET_ID" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_PASSPHRASE_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode > passphrase + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - echo "------------------------------------------------------------------------" echo "Start Signing Rpm" @@ -85,8 +87,8 @@ echo "------------------------------------------------------------------------" echo "Clean up gpg" - gpg --batch --yes --delete-secret-keys sign_asm_keyid - gpg --batch --yes --delete-keys sign_asm_keyid + gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID + gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID rm -v passphrase ) diff --git a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt index 5a396f87ca..0870027038 100644 --- a/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt @@ -23,7 +23,8 @@ uploadArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) - uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT], groovy.lang.Closure) + uploadArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PROMOTION_ROLE_NAME], groovy.lang.Closure) uploadArtifacts.echo(Uploading to s3://ARTIFACT_BUCKET_NAME/vars-build/1.1.0/33/linux/x64/tar) uploadArtifacts.uploadToS3({sourcePath=tar/builds, bucket=ARTIFACT_BUCKET_NAME, path=vars-build/1.1.0/33/linux/x64/tar/builds}) uploadToS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -36,7 +37,7 @@ uploadToS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadToS3.s3Upload({file=tar/dist, bucket=ARTIFACT_BUCKET_NAME, path=vars-build/1.1.0/33/linux/x64/tar/dist}) uploadArtifacts.echo(Uploading to s3://ARTIFACT_PRODUCTION_BUCKET_NAME/vars-build/1.1.0/33/linux/x64/tar) - uploadArtifacts.withAWS({role=role, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) uploadArtifacts.s3Upload({file=tar/builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=release-candidates/core/opensearch/1.1.0/}) uploadArtifacts.s3Upload({file=tar/dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=release-candidates/bundle/opensearch/1.1.0/}) BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33) diff --git a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt index 72dbc4aa22..f00e051369 100644 --- a/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt @@ -32,7 +32,8 @@ uploadArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) uploadArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) - uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT], groovy.lang.Closure) + uploadArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + uploadArtifacts.withCredentials([ARTIFACT_BUCKET_NAME, ARTIFACT_PRODUCTION_BUCKET_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PROMOTION_ROLE_NAME], groovy.lang.Closure) uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.3.0/33/linux/x64/tar) uploadArtifacts.uploadToS3({sourcePath=tar/builds, bucket=artifact-bucket, path=vars-build/1.3.0/33/linux/x64/tar/builds}) uploadToS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) diff --git a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt index bffbe238c1..c163c31312 100644 --- a/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BuildUploadManifestSHA_Jenkinsfile.txt @@ -18,8 +18,14 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: false) - buildUploadManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - buildUploadManifestSHA.s3Upload({bucket=artifact-bucket, file=tests/jenkins/data/opensearch-1.3.0.yml.lock, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + buildUploadManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + buildUploadManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + buildUploadManifestSHA.withCredentials([ARTIFACT_BUCKET_NAME, AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) + buildUploadManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + buildUploadManifestSHA.s3Upload({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/data/opensearch-1.3.0.yml.lock, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) diff --git a/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile b/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile index 61fd1c0786..65599680cf 100644 --- a/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile +++ b/tests/jenkins/jobs/Build_OpenSearch_Dashboards_Jenkinsfile @@ -18,7 +18,7 @@ pipeline { stage('detect docker image + args') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028' alwaysPull true } @@ -35,7 +35,7 @@ pipeline { stage('build-linux-x64') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image dockerAgent.image args dockerAgent.args alwaysPull true @@ -67,7 +67,7 @@ pipeline { stage('build-archive-linux-arm64') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image dockerAgent.image args dockerAgent.args alwaysPull true @@ -92,7 +92,7 @@ pipeline { stage('assemble-archive-linux-arm64') { agent { docker { - label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' image dockerAgent.image args dockerAgent.args alwaysPull true @@ -124,7 +124,7 @@ pipeline { } stage('docker build') { steps { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { buildDockerImage( inputManifest: "manifests/${INPUT_MANIFEST}", @@ -138,7 +138,7 @@ pipeline { } post { success { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { def stashed = lib.jenkins.Messages.new(this).get([ 'build-linux-x64', @@ -153,7 +153,7 @@ pipeline { } } failure { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { echo 'FAILURE' diff --git a/tests/jenkins/jobs/CreateReleaseTag_Jenkinsfile.txt b/tests/jenkins/jobs/CreateReleaseTag_Jenkinsfile.txt index 6f87b91eb6..cbf5415cc4 100644 --- a/tests/jenkins/jobs/CreateReleaseTag_Jenkinsfile.txt +++ b/tests/jenkins/jobs/CreateReleaseTag_Jenkinsfile.txt @@ -7,20 +7,20 @@ createReleaseTag.legacySCM(groovy.lang.Closure) createReleaseTag.library({identifier=jenkins@20211123, retriever=null}) createReleaseTag.readYaml({file=tests/data/opensearch-build-1.1.0.yml}) - BuildManifest.asBoolean() - BuildManifest.getNames() + BundleManifest.asBoolean() + BundleManifest.getNames() createReleaseTag.echo(Creating 1.1.0 release tag for 15 components in the manifest) createReleaseTag.usernamePassword({credentialsId=dummy_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) createReleaseTag.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - BuildManifest.getCommitId(index-management) - BuildManifest.getRepo(index-management) + BundleManifest.getCommitId(index-management) + BundleManifest.getRepo(index-management) createReleaseTag.echo(Tagging index-management at 7897e9ae9cd5b49535e6a8bbf4c2f73cb458af24 ...) createReleaseTag.dir(index-management, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=7897e9ae9cd5b49535e6a8bbf4c2f73cb458af24}], userRemoteConfigs=[{url=https://github.com/opensearch-project/index-management.git}]}) createReleaseTag.sh({script=git ls-remote --tags https://github.com/opensearch-project/index-management.git 1.1.0.0 | awk 'NR==1{print $1}', returnStdout=true}) createReleaseTag.echo(Tag 1.1.0.0 has been created with identical commit ID. Skipping creating new tag for index-management.) - BuildManifest.getCommitId(job-scheduler) - BuildManifest.getRepo(job-scheduler) + BundleManifest.getCommitId(job-scheduler) + BundleManifest.getRepo(job-scheduler) createReleaseTag.echo(Tagging job-scheduler at 4504dabfc67dd5628c1451e91e9a1c3c4ca71525 ...) createReleaseTag.dir(job-scheduler, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=4504dabfc67dd5628c1451e91e9a1c3c4ca71525}], userRemoteConfigs=[{url=https://github.com/opensearch-project/job-scheduler.git}]}) @@ -29,8 +29,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/job-scheduler.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/job-scheduler.git 1.1.0.0) - BuildManifest.getCommitId(anomaly-detection) - BuildManifest.getRepo(anomaly-detection) + BundleManifest.getCommitId(anomaly-detection) + BundleManifest.getRepo(anomaly-detection) createReleaseTag.echo(Tagging anomaly-detection at bedc5b620384163abe272e913705fa23cfd3b3a3 ...) createReleaseTag.dir(anomaly-detection, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=bedc5b620384163abe272e913705fa23cfd3b3a3}], userRemoteConfigs=[{url=https://github.com/opensearch-project/anomaly-detection.git}]}) @@ -39,8 +39,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/anomaly-detection.git 1.1.0.0) - BuildManifest.getCommitId(performance-analyzer) - BuildManifest.getRepo(performance-analyzer) + BundleManifest.getCommitId(performance-analyzer) + BundleManifest.getRepo(performance-analyzer) createReleaseTag.echo(Tagging performance-analyzer at f184f0bc39302ac38af2585c663d619048f6cffe ...) createReleaseTag.dir(performance-analyzer, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=f184f0bc39302ac38af2585c663d619048f6cffe}], userRemoteConfigs=[{url=https://github.com/opensearch-project/performance-analyzer.git}]}) @@ -49,8 +49,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/performance-analyzer.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/performance-analyzer.git 1.1.0.0) - BuildManifest.getCommitId(common-utils) - BuildManifest.getRepo(common-utils) + BundleManifest.getCommitId(common-utils) + BundleManifest.getRepo(common-utils) createReleaseTag.echo(Tagging common-utils at 3913d7097934cbfe1fdcf919347f22a597d00b76 ...) createReleaseTag.dir(common-utils, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=3913d7097934cbfe1fdcf919347f22a597d00b76}], userRemoteConfigs=[{url=https://github.com/opensearch-project/common-utils.git}]}) @@ -59,8 +59,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/common-utils.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/common-utils.git 1.1.0.0) - BuildManifest.getCommitId(asynchronous-search) - BuildManifest.getRepo(asynchronous-search) + BundleManifest.getCommitId(asynchronous-search) + BundleManifest.getRepo(asynchronous-search) createReleaseTag.echo(Tagging asynchronous-search at aa344cc1ecdf9ad21d6cb8d9e368361e6a0f3132 ...) createReleaseTag.dir(asynchronous-search, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=aa344cc1ecdf9ad21d6cb8d9e368361e6a0f3132}], userRemoteConfigs=[{url=https://github.com/opensearch-project/asynchronous-search.git}]}) @@ -69,8 +69,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/asynchronous-search.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/asynchronous-search.git 1.1.0.0) - BuildManifest.getCommitId(dashboards-notebooks) - BuildManifest.getRepo(dashboards-notebooks) + BundleManifest.getCommitId(dashboards-notebooks) + BundleManifest.getRepo(dashboards-notebooks) createReleaseTag.echo(Tagging dashboards-notebooks at 5a996eb8ec8c92e7469cd5e5f87b237352f60f61 ...) createReleaseTag.dir(dashboards-notebooks, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=5a996eb8ec8c92e7469cd5e5f87b237352f60f61}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-notebooks.git}]}) @@ -79,8 +79,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-notebooks.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-notebooks.git 1.1.0.0) - BuildManifest.getCommitId(OpenSearch) - BuildManifest.getRepo(OpenSearch) + BundleManifest.getCommitId(OpenSearch) + BundleManifest.getRepo(OpenSearch) createReleaseTag.echo(Tagging OpenSearch at b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583 ...) createReleaseTag.dir(OpenSearch, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=b7334f49d530ffd1a3f7bd0e5832b9b2a9caa583}], userRemoteConfigs=[{url=https://github.com/opensearch-project/OpenSearch.git}]}) @@ -89,8 +89,8 @@ createReleaseTag.sh(git tag 1.1.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 1.1.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/OpenSearch.git 1.1.0) - BuildManifest.getCommitId(sql) - BuildManifest.getRepo(sql) + BundleManifest.getCommitId(sql) + BundleManifest.getRepo(sql) createReleaseTag.echo(Tagging sql at d68547d585092af1e053d01e1b834259723cd304 ...) createReleaseTag.dir(sql, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=d68547d585092af1e053d01e1b834259723cd304}], userRemoteConfigs=[{url=https://github.com/opensearch-project/sql.git}]}) @@ -99,8 +99,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/sql.git 1.1.0.0) - BuildManifest.getCommitId(alerting) - BuildManifest.getRepo(alerting) + BundleManifest.getCommitId(alerting) + BundleManifest.getRepo(alerting) createReleaseTag.echo(Tagging alerting at 8024b8b9195f837e49e5bebd7f4a31dfc333eb4d ...) createReleaseTag.dir(alerting, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=8024b8b9195f837e49e5bebd7f4a31dfc333eb4d}], userRemoteConfigs=[{url=https://github.com/opensearch-project/alerting.git}]}) @@ -109,8 +109,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/alerting.git 1.1.0.0) - BuildManifest.getCommitId(security) - BuildManifest.getRepo(security) + BundleManifest.getCommitId(security) + BundleManifest.getRepo(security) createReleaseTag.echo(Tagging security at 534fffe0e6cf2b33b9abcbc6508e98fc2d077a3d ...) createReleaseTag.dir(security, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=534fffe0e6cf2b33b9abcbc6508e98fc2d077a3d}], userRemoteConfigs=[{url=https://github.com/opensearch-project/security.git}]}) @@ -119,8 +119,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/security.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/security.git 1.1.0.0) - BuildManifest.getCommitId(k-NN) - BuildManifest.getRepo(k-NN) + BundleManifest.getCommitId(k-NN) + BundleManifest.getRepo(k-NN) createReleaseTag.echo(Tagging k-NN at 6a3fdcafd75e63521bcf7893ce908642e1d9fcb6 ...) createReleaseTag.dir(k-NN, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=6a3fdcafd75e63521bcf7893ce908642e1d9fcb6}], userRemoteConfigs=[{url=https://github.com/opensearch-project/k-NN.git}]}) @@ -129,8 +129,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/k-NN.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/k-NN.git 1.1.0.0) - BuildManifest.getCommitId(dashboards-reports) - BuildManifest.getRepo(dashboards-reports) + BundleManifest.getCommitId(dashboards-reports) + BundleManifest.getRepo(dashboards-reports) createReleaseTag.echo(Tagging dashboards-reports at 622f334b0724e47f7ffd21cf7e7d521a9f6c949e ...) createReleaseTag.dir(dashboards-reports, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=622f334b0724e47f7ffd21cf7e7d521a9f6c949e}], userRemoteConfigs=[{url=https://github.com/opensearch-project/dashboards-reports.git}]}) @@ -139,8 +139,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/dashboards-reports.git 1.1.0.0) - BuildManifest.getCommitId(notifications) - BuildManifest.getRepo(notifications) + BundleManifest.getCommitId(notifications) + BundleManifest.getRepo(notifications) createReleaseTag.echo(Tagging notifications at d0d3e485c4a850f73652a989eeec795b7347fbb6 ...) createReleaseTag.dir(notifications, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=d0d3e485c4a850f73652a989eeec795b7347fbb6}], userRemoteConfigs=[{url=https://github.com/opensearch-project/notifications.git}]}) @@ -149,8 +149,8 @@ createReleaseTag.sh(git tag 1.1.0.0) createReleaseTag.sh({script=git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 1.1.0.0, returnStatus=true}) createReleaseTag.sh(git push https://GITHUB_TOKEN@github.com/opensearch-project/notifications.git 1.1.0.0) - BuildManifest.getCommitId(performance-analyzer-rca) - BuildManifest.getRepo(performance-analyzer-rca) + BundleManifest.getCommitId(performance-analyzer-rca) + BundleManifest.getRepo(performance-analyzer-rca) createReleaseTag.echo(Tagging performance-analyzer-rca at 345a10fd4f4e94d6392c925ad95503ba8addd152 ...) createReleaseTag.dir(performance-analyzer-rca, groovy.lang.Closure) createReleaseTag.checkout({$class=GitSCM, branches=[{name=345a10fd4f4e94d6392c925ad95503ba8addd152}], userRemoteConfigs=[{url=https://github.com/opensearch-project/performance-analyzer-rca.git}]}) diff --git a/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile b/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile index c72b088cfa..b3c6fe5ed3 100644 --- a/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile +++ b/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile @@ -6,10 +6,11 @@ pipeline { script { currentBuild.description = "1.1.0/opensearch-1.1.0.yml" dockerAgent = detectDockerAgent( - manifest: 'manifests/1.1.0/opensearch-1.1.0.yml' + manifest: 'manifests/templates/opensearch/1.x/os-template-1.1.0.yml' ) echo dockerAgent.image echo dockerAgent.args + echo dockerAgent.javaVersion } } } @@ -17,10 +18,11 @@ pipeline { steps { script { dockerAgent = detectDockerAgent( - manifest: 'manifests/1.2.0/opensearch-1.2.0.yml' + manifest: 'manifests/templates/opensearch/1.x/os-template-1.2.0.yml' ) echo dockerAgent.image echo dockerAgent.args + echo dockerAgent.javaVersion } } } @@ -28,10 +30,11 @@ pipeline { steps { script { dockerAgent = detectDockerAgent( - manifest: 'manifests/1.1.0/opensearch-dashboards-1.1.0.yml' + manifest: 'manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml' ) echo dockerAgent.image echo dockerAgent.args + echo dockerAgent.javaVersion } } } @@ -39,10 +42,11 @@ pipeline { steps { script { dockerAgent = detectDockerAgent( - manifest: 'manifests/1.2.0/opensearch-dashboards-1.2.0.yml' + manifest: 'legacy-manifests/1.2.0/opensearch-dashboards-1.2.0.yml' ) echo dockerAgent.image echo dockerAgent.args + echo dockerAgent.javaVersion } } } diff --git a/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile.txt b/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile.txt index 80b3e205a8..8b7a5111af 100644 --- a/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile.txt +++ b/tests/jenkins/jobs/DetectDockerImage_Jenkinsfile.txt @@ -3,41 +3,49 @@ DetectDockerImage_Jenkinsfile.echo(Executing on agent [label:none]) DetectDockerImage_Jenkinsfile.stage(opensearch-1.1.0, groovy.lang.Closure) DetectDockerImage_Jenkinsfile.script(groovy.lang.Closure) - DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/1.1.0/opensearch-1.1.0.yml}) + DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/templates/opensearch/1.x/os-template-1.1.0.yml}) detectDockerAgent.legacySCM(groovy.lang.Closure) detectDockerAgent.library({identifier=jenkins@20211123, retriever=null}) - detectDockerAgent.readYaml({file=manifests/1.1.0/opensearch-1.1.0.yml}) + detectDockerAgent.readYaml({file=manifests/templates/opensearch/1.x/os-template-1.1.0.yml}) InputManifest.asBoolean() detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:ci-runner-centos7-v1 (null)) + detectDockerAgent.echo(Using java version openjdk-17) DetectDockerImage_Jenkinsfile.echo(opensearchstaging/ci-runner:ci-runner-centos7-v1) DetectDockerImage_Jenkinsfile.echo(null) + DetectDockerImage_Jenkinsfile.echo(openjdk-17) DetectDockerImage_Jenkinsfile.stage(opensearch-1.2.0, groovy.lang.Closure) DetectDockerImage_Jenkinsfile.script(groovy.lang.Closure) - DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/1.2.0/opensearch-1.2.0.yml}) + DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/templates/opensearch/1.x/os-template-1.2.0.yml}) detectDockerAgent.legacySCM(groovy.lang.Closure) detectDockerAgent.library({identifier=jenkins@20211123, retriever=null}) - detectDockerAgent.readYaml({file=manifests/1.2.0/opensearch-1.2.0.yml}) + detectDockerAgent.readYaml({file=manifests/templates/opensearch/1.x/os-template-1.2.0.yml}) InputManifest.asBoolean() detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 (-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot)) + detectDockerAgent.echo(Using java version openjdk-14) DetectDockerImage_Jenkinsfile.echo(opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028) DetectDockerImage_Jenkinsfile.echo(-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot) + DetectDockerImage_Jenkinsfile.echo(openjdk-14) DetectDockerImage_Jenkinsfile.stage(opensearch-dashboards-1.1.0, groovy.lang.Closure) DetectDockerImage_Jenkinsfile.script(groovy.lang.Closure) - DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/1.1.0/opensearch-dashboards-1.1.0.yml}) + DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml}) detectDockerAgent.legacySCM(groovy.lang.Closure) detectDockerAgent.library({identifier=jenkins@20211123, retriever=null}) - detectDockerAgent.readYaml({file=manifests/1.1.0/opensearch-dashboards-1.1.0.yml}) + detectDockerAgent.readYaml({file=manifests/templates/opensearch-dashboards/1.x/osd-template-1.1.0.yml}) InputManifest.asBoolean() detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 (null)) + detectDockerAgent.echo(Using java version openjdk-17) DetectDockerImage_Jenkinsfile.echo(opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028) DetectDockerImage_Jenkinsfile.echo(null) + DetectDockerImage_Jenkinsfile.echo(openjdk-17) DetectDockerImage_Jenkinsfile.stage(opensearch-dashboards-1.2.0, groovy.lang.Closure) DetectDockerImage_Jenkinsfile.script(groovy.lang.Closure) - DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=manifests/1.2.0/opensearch-dashboards-1.2.0.yml}) + DetectDockerImage_Jenkinsfile.detectDockerAgent({manifest=legacy-manifests/1.2.0/opensearch-dashboards-1.2.0.yml}) detectDockerAgent.legacySCM(groovy.lang.Closure) detectDockerAgent.library({identifier=jenkins@20211123, retriever=null}) - detectDockerAgent.readYaml({file=manifests/1.2.0/opensearch-dashboards-1.2.0.yml}) + detectDockerAgent.readYaml({file=legacy-manifests/1.2.0/opensearch-dashboards-1.2.0.yml}) InputManifest.asBoolean() detectDockerAgent.echo(Using Docker image opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028 (null)) + detectDockerAgent.echo(Using java version openjdk-17) DetectDockerImage_Jenkinsfile.echo(opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028) DetectDockerImage_Jenkinsfile.echo(null) + DetectDockerImage_Jenkinsfile.echo(openjdk-17) diff --git a/tests/jenkins/jobs/DownloadFromS3_Jenkinsfile.txt b/tests/jenkins/jobs/DownloadFromS3_Jenkinsfile.txt index 992c7ff82c..d8eb6c1c83 100644 --- a/tests/jenkins/jobs/DownloadFromS3_Jenkinsfile.txt +++ b/tests/jenkins/jobs/DownloadFromS3_Jenkinsfile.txt @@ -4,5 +4,7 @@ DownloadFromS3_Jenkinsfile.stage(download, groovy.lang.Closure) DownloadFromS3_Jenkinsfile.script(groovy.lang.Closure) DownloadFromS3_Jenkinsfile.downloadFromS3({destPath=/tmp/src/path, bucket=dummy_bucket, path=/download/path, force=true}) - downloadFromS3.withAWS({role=Dummy_Download_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - downloadFromS3.s3Download({file=/tmp/src/path, bucket=dummy_bucket, path=/download/path, force=true}) + downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + downloadFromS3.withCredentials([AWS_ACCOUNT_PUBLIC], groovy.lang.Closure) + downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + downloadFromS3.s3Download({file=/tmp/src/path, bucket=dummy_bucket, path=/download/path, force=true}) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt index ac58fb01af..219eadbcc4 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_does_not_exist.txt @@ -15,8 +15,11 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: false) GetManifestSHA_Jenkinsfile.echo(sha: sha1) GetManifestSHA_Jenkinsfile.echo(exists: false) diff --git a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt index 81b97f07b5..5a18992ce9 100644 --- a/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt +++ b/tests/jenkins/jobs/GetManifestSHA_Jenkinsfile_exists.txt @@ -15,8 +15,11 @@ InputManifest.getSHAsRoot(get-manifest-sha-build) getManifestSHA.echo(Manifest lock: tests/jenkins/data/opensearch-1.3.0.yml.lock) getManifestSHA.echo(Manifest SHA path: get-manifest-sha-build/1.3.0/shas/sha1.yml) - getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - getManifestSHA.s3DoesObjectExist({bucket=artifact-bucket, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) + getManifestSHA.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + getManifestSHA.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + getManifestSHA.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + getManifestSHA.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + getManifestSHA.s3DoesObjectExist({bucket=ARTIFACT_BUCKET_NAME, path=get-manifest-sha-build/1.3.0/shas/sha1.yml}) getManifestSHA.echo(Manifest SHA exists: true) GetManifestSHA_Jenkinsfile.echo(sha: sha1) GetManifestSHA_Jenkinsfile.echo(exists: true) diff --git a/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile b/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile new file mode 100644 index 0000000000..f50c4200d2 --- /dev/null +++ b/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile @@ -0,0 +1,24 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('get repository commits') { + steps { + script { + getRepositoryCommit( + componentName: "", + inputManifest: "tests/jenkins/data/opensearch-2.0.0.yml", + outputFile: "commits.yml" + ) + } + } + } + } +} diff --git a/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile.txt b/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile.txt new file mode 100644 index 0000000000..c5fef559db --- /dev/null +++ b/tests/jenkins/jobs/GetRepositoryCommit_Jenkinsfile.txt @@ -0,0 +1,62 @@ + GetRepositoryCommit_Jenkinsfile.run() + GetRepositoryCommit_Jenkinsfile.pipeline(groovy.lang.Closure) + GetRepositoryCommit_Jenkinsfile.echo(Executing on agent [label:none]) + GetRepositoryCommit_Jenkinsfile.stage(get repository commits, groovy.lang.Closure) + GetRepositoryCommit_Jenkinsfile.script(groovy.lang.Closure) + GetRepositoryCommit_Jenkinsfile.getRepositoryCommit({componentName=, inputManifest=tests/jenkins/data/opensearch-2.0.0.yml, outputFile=commits.yml}) + getRepositoryCommit.sh(#!/bin/bash + + set +x + set -e + + COMPONENT_LIST=() + COMPONENT_URL_LIST=() + COMPONENT_REF_LIST=() + + mkdir -p commits; cd commits + CURR_DIR=`pwd` + OUTPUT_FILE=commits.yml + cp -v /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml /tmp/workspace/$OUTPUT_FILE + + if [ -z "" ]; then + echo "Component list not specified so search the entire input manifest: /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml" + read -r -a COMPONENT_LIST <<< `yq e '.components[].name' /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml | tr ' +' ' '` + echo "Component list: ${COMPONENT_LIST[@]}" + else + echo "Specified component list: " + for comp in ; do + comp_temp=`yq e ".components[] | select(.name == \"$comp\") | .name" /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml | tr ' +' ' ' | head -n 1` + if [ -z "$comp_temp" ] || [ "$comp_temp" = "null" ]; then + echo "ERROR: $comp does not exist in manifest /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml" + exit 1 + fi + done + read -r -a COMPONENT_LIST <<< "" + echo "Component list: ${COMPONENT_LIST[@]}" + fi + + for entry in ${COMPONENT_LIST[@]}; do + COMPONENT_URL_LIST+=(`yq e ".components[] | select(.name == \"$entry\") | .repository" /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml | tr ' +' ' ' | head -n 1`) + COMPONENT_REF_LIST+=(`yq e ".components[] | select(.name == \"$entry\") | .ref" /tmp/workspace/tests/jenkins/data/opensearch-2.0.0.yml | tr ' +' ' ' | head -n 1`) + done + + echo "Component url list: ${COMPONENT_URL_LIST[@]}" + echo "Component ref list: ${COMPONENT_REF_LIST[@]}" + + for index in ${!COMPONENT_LIST[@]}; do + cd $CURR_DIR + mkdir -p ${COMPONENT_LIST[$index]} + cd ${COMPONENT_LIST[$index]} + git init -q + git remote add origin ${COMPONENT_URL_LIST[$index]} + git fetch --depth 1 origin ${COMPONENT_REF_LIST[$index]} + git checkout -q FETCH_HEAD + REAL_REF=`git rev-parse HEAD` + echo $REAL_REF + yq -i ".components |= map(select(.name == \"${COMPONENT_LIST[$index]}\").ref=\"$REAL_REF\")" /tmp/workspace/$OUTPUT_FILE + done + ) diff --git a/tests/jenkins/jobs/InputManifest_Jenkinsfile b/tests/jenkins/jobs/InputManifest_Jenkinsfile index 7b4ecfc423..d57a09bb46 100644 --- a/tests/jenkins/jobs/InputManifest_Jenkinsfile +++ b/tests/jenkins/jobs/InputManifest_Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('input manifest 1.1.0') { steps { script { - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/1.1.0/opensearch-1.1.0.yml")) + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/templates/opensearch/1.x/os-template-1.1.0.yml")) echo inputManifest.ci echo inputManifest.build.name echo inputManifest.build.version @@ -26,7 +26,7 @@ pipeline { stage('input manifest 1.2.0') { steps { script { - def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/1.2.0/opensearch-1.2.0.yml")) + def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: "manifests/templates/opensearch/1.x/os-template-1.2.0.yml")) echo inputManifest.ci.image.name echo inputManifest.ci.image.args echo inputManifest.build.name diff --git a/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt b/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt index 2294fd9937..d6cee7b09b 100644 --- a/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/InputManifest_Jenkinsfile.txt @@ -4,7 +4,7 @@ InputManifest_Jenkinsfile.echo(Executing on agent [label:none]) InputManifest_Jenkinsfile.stage(input manifest 1.1.0, groovy.lang.Closure) InputManifest_Jenkinsfile.script(groovy.lang.Closure) - InputManifest_Jenkinsfile.readYaml({file=manifests/1.1.0/opensearch-1.1.0.yml}) + InputManifest_Jenkinsfile.readYaml({file=manifests/templates/opensearch/1.x/os-template-1.1.0.yml}) InputManifest.asBoolean() InputManifest_Jenkinsfile.echo(null) InputManifest_Jenkinsfile.echo(OpenSearch) @@ -14,7 +14,7 @@ InputManifest_Jenkinsfile.echo(bundle-build/1.1.0/shas) InputManifest_Jenkinsfile.stage(input manifest 1.2.0, groovy.lang.Closure) InputManifest_Jenkinsfile.script(groovy.lang.Closure) - InputManifest_Jenkinsfile.readYaml({file=manifests/1.2.0/opensearch-1.2.0.yml}) + InputManifest_Jenkinsfile.readYaml({file=manifests/templates/opensearch/1.x/os-template-1.2.0.yml}) InputManifest.asBoolean() InputManifest_Jenkinsfile.echo(opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028) InputManifest_Jenkinsfile.echo(-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot) diff --git a/tests/jenkins/jobs/ParallelMessages_Jenkinsfile b/tests/jenkins/jobs/ParallelMessages_Jenkinsfile index 18aa6d382e..d2adf8a1d6 100644 --- a/tests/jenkins/jobs/ParallelMessages_Jenkinsfile +++ b/tests/jenkins/jobs/ParallelMessages_Jenkinsfile @@ -7,7 +7,7 @@ pipeline { parallel { stage('build-on-host') { steps { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { def messages = lib.jenkins.Messages.new(this); messages.add("${STAGE_NAME}", "built ${STAGE_NAME}") @@ -18,7 +18,7 @@ pipeline { stage('build-snapshots') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'ubuntu:latest' alwaysPull } @@ -32,7 +32,7 @@ pipeline { stage('build-x86') { agent { docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host' image 'amazonlinux' alwaysPull } @@ -46,7 +46,7 @@ pipeline { stage('build-arm64') { agent { docker { - label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host' + label 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host' image 'ubuntu:jammy' alwaysPull } @@ -60,7 +60,7 @@ pipeline { } post() { success { - node('Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') { + node('Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') { script { def messages = lib.jenkins.Messages.new(this) def stashed = messages.get(['build-on-host', 'build-snapshots', 'build-x86', 'build-arm64']) diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt index d1f3dfe25c..1a75c7f2b9 100644 --- a/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt @@ -8,16 +8,28 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-2.0.0-rc1.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-2.0.0-rc1-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-2.0.0-rc1-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt index b15ad19306..d72924bc28 100644 --- a/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt @@ -8,16 +8,28 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-dashboards-2.0.0-rc1.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-2.0.0-rc1-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-2.0.0-rc1-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile.txt index 41c2291da9..bbf50a3b62 100644 --- a/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile.txt @@ -12,64 +12,152 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-2.0.0-rc1.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.tar*,**/opensearch-2.0.0-rc1*.tar*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.tar*,**/opensearch-2.0.0-rc1*.tar*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.tar*,**/opensearch-2.0.0-rc1*.tar*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.tar*,**/opensearch-2.0.0-rc1*.tar*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-2.0.0-rc1-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.rpm*,**/opensearch-2.0.0-rc1*.rpm*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.rpm*,**/opensearch-2.0.0-rc1*.rpm*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-2.0.0-rc1-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.rpm*,**/opensearch-2.0.0-rc1*.rpm*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-min-2.0.0-rc1*.rpm*,**/opensearch-2.0.0-rc1*.rpm*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile.txt index 5c4f7880ef..ad7c215ba6 100644 --- a/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile.txt @@ -12,64 +12,152 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-dashboards-2.0.0-rc1.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.tar*,**/opensearch-dashboards-2.0.0-rc1*.tar*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.tar*,**/opensearch-dashboards-2.0.0-rc1*.tar*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.tar*,**/opensearch-dashboards-2.0.0-rc1*.tar*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.tar*,**/opensearch-dashboards-2.0.0-rc1*.tar*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-2.0.0-rc1-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.rpm*,**/opensearch-dashboards-2.0.0-rc1*.rpm*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.rpm*,**/opensearch-dashboards-2.0.0-rc1*.rpm*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-2.0.0-rc1-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.rpm*,**/opensearch-dashboards-2.0.0-rc1*.rpm*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-2.0.0-rc1*.rpm*,**/opensearch-dashboards-2.0.0-rc1*.rpm*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile.txt index 39a26b8005..8eab96af73 100644 --- a/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile.txt @@ -8,36 +8,49 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-ec2/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-ec2*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/transport-nio/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/transport-nio*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-gce/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-gce*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-ukrainian/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-ukrainian*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-azure-classic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-azure-classic*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-phonetic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-phonetic*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-murmur3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-murmur3*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-kuromoji/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-kuromoji*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-stempel/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-stempel*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-annotated-text/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-annotated-text*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-hdfs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-hdfs*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-icu/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-icu*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-size/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-size*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/ingest-attachment/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/ingest-attachment*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-azure/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-azure*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-s3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-s3*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-nori/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-nori*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/store-smb/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/store-smb*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-smartcn/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-smartcn*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-gcs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-gcs*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-1.3.0-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core Plugins) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-ec2/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-ec2*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/transport-nio/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/transport-nio*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-gce/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-gce*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-ukrainian/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-ukrainian*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-azure-classic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-azure-classic*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-phonetic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-phonetic*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-murmur3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-murmur3*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-kuromoji/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-kuromoji*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-stempel/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-stempel*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-annotated-text/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-annotated-text*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-hdfs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-hdfs*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-icu/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-icu*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-size/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-size*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/ingest-attachment/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/ingest-attachment*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-azure/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-azure*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-s3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-s3*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-nori/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-nori*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/store-smb/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/store-smb*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-smartcn/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-smartcn*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-gcs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-gcs*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-1.3.0-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile.txt index d3529d9c6a..8528bfb223 100644 --- a/tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile.txt @@ -8,16 +8,28 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-dashboards-1.3.0.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-1.3.0-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-1.3.0-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt index 717b762199..90263b48b8 100644 --- a/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt @@ -12,111 +12,238 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins) - createSha512Checksums.sh({script=find tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins -type f, returnStdout=true}) - createSha512Checksums.echo(Creating sha for tar_dummy_artifact_1.3.0.tar.gz) - createSha512Checksums.sh({script=sha512sum tar_dummy_artifact_1.3.0.tar.gz, returnStdout=true}) - createSha512Checksums.sh({script=basename tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/tar_dummy_artifact_1.3.0.tar.gz, returnStdout=true}) - createSha512Checksums.writeFile({file=tar_dummy_artifact_1.3.0.tar.gz.sha512, text=shaHashDummy_tar_dummy_artifact_1.3.0.tar.gz tar_dummy_artifact_1.3.0.tar.gz}) - createSha512Checksums.echo(Creating sha for zip_dummy_artifact_1.3.0.zip) - createSha512Checksums.sh({script=sha512sum zip_dummy_artifact_1.3.0.zip, returnStdout=true}) - createSha512Checksums.sh({script=basename tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/zip_dummy_artifact_1.3.0.zip, returnStdout=true}) - createSha512Checksums.writeFile({file=zip_dummy_artifact_1.3.0.zip.sha512, text=shaHashDummy_zip_dummy_artifact_1.3.0.zip zip_dummy_artifact_1.3.0.zip}) - createSha512Checksums.echo(Not generating sha for dummy_artifact_1.3.0.dummy in tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core Plugins) + createSha512Checksums.sh({script=find tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins -type f, returnStdout=true}) + createSha512Checksums.echo(Creating sha for tar_dummy_artifact_1.3.0.tar.gz) + createSha512Checksums.sh({script=sha512sum tar_dummy_artifact_1.3.0.tar.gz, returnStdout=true}) + createSha512Checksums.sh({script=basename tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/tar_dummy_artifact_1.3.0.tar.gz, returnStdout=true}) + createSha512Checksums.writeFile({file=tar_dummy_artifact_1.3.0.tar.gz.sha512, text=shaHashDummy_tar_dummy_artifact_1.3.0.tar.gz tar_dummy_artifact_1.3.0.tar.gz}) + createSha512Checksums.echo(Creating sha for zip_dummy_artifact_1.3.0.zip) + createSha512Checksums.sh({script=sha512sum zip_dummy_artifact_1.3.0.zip, returnStdout=true}) + createSha512Checksums.sh({script=basename tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/zip_dummy_artifact_1.3.0.zip, returnStdout=true}) + createSha512Checksums.writeFile({file=zip_dummy_artifact_1.3.0.zip.sha512, text=shaHashDummy_zip_dummy_artifact_1.3.0.zip zip_dummy_artifact_1.3.0.zip}) + createSha512Checksums.echo(Not generating sha for dummy_artifact_1.3.0.dummy in tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins --sigtype=.sig - ) - promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.tar*,**/opensearch-1.3.0*.tar*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.tar*,**/opensearch-1.3.0*.tar*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-ec2/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-ec2*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/transport-nio/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/transport-nio*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-gce/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-gce*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-ukrainian/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-ukrainian*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/discovery-azure-classic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-azure-classic*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-phonetic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-phonetic*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-murmur3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-murmur3*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-kuromoji/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-kuromoji*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-stempel/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-stempel*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-annotated-text/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-annotated-text*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-hdfs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-hdfs*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-icu/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-icu*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/mapper-size/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-size*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/ingest-attachment/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/ingest-attachment*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-azure/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-azure*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-s3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-s3*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-nori/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-nori*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/store-smb/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/store-smb*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/analysis-smartcn/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-smartcn*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/plugins/repository-gcs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-gcs*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-1.3.0-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.rpm*,**/opensearch-1.3.0*.rpm*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.rpm*,**/opensearch-1.3.0*.rpm*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins --sigtype=.sig + ) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.tar*,**/opensearch-1.3.0*.tar*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.tar*,**/opensearch-1.3.0*.tar*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-ec2/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-ec2*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/transport-nio/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/transport-nio*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-gce/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-gce*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-ukrainian/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-ukrainian*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/discovery-azure-classic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/discovery-azure-classic*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-phonetic/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-phonetic*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-murmur3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-murmur3*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-kuromoji/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-kuromoji*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-stempel/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-stempel*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-annotated-text/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-annotated-text*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-hdfs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-hdfs*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-icu/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-icu*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/mapper-size/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/mapper-size*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/ingest-attachment/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/ingest-attachment*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-azure/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-azure*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-s3/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-s3*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-nori/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-nori*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/store-smb/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/store-smb*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/analysis-smartcn/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/analysis-smartcn*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/plugins/repository-gcs/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/core-plugins/, includePathPattern=**/repository-gcs*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-1.3.0-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.rpm*,**/opensearch-1.3.0*.rpm*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-min-1.3.0*.rpm*,**/opensearch-1.3.0*.rpm*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-1.3.0-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile.txt index 5fbb75ddb5..a4e2ee002a 100644 --- a/tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile.txt @@ -12,64 +12,152 @@ promoteArtifacts.library({identifier=jenkins@20211123, retriever=null}) promoteArtifacts.readYaml({file=tests/jenkins/data/opensearch-dashboards-1.3.0.yml}) InputManifest.asBoolean() - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.tar*,**/opensearch-dashboards-1.3.0*.tar*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.tar*,**/opensearch-dashboards-1.3.0*.tar*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteArtifacts.println(S3 download tar artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/1.3.0/33/linux/x64/tar/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.tar*,**/opensearch-dashboards-1.3.0*.tar*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.tar*,**/opensearch-dashboards-1.3.0*.tar*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/core/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-1.3.0-linux-x64*}) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) - promoteArtifacts.withAWS({role=downloadRoleName, roleAccount=publicAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Download({bucket=artifact-bucket, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) - promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) - promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.rpm*,**/opensearch-dashboards-1.3.0*.rpm*}) - promoteArtifacts.getPath() - createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) - promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.rpm*,**/opensearch-dashboards-1.3.0*.rpm*}) - promoteArtifacts.getPath() - createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(tests/jenkins/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET - tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig - ) - promoteArtifacts.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteArtifacts.s3Upload({bucket=prod-bucket-name, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/builds/opensearch-dashboards/dist/, includePathPattern=**/opensearch-dashboards-min-1.3.0-linux-x64*}) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/tar/vars-build/1.3.0/33/linux/x64/tar/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) + promoteArtifacts.println(S3 download rpm artifacts before creating signatures) + promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/1.3.0/33/linux/x64/rpm/, force=true}) + promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml}) + promoteArtifacts.fileExists(tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/builds/opensearch-dashboards/core-plugins) + promoteArtifacts.println(Signing Starts) + promoteArtifacts.println(Signing Core/Bundle Artifacts) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.rpm*,**/opensearch-dashboards-1.3.0*.rpm*}) + promoteArtifacts.getPath() + createSha512Checksums.sh({script=find tests/jenkins/tests/jenkins/file/found.zip -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + createSha512Checksums.echo(Not generating sha for ccc in tests/jenkins/tests/jenkins/file/found.zip, doesn't match allowed types [.tar.gz, .zip, .rpm]) + promoteArtifacts.findFiles({glob=**/opensearch-dashboards-min-1.3.0*.rpm*,**/opensearch-dashboards-1.3.0*.rpm*}) + promoteArtifacts.getPath() + createSignatureFiles.signArtifacts({sigtype=.sig, artifactPath=tests/jenkins/tests/jenkins/file/found.zip}) + signArtifacts.echo(PGP or Windows Signature Signing) + signArtifacts.fileExists(tests/jenkins/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + tests/jenkins/sign.sh tests/jenkins/tests/jenkins/file/found.zip --sigtype=.sig + ) + promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/1.3.0/, workingDir=tests/jenkins/artifacts/rpm/vars-build/1.3.0/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-1.3.0-linux-x64*}) diff --git a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt index 4da623bf79..0cd8a64aeb 100644 --- a/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PromoteYumRepos_Jenkinsfile.txt @@ -8,75 +8,132 @@ promoteYumRepos.library({identifier=jenkins@20211123, retriever=null}) promoteYumRepos.readYaml({file=tests/jenkins/data/opensearch-1.3.0.yml}) InputManifest.asBoolean() - promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteYumRepos.println(Pulling Prod Yumrepo) - promoteYumRepos.sh(aws s3 sync s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress) - promoteYumRepos.sh( - set -e - set +x - set +x - - echo "Pulling 1.3.0 rpms" - cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum - curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm - curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm - - ls -l - - rm -vf repodata/repomd.xml.asc - - echo "Update repo metadata" - createrepo --update . - - # Rename .xml to .pom for signing - # Please do not add .xml to signer filter - # As maven have many .xml and we do not want to sign them - # This is an outlier case for yum repo only - mv -v repodata/repomd.xml repodata/repomd.pom - - echo "Complete metadata update, awaiting signing repomd.xml" + promoteYumRepos.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + promoteYumRepos.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + promoteYumRepos.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + promoteYumRepos.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteYumRepos.println(Pulling Prod Yumrepo) + promoteYumRepos.sh(aws s3 sync s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ --no-progress) + promoteYumRepos.sh( + set -e + set +x + set +x + + echo "Pulling 1.3.0 rpms" + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/x64/rpm/dist/opensearch/opensearch-1.3.0-linux-x64.rpm + curl -SLO https://ci.opensearch.org/dbc/opensearch/1.3.0/123/linux/arm64/rpm/dist/opensearch/opensearch-1.3.0-linux-arm64.rpm + + ls -l + + rm -vf repodata/repomd.xml.asc + + echo "Update repo metadata" + createrepo --update . + + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom + + echo "Complete metadata update, awaiting signing repomd.xml" cd - ) - promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux}) - signArtifacts.echo(PGP Signature Signing) - signArtifacts.fileExists(/tmp/workspace/sign.sh) - signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) - signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) - signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) - signArtifacts.sh( + promoteYumRepos.signArtifacts({artifactPath=/tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom, sigtype=.sig, platform=linux}) +<<<<<<< HEAD +<<<<<<< HEAD + signArtifacts.echo(PGP or Windows Signature Signing) +======= + signArtifacts.echo(PGP Signature Signing) +>>>>>>> 071d6df (Update remaining env vars with credentials in jenkins libraries (#2286)) +======= + signArtifacts.echo(PGP or Windows Signature Signing) +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) + signArtifacts.fileExists(/tmp/workspace/sign.sh) + signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) + signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) + signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + + /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype=.sig --platform=linux + ) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=signer_client_creds}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], signer_client_creds], groovy.lang.Closure) + signArtifacts.readJSON({text=signer_client_creds}) + signArtifacts.sh( #!/bin/bash set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype=.sig --platform=linux ) - promoteYumRepos.sh( - set -e - set +x - - cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/ - - ls -l - - mv -v repomd.pom repomd.xml - mv -v repomd.pom.sig repomd.xml.sig - - # This step is required as yum only accept .asc and signing workflow only support .sig - cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc - - rm -vf repomd.xml.sig - - ls -l - - cd - - +>>>>>>> 071d6df (Update remaining env vars with credentials in jenkins libraries (#2286)) +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + + /tmp/workspace/sign.sh /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/repomd.pom --sigtype=.sig --platform=linux + ) +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) + promoteYumRepos.sh( + set -e + set +x + + cd /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/repodata/ + + ls -l + + mv -v repomd.pom repomd.xml + mv -v repomd.pom.sig repomd.xml.sig + + # This step is required as yum only accept .asc and signing workflow only support .sig + cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc + + rm -vf repomd.xml.sig + + ls -l + + cd - ) - promoteYumRepos.withAWS({role=artifactPromotionRole, roleAccount=artifactsAccount, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - promoteYumRepos.println(Pushing Prod Yumrepo) - promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://prod-bucket-name/releases/bundle/opensearch/1.x/yum/ --no-progress) + promoteYumRepos.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + promoteYumRepos.println(Pushing Prod Yumrepo) + promoteYumRepos.sh(aws s3 sync /tmp/workspace/artifacts/releases/bundle/opensearch/1.x/yum/ s3://ARTIFACT_PRODUCTION_BUCKET_NAME/releases/bundle/opensearch/1.x/yum/ --no-progress) diff --git a/tests/jenkins/jobs/PublishNotification_Jenkinsfile b/tests/jenkins/jobs/PublishNotification_Jenkinsfile index 8035188d23..7b70b8d269 100644 --- a/tests/jenkins/jobs/PublishNotification_Jenkinsfile +++ b/tests/jenkins/jobs/PublishNotification_Jenkinsfile @@ -10,7 +10,7 @@ pipeline { message: 'Successful Build', extra: 'extra', manifest: '1.2.0/opensearch-1.2.0.yml', - credentialsId: 'BUILD_NOTICE_WEBHOOK' + credentialsId: 'jenkins-build-notice-webhook' ) }catch (Exception e) { echo 'Exception occurred: ' + e.toString() diff --git a/tests/jenkins/jobs/PublishNotification_Jenkinsfile.txt b/tests/jenkins/jobs/PublishNotification_Jenkinsfile.txt index 241afd26f8..5a7add52e0 100644 --- a/tests/jenkins/jobs/PublishNotification_Jenkinsfile.txt +++ b/tests/jenkins/jobs/PublishNotification_Jenkinsfile.txt @@ -3,8 +3,8 @@ PublishNotification_Jenkinsfile.echo(Executing on agent [label:none]) PublishNotification_Jenkinsfile.stage(notify, groovy.lang.Closure) PublishNotification_Jenkinsfile.script(groovy.lang.Closure) - PublishNotification_Jenkinsfile.publishNotification({icon=:white_check_mark:, message=Successful Build, extra=extra, manifest=1.2.0/opensearch-1.2.0.yml, credentialsId=BUILD_NOTICE_WEBHOOK}) - publishNotification.string({credentialsId=BUILD_NOTICE_WEBHOOK, variable=WEBHOOK_URL}) + PublishNotification_Jenkinsfile.publishNotification({icon=:white_check_mark:, message=Successful Build, extra=extra, manifest=1.2.0/opensearch-1.2.0.yml, credentialsId=jenkins-build-notice-webhook}) + publishNotification.string({credentialsId=jenkins-build-notice-webhook, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: JOB_NAME=dummy_job diff --git a/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt b/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt index 8d3524c227..47ab1dc84b 100644 --- a/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt @@ -10,12 +10,11 @@ runGradleCheck.usernamePassword({credentialsId=jenkins-gradle-check-s3-aws-resources, usernameVariable=amazon_s3_base_path, passwordVariable=amazon_s3_bucket}) runGradleCheck.withCredentials([[amazon_s3_access_key, amazon_s3_secret_key], [amazon_s3_base_path, amazon_s3_bucket]], groovy.lang.Closure) runGradleCheck.sh( + #!/bin/bash set -e set +x - env | grep JAVA | grep HOME - echo "Git clone: https://github.com/opensearch-project/OpenSearch with ref: main" rm -rf search git clone https://github.com/opensearch-project/OpenSearch search @@ -23,9 +22,24 @@ git checkout -f main git rev-parse HEAD - echo "Stop existing gradledaemon" + echo "Get Major Version" + OS_VERSION=`cat buildSrc/version.properties | grep opensearch | cut -d= -f2 | grep -oE '[0-9.]+'` + OS_MAJOR_VERSION=`echo $OS_VERSION | grep -oE '[0-9]+' | head -n 1` + echo "Version: $OS_VERSION, Major Version: $OS_MAJOR_VERSION" + + if [ "$OS_MAJOR_VERSION" -lt 2 ]; then + echo "Using JAVA 11" + export JAVA_HOME=$JAVA11_HOME + else + echo "Using JAVA 17" + export JAVA_HOME=$JAVA17_HOME + fi + + env | grep JAVA | grep HOME + + echo "Gradle clean cache and stop existing gradledaemon" ./gradlew --stop - find ~/.gradle -type f -name "*.lock" -delete + rm -rf ~/.gradle echo "Check existing dockercontainer" docker ps -a @@ -37,9 +51,15 @@ echo "Check docker-compose version" docker-compose version + echo "Check existing processes" + ps -ef | grep [o]pensearch | wc -l + echo "Cleanup existing processes" + kill -9 `ps -ef | grep [o]pensearch | awk '{print $2}'` > /dev/null 2>&1 || echo + ps -ef | grep [o]pensearch | wc -l + echo "Start gradlecheck" GRADLE_CHECK_STATUS=0 - ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1 + ./gradlew clean && ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1 if [ "$GRADLE_CHECK_STATUS" != 0 ]; then echo Gradle Check Failed! diff --git a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt index 6bf5908794..637da7a0ab 100644 --- a/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt +++ b/tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt @@ -4,28 +4,76 @@ SignArtifacts_Jenkinsfile.stage(sign, groovy.lang.Closure) SignArtifacts_Jenkinsfile.script(groovy.lang.Closure) SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=/tmp/workspace/artifacts, sigtype=.sig, platform=linux}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket - - /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux - ) + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/artifacts --sigtype=.sig --platform=linux + ) SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=/tmp/workspace/artifacts, sigtype=.rpm, platform=linux}) - signArtifacts.echo(RPM Add Sign) - signArtifacts.withAWS({role=sign_asm_role, roleAccount=sign_asm_account, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) - signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-pass-id, variable=SIGNING_PASS_ID}) - signArtifacts.string({credentialsId=jenkins-rpm-signing-asm-secret-id, variable=SIGNING_SECRET_ID}) - signArtifacts.withCredentials([SIGNING_PASS_ID, SIGNING_SECRET_ID], groovy.lang.Closure) + signArtifacts.string({credentialsId=jenkins-rpm-signing-account-number, variable=RPM_SIGNING_ACCOUNT_NUMBER}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-passphrase-secrets-arn, variable=RPM_SIGNING_PASSPHRASE_SECRETS_ARN}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-secret-key-secrets-arn, variable=RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN}) + signArtifacts.string({credentialsId=jenkins-rpm-signing-key-id, variable=RPM_SIGNING_KEY_ID}) + signArtifacts.withCredentials([RPM_SIGNING_ACCOUNT_NUMBER, RPM_SIGNING_PASSPHRASE_SECRETS_ARN, RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN, RPM_SIGNING_KEY_ID], groovy.lang.Closure) + signArtifacts.echo(RPM Add Sign) +<<<<<<< HEAD +<<<<<<< HEAD + signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) +======= + signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=null, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) +>>>>>>> 21fe974 (Change rpm signing role with new one (#2352)) +======= + signArtifacts.withAWS({role=jenkins-prod-rpm-signing-assume-role, roleAccount=RPM_SIGNING_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-signing-session}, groovy.lang.Closure) +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) signArtifacts.sh( set -e set +x @@ -65,8 +113,8 @@ echo "------------------------------------------------------------------------" echo "Import OpenSearch keys" - aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_PASS_ID" | jq -r .SecretBinary | base64 --decode > passphrase - aws secretsmanager get-secret-value --region "sign_asm_region" --secret-id "SIGNING_SECRET_ID" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_PASSPHRASE_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode > passphrase + aws secretsmanager get-secret-value --region us-west-2 --secret-id "RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - echo "------------------------------------------------------------------------" echo "Start Signing Rpm" @@ -93,25 +141,63 @@ echo "------------------------------------------------------------------------" echo "Clean up gpg" - gpg --batch --yes --delete-secret-keys sign_asm_keyid - gpg --batch --yes --delete-keys sign_asm_keyid + gpg --batch --yes --delete-secret-keys RPM_SIGNING_KEY_ID + gpg --batch --yes --delete-keys RPM_SIGNING_KEY_ID rm -v passphrase ) SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=/tmp/workspace/file.yml, platform=linux, type=maven}) - signArtifacts.echo(PGP Signature Signing) + signArtifacts.echo(PGP or Windows Signature Signing) signArtifacts.fileExists(/tmp/workspace/sign.sh) signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main}) signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -) signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN}) - signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure) +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + signArtifacts.string({credentialsId=jenkins-signer-client-role, variable=SIGNER_CLIENT_ROLE}) + signArtifacts.string({credentialsId=jenkins-signer-client-external-id, variable=SIGNER_CLIENT_EXTERNAL_ID}) + signArtifacts.string({credentialsId=jenkins-signer-client-unsigned-bucket, variable=SIGNER_CLIENT_UNSIGNED_BUCKET}) + signArtifacts.string({credentialsId=jenkins-signer-client-signed-bucket, variable=SIGNER_CLIENT_SIGNED_BUCKET}) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], SIGNER_CLIENT_ROLE, SIGNER_CLIENT_EXTERNAL_ID, SIGNER_CLIENT_UNSIGNED_BUCKET, SIGNER_CLIENT_SIGNED_BUCKET], groovy.lang.Closure) +<<<<<<< HEAD + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +======= + signArtifacts.string({credentialsId=signer-pgp-config, variable=configs}) +======= + signArtifacts.string({credentialsId=jenkins-signer-client-creds, variable=configs}) +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN], configs], groovy.lang.Closure) + signArtifacts.readJSON({text=configs}) + signArtifacts.sh( + #!/bin/bash + set +x + export ROLE=dummy_role + export EXTERNAL_ID=dummy_ID + export UNSIGNED_BUCKET=dummy_unsigned_bucket + export SIGNED_BUCKET=dummy_signed_bucket + export PROFILE_IDENTIFIER=null + export PLATFORM_IDENTIFIER=null + +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= signArtifacts.sh( - #!/bin/bash - set +x - export ROLE=dummy_signer_client_role - export EXTERNAL_ID=signer_client_external_id - export UNSIGNED_BUCKET=signer_client_unsigned_bucket - export SIGNED_BUCKET=signer_client_signed_bucket - - /tmp/workspace/sign.sh /tmp/workspace/file.yml --platform=linux --type=maven - ) + #!/bin/bash + set +x + export ROLE=SIGNER_CLIENT_ROLE + export EXTERNAL_ID=SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=SIGNER_CLIENT_SIGNED_BUCKET + +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + /tmp/workspace/sign.sh /tmp/workspace/file.yml --platform=linux --type=maven + ) diff --git a/tests/jenkins/jobs/data-prepper/promote-distribution-artifacts.jenkinsfile.txt b/tests/jenkins/jobs/data-prepper/promote-distribution-artifacts.jenkinsfile.txt index 9fbb333f10..9ab718aa30 100644 --- a/tests/jenkins/jobs/data-prepper/promote-distribution-artifacts.jenkinsfile.txt +++ b/tests/jenkins/jobs/data-prepper/promote-distribution-artifacts.jenkinsfile.txt @@ -2,7 +2,7 @@ promote-distribution-artifacts.legacySCM(groovy.lang.Closure) promote-distribution-artifacts.library({identifier=jenkins@20211123, retriever=null}) promote-distribution-artifacts.pipeline(groovy.lang.Closure) - promote-distribution-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + promote-distribution-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]]) promote-distribution-artifacts.stage(Download Staging Artifacts, groovy.lang.Closure) promote-distribution-artifacts.script(groovy.lang.Closure) promote-distribution-artifacts.sh(mkdir distribution) diff --git a/tests/jenkins/lib-testers/CreateReleaseTagLibTester.groovy b/tests/jenkins/lib-testers/CreateReleaseTagLibTester.groovy index 09087312b1..268532c9fe 100644 --- a/tests/jenkins/lib-testers/CreateReleaseTagLibTester.groovy +++ b/tests/jenkins/lib-testers/CreateReleaseTagLibTester.groovy @@ -1,4 +1,4 @@ -import jenkins.BuildManifest +import jenkins.BundleManifest import org.yaml.snakeyaml.Yaml import static org.hamcrest.CoreMatchers.notNullValue @@ -9,12 +9,12 @@ class CreateReleaseTagLibTester extends LibFunctionTester { private String distManifest private String tagVersion - private ArrayList buildManifestComponentsList + private ArrayList bundleManifestComponentsList public CreateReleaseTagLibTester(distManifest, tagVersion){ this.distManifest = distManifest this.tagVersion = tagVersion - this.buildManifestComponentsList = [] + this.bundleManifestComponentsList = [] } void parameterInvariantsAssertions(call){ @@ -25,7 +25,7 @@ class CreateReleaseTagLibTester extends LibFunctionTester { boolean expectedParametersMatcher(call) { return call.args.distManifest.first().toString().equals(this.distManifest) && call.args.tagVersion.first().toString().equals(this.tagVersion) - && this.buildManifestComponentsList.size() > 1 + && this.bundleManifestComponentsList.size() > 1 } String libFunctionName(){ @@ -42,11 +42,11 @@ class CreateReleaseTagLibTester extends LibFunctionTester { InputStream inputStream = new FileInputStream(new File(this.distManifest)); Yaml yaml = new Yaml() Map ymlMap = yaml.load(inputStream) - BuildManifest buildManifestObj = new BuildManifest(ymlMap) - this.buildManifestComponentsList = buildManifestObj.getNames() + BundleManifest bundleManifestObj = new BundleManifest(ymlMap) + this.bundleManifestComponentsList = bundleManifestObj.getNames() boolean checkFirst = true - for (component in this.buildManifestComponentsList) { - def repo = buildManifestObj.getRepo(component) + for (component in this.bundleManifestComponentsList) { + def repo = bundleManifestObj.getRepo(component) def version = tagVersion if (tagVersion.contains("-")) { version = tagVersion.split("-").first() + ".0-" + tagVersion.split("-").last() @@ -58,7 +58,7 @@ class CreateReleaseTagLibTester extends LibFunctionTester { } def out = "" if (checkFirst) { - out = buildManifestObj.getCommitId(component) + out = bundleManifestObj.getCommitId(component) checkFirst = false } helper.addShMock("git ls-remote --tags $repo $version | awk 'NR==1{print \$1}'") { script -> diff --git a/tests/jenkins/lib-testers/GetRepositoryCommitLibTester.groovy b/tests/jenkins/lib-testers/GetRepositoryCommitLibTester.groovy new file mode 100644 index 0000000000..2bb4fa6e54 --- /dev/null +++ b/tests/jenkins/lib-testers/GetRepositoryCommitLibTester.groovy @@ -0,0 +1,34 @@ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + + +class GetRepositoryCommitLibTester extends LibFunctionTester { + + private String componentName + private String inputManifest + private String outputFile + + public GetRepositoryCommitLibTester(componentName, inputManifest, outputFile){ + this.componentName = componentName + this.inputManifest = inputManifest + this.outputFile = outputFile + } + + void configure(helper, binding) { + // N/A + } + + void parameterInvariantsAssertions(call) { + assertThat(call.args.inputManifest.first(), notNullValue()) + assertThat(call.args.outputFile.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + return call.args.inputManifest.first().toString().equals(this.inputManifest) + && call.args.outputFile.first().toString().equals(this.outputFile) + } + + String libFunctionName() { + return 'getRepositoryCommit' + } +} diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 7a042d4c67..8747fe2730 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -37,8 +37,8 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { helper.registerAllowedMethod("downloadBuildManifest", [Map], { c -> lib.jenkins.BuildManifest.new(readYaml(file: bundleManifest)) }) - - binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') + helper.registerAllowedMethod('parameterizedCron', [String], null) + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') binding.setVariable('ARCHITECTURE', 'x64') binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') diff --git a/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy b/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy index 40d502246d..f20591bf81 100644 --- a/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy +++ b/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy @@ -1,7 +1,6 @@ import static org.hamcrest.CoreMatchers.notNullValue import static org.hamcrest.MatcherAssert.assertThat - class SignArtifactsLibTester extends LibFunctionTester { private String sigtype @@ -10,7 +9,7 @@ class SignArtifactsLibTester extends LibFunctionTester { private String type private String component - public SignArtifactsLibTester(sigtype, platform, artifactPath, type, component){ + public SignArtifactsLibTester(sigtype, platform, artifactPath, type, component) { this.sigtype = sigtype this.platform = platform this.artifactPath = artifactPath @@ -20,18 +19,34 @@ class SignArtifactsLibTester extends LibFunctionTester { void configure(helper, binding) { binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name') - binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role') - binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id') - binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket') - binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket') - binding.setVariable('SIGN_ASM_ROLE', 'sign_asm_role') - binding.setVariable('SIGN_ASM_ACCOUNT', 'sign_asm_account') - binding.setVariable('SIGN_ASM_REGION', 'sign_asm_region') - binding.setVariable('SIGN_ASM_KEYID', 'sign_asm_keyid') - - helper.registerAllowedMethod("git", [Map]) - helper.registerAllowedMethod("withCredentials", [Map]) - helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> +<<<<<<< HEAD +<<<<<<< HEAD +======= + if (this.sigtype.equals('.rpm')) { + def configs = ['account': '1234', + 'passphrase_secrets_arn': 'ARN::123456', + 'secret_key_id_secrets_arn': 'ARN::56789', + 'key_id': 'abcd1234'] + binding.setVariable('configs', configs) + helper.registerAllowedMethod('readJSON', [Map.class], { c -> configs }) + } + else { + def configs = ["role": "dummy_role", + "external_id": "dummy_ID", + "unsigned_bucket": "dummy_unsigned_bucket", + "signed_bucket": "dummy_signed_bucket"] + binding.setVariable('configs', configs) + helper.registerAllowedMethod('readJSON', [Map.class], { c -> configs }) + } +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + helper.registerAllowedMethod('git', [Map]) + helper.registerAllowedMethod('withCredentials', [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('withAWS', [Map, Closure], { args, closure -> closure.delegate = delegate return helper.callClosure(closure) }) @@ -40,15 +55,15 @@ class SignArtifactsLibTester extends LibFunctionTester { void parameterInvariantsAssertions(call) { assertThat(call.args.artifactPath.first(), notNullValue()) assertThat(call.args.platform.first(), notNullValue()) - if(call.args.artifactPath.first().toString().endsWith(".yml")){ + if (call.args.artifactPath.first().toString().endsWith('.yml')) { assertThat(call.args.type.first(), notNullValue()) - } else if(call.args.type.first() != 'maven'){ + } else if (call.args.type.first() != 'maven') { assertThat(call.args.sigtype.first(), notNullValue()) } } boolean expectedParametersMatcher(call) { - if(call.args.artifactPath.first().toString().endsWith(".yml")){ + if (call.args.artifactPath.first().toString().endsWith('.yml')) { return call.args.platform.first().toString().equals(this.platform) && call.args.artifactPath.first().toString().equals(this.artifactPath) && call.args.type.first().toString().equals(this.type) @@ -63,4 +78,5 @@ class SignArtifactsLibTester extends LibFunctionTester { String libFunctionName() { return 'signArtifacts' } + } diff --git a/tests/test_run_build.py b/tests/test_run_build.py index ec45c83261..dbc77469df 100644 --- a/tests/test_run_build.py +++ b/tests/test_run_build.py @@ -35,8 +35,8 @@ def test_usage(self) -> None: "manifests", ) - OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(MANIFESTS, "1.1.0", "opensearch-1.1.0.yml")) - OPENSEARCH_MANIFEST_1_2 = os.path.realpath(os.path.join(MANIFESTS, "1.2.0", "opensearch-1.2.0.yml")) + OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(MANIFESTS, "templates", "opensearch", "1.x", "os-template-1.1.0.yml")) + OPENSEARCH_MANIFEST_1_2 = os.path.realpath(os.path.join(MANIFESTS, "templates", "opensearch", "1.x", "os-template-1.2.0.yml")) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "linux"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @@ -84,8 +84,10 @@ def test_main_platform_windows(self, mock_temp: Mock, mock_recorder: Mock, mock_ os.path.dirname(__file__), "..", "manifests", - "1.1.0", - "opensearch-dashboards-1.1.0.yml", + "templates", + "opensearch-dashboards", + "1.x", + "osd-template-1.1.0.yml", ) ) diff --git a/tests/test_run_checkout.py b/tests/test_run_checkout.py index d0fdd2ceb6..dd60b671de 100644 --- a/tests/test_run_checkout.py +++ b/tests/test_run_checkout.py @@ -28,7 +28,17 @@ def test_usage(self) -> None: out, _ = self.capfd.readouterr() self.assertTrue(out.startswith("usage:")) - OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.1.0/opensearch-1.1.0.yml")) + OPENSEARCH_MANIFEST = os.path.realpath( + os.path.join( + os.path.dirname(__file__), + "..", + "manifests", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml" + ) + ) @patch("argparse._sys.argv", ["run_checkout.py", OPENSEARCH_MANIFEST]) @patch("run_checkout.GitRepository") diff --git a/tests/test_run_ci.py b/tests/test_run_ci.py index cd313505df..f0f4b8e729 100644 --- a/tests/test_run_ci.py +++ b/tests/test_run_ci.py @@ -28,7 +28,17 @@ def test_usage(self) -> None: out, _ = self.capfd.readouterr() self.assertTrue(out.startswith("usage:")) - OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "../manifests/1.1.1/opensearch-1.1.1.yml")) + OPENSEARCH_MANIFEST = os.path.realpath( + os.path.join( + os.path.dirname(__file__), + "..", + "manifests", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml" + ) + ) @patch("argparse._sys.argv", ["run_ci.py", OPENSEARCH_MANIFEST]) @patch("ci_workflow.ci_input_manifest.TemporaryDirectory") diff --git a/tests/test_run_manifests.py b/tests/test_run_manifests.py index d3d8fb711c..05d8f29a82 100644 --- a/tests/test_run_manifests.py +++ b/tests/test_run_manifests.py @@ -32,18 +32,24 @@ def test_main_list(self, mock_logging: Mock, *mocks: Any) -> None: main() mock_logging.info.assert_has_calls([ - call("OpenSearch 1.0.0"), - call("OpenSearch 1.0.1"), - call("OpenSearch 1.1.0"), - call("OpenSearch 1.1.1") + call("OpenSearch 1.3.0"), + call("OpenSearch 1.3.1"), + call("OpenSearch 1.3.2"), + call("OpenSearch 1.3.3"), + call("OpenSearch 1.3.4") ]) mock_logging.info.assert_has_calls([ - call("OpenSearch 2.0.0") + call("OpenSearch 2.0.0"), + call("OpenSearch 2.0.1") ]) mock_logging.info.assert_has_calls([ - call("OpenSearch Dashboards 1.1.0") + call("OpenSearch 3.0.0") + ]) + + mock_logging.info.assert_has_calls([ + call("OpenSearch Dashboards 1.3.0") ]) mock_logging.info.assert_has_calls([ diff --git a/tests/test_run_releasenotes_check.py b/tests/test_run_releasenotes_check.py new file mode 100644 index 0000000000..d1a98d7a74 --- /dev/null +++ b/tests/test_run_releasenotes_check.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import unittest +from typing import Any +from unittest.mock import patch + +import pytest + +from run_releasenotes_check import main + +gitLogDate = '2022-07-26' +# For testing addComment should be set to false +addComment = 'false' + + +class TestRunReleaseNotesCheck(unittest.TestCase): + @pytest.fixture(autouse=True) + def _capfd(self, capfd: Any) -> None: + self.capfd = capfd + + @patch("argparse._sys.argv", ["run_releasenotes_check.py", "--help"]) + def test_usage(self) -> None: + with self.assertRaises(SystemExit): + main(gitLogDate, addComment) + + out, _ = self.capfd.readouterr() + self.assertTrue(out.startswith("usage:")) + + OPENSEARCH_MANIFEST = os.path.realpath( + os.path.join( + os.path.dirname(__file__), + "..", + "manifests", + "templates", + "opensearch", + "2.x", + "manifest.yml" + ) + ) + + @patch("argparse._sys.argv", ["run_releasenotes_check.py", OPENSEARCH_MANIFEST]) + def test_main(self) -> None: + assert main(gitLogDate, addComment) == 0 + if gitLogDate is None: + assert main(gitLogDate, addComment) == 1 diff --git a/tests/test_run_sign.py b/tests/test_run_sign.py index 89e3186e74..606a1fea5d 100644 --- a/tests/test_run_sign.py +++ b/tests/test_run_sign.py @@ -33,8 +33,7 @@ def test_usage(self, *mocks: Any) -> None: @patch("argparse._sys.argv", ["run_sign.py", BUILD_MANIFEST]) @patch("run_sign.SignArtifacts") - @patch("run_sign.Signer") - def test_main(self, mock_signer: Mock, mock_sign_artifacts: Mock, *mocks: Any) -> None: + def test_main(self, mock_sign_artifacts: Mock, *mocks: Any) -> None: main() mock_sign_artifacts.from_path.assert_called_once() diff --git a/tests/tests_assemble_workflow/test_assemble_args.py b/tests/tests_assemble_workflow/test_assemble_args.py index fe6abfecbc..e7dbe4f9e8 100644 --- a/tests/tests_assemble_workflow/test_assemble_args.py +++ b/tests/tests_assemble_workflow/test_assemble_args.py @@ -16,7 +16,18 @@ class TestAssembleArgs(unittest.TestCase): ASSEMBLE_PY = "./src/run_assembly.py" - OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests", "1.1.0", "opensearch-1.1.0.yml")) + OPENSEARCH_MANIFEST = os.path.realpath( + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "manifests", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml", + ) + ) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST]) def test_manifest(self) -> None: diff --git a/tests/tests_build_workflow/test_build_args.py b/tests/tests_build_workflow/test_build_args.py index 3150d6b8c9..18c32c58ed 100644 --- a/tests/tests_build_workflow/test_build_args.py +++ b/tests/tests_build_workflow/test_build_args.py @@ -24,8 +24,10 @@ class TestBuildArgs(unittest.TestCase): "..", "..", "manifests", - "1.1.0", - "opensearch-1.1.0.yml", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml", ) ) diff --git a/tests/tests_checkout_workflow/test_checkout_args.py b/tests/tests_checkout_workflow/test_checkout_args.py index 5ff6e4ac99..56074cc457 100644 --- a/tests/tests_checkout_workflow/test_checkout_args.py +++ b/tests/tests_checkout_workflow/test_checkout_args.py @@ -22,8 +22,10 @@ class TestCheckoutArgs(unittest.TestCase): "..", "..", "manifests", - "1.1.0", - "opensearch-1.1.0.yml", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml", ) ) diff --git a/tests/tests_ci_workflow/test_ci_args.py b/tests/tests_ci_workflow/test_ci_args.py index 88b4aea9c7..12185c3163 100644 --- a/tests/tests_ci_workflow/test_ci_args.py +++ b/tests/tests_ci_workflow/test_ci_args.py @@ -23,8 +23,10 @@ class TestCiArgs(unittest.TestCase): "..", "..", "manifests", - "1.1.0", - "opensearch-1.1.0.yml", + "templates", + "opensearch", + "1.x", + "os-template-1.1.0.yml", ) ) diff --git a/tests/tests_manifests/test_input_manifest.py b/tests/tests_manifests/test_input_manifest.py index 39daa36ac1..ccbeb0ef84 100644 --- a/tests/tests_manifests/test_input_manifest.py +++ b/tests/tests_manifests/test_input_manifest.py @@ -43,7 +43,7 @@ def test_1_1_1_dist(self) -> None: self.assertIsInstance(component, InputComponentFromDist) def test_1_0(self) -> None: - path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.0.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") @@ -62,7 +62,7 @@ def test_1_0(self) -> None: self.assertIsInstance(component, InputComponentFromSource) def test_1_1(self) -> None: - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") @@ -122,7 +122,7 @@ def test_1_2(self) -> None: self.assertEqual(alerting_component.checks[1].args, "alerting") def test_to_dict(self) -> None: - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path) data = manifest.to_dict() with open(path) as f: @@ -137,14 +137,14 @@ def test_invalid_ref(self) -> None: self.assertTrue(str(context.exception).startswith("Invalid manifest schema: {'components': ")) def test_select(self) -> None: - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(len(list(manifest.components.select(focus=["common-utils"]))), 1) self.assertNotEqual(len(list(manifest.components.select(platform="windows"))), 0) self.assertEqual(len(list(manifest.components.select(focus=["k-NN"], platform="linux"))), 1) def test_select_none(self) -> None: - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path) with self.assertRaises(ValueError) as ctx: self.assertEqual(len(list(manifest.components.select(focus=["k-NN"], platform="windows"))), 0) @@ -172,7 +172,7 @@ def test_component___matches_focus__(self) -> None: @patch("subprocess.check_output") def test_stable(self, mock_output: Mock) -> None: mock_output.return_value.decode.return_value = "updated\tHEAD" - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path).stable() opensearch: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch.ref, "updated") @@ -181,27 +181,27 @@ def test_stable(self, mock_output: Mock) -> None: @patch("git.git_repository.GitRepository.stable_ref", return_value=("abcd", "1234")) def test_stable_override_build(self, git_repo: Mock, mock_output: Mock) -> None: mock_output.return_value.decode.return_value = "updated\tHEAD" - path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest = InputManifest.from_path(path).stable() opensearch: InputComponentFromSource = manifest.components["OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch.ref, "abcd") def test_eq(self) -> None: - path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = InputManifest.from_path(path) self.assertEqual(manifest1, manifest1) self.assertEqual(manifest1, manifest2) def test_neq(self) -> None: - path1 = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") - path2 = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") + path1 = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.0.0.yml") + path2 = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.1.0.yml") manifest1 = InputManifest.from_path(path1) manifest2 = InputManifest.from_path(path2) self.assertNotEqual(manifest1, manifest2) def test_neq_update(self) -> None: - path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") + path = os.path.join(self.manifests_path, "templates", "opensearch", "1.x", "os-template-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = copy.deepcopy(manifest1) self.assertEqual(manifest1, manifest2) diff --git a/tests/tests_manifests/test_input_manifests.py b/tests/tests_manifests/test_input_manifests.py index f15cabf98d..b61c5ab7fe 100644 --- a/tests/tests_manifests/test_input_manifests.py +++ b/tests/tests_manifests/test_input_manifests.py @@ -16,11 +16,11 @@ def setUp(self) -> None: def tests_configs(self) -> None: self.assertTrue(len(self.manifests)) - def test_1_1_0(self) -> None: - manifest = self.manifests["1.1.0"] + def test_3_0_0(self) -> None: + manifest = self.manifests["3.0.0"] self.assertIsNotNone(manifest) self.assertEqual(manifest.version, "1.0") - self.assertEqual(manifest.build.version, "1.1.0") + self.assertEqual(manifest.build.version, "3.0.0") self.assertEqual(manifest.build.name, "OpenSearch") self.assertEqual(manifest.build.filename, "opensearch") diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py index 46393ccd0a..4bfb97ae11 100644 --- a/tests/tests_manifests_workflow/test_input_manifests.py +++ b/tests/tests_manifests_workflow/test_input_manifests.py @@ -17,26 +17,52 @@ def test_manifests_path(self) -> None: self.assertEqual(path, InputManifests.manifests_path()) def test_create_manifest_opensearch(self) -> None: - input_manifests = InputManifests("opensearch") + input_manifests = InputManifests("OpenSearch") input_manifest = input_manifests.create_manifest("1.2.3", []) self.assertEqual( input_manifest.to_dict(), { "schema-version": "1.0", - "build": {"name": "opensearch", "version": "1.2.3"}, - "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2"}}, + "build": {"name": "OpenSearch", "version": "1.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", + "args": "-e JAVA_HOME=/opt/java/openjdk-11"}}, + }, + ) + + def test_create_manifest_opensearch_from_default(self) -> None: + input_manifests = InputManifests("OpenSearch") + input_manifest = input_manifests.create_manifest("0.2.3", []) + self.assertEqual( + input_manifest.to_dict(), + { + "schema-version": "1.0", + "build": {"name": "OpenSearch", "version": "0.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", + "args": "-e JAVA_HOME=/opt/java/openjdk-17"}}, }, ) def test_create_manifest_opensearch_dashboards(self) -> None: - input_manifests = InputManifests("opensearch-dashboards") + input_manifests = InputManifests("OpenSearch Dashboards") input_manifest = input_manifests.create_manifest("1.2.3", []) self.assertEqual( input_manifest.to_dict(), { "schema-version": "1.0", - "build": {"name": "opensearch-dashboards", "version": "1.2.3"}, - "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2"}}, + "build": {"name": "OpenSearch Dashboards", "version": "1.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }}, + }, + ) + + def test_create_manifest_opensearch_dashboards_from_default(self) -> None: + input_manifests = InputManifests("OpenSearch Dashboards") + input_manifest = input_manifests.create_manifest("4.2.3", []) + self.assertEqual( + input_manifest.to_dict(), + { + "schema-version": "1.0", + "build": {"name": "OpenSearch Dashboards", "version": "4.2.3"}, + "ci": {"image": {"name": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2", }}, }, ) diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py index f130594744..afd6d2e9d8 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py @@ -22,14 +22,16 @@ def test_files(self) -> None: "..", "..", "manifests", - "1.1.0", - "opensearch-1.1.0.yml", + "3.0.0", + "opensearch-3.0.0.yml", ) ) self.assertTrue(manifest in files) @patch("os.makedirs") @patch("os.chdir") + @patch("manifests_workflow.input_manifests.InputComponents") + @patch("manifests_workflow.input_manifests.InputManifest.from_file") @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron") @patch("manifests_workflow.input_manifests.InputManifest.from_path") @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin") @@ -37,7 +39,8 @@ def test_files(self) -> None: @patch("manifests_workflow.input_manifests.InputManifest") def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: MagicMock, mock_component_opensearch_min: MagicMock, mock_input_manifest_from_path: MagicMock, - mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None: + mock_add_to_cron: MagicMock, mock_input_manifest_from_file: MagicMock, + mock_input_manifest_component: MagicMock, *mocks: MagicMock) -> None: mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch") mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"] mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0") @@ -48,7 +51,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: } manifests = InputManifestsOpenSearch() manifests.update() - self.assertEqual(mock_input_manifest().to_file.call_count, 2) + self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 2) calls = [ call( os.path.join( @@ -65,7 +68,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: ) ), ] - mock_input_manifest().to_file.assert_has_calls(calls) + mock_input_manifest_from_file().to_file.assert_has_calls(calls) mock_add_to_cron.assert_has_calls([ call('0.10.0'), call('0.9.0') diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py index f529884e69..bba62f27bf 100644 --- a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py +++ b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py @@ -21,27 +21,32 @@ def test_files(self) -> None: "..", "..", "manifests", - "1.1.0", - "opensearch-dashboards-1.1.0.yml", + "3.0.0", + "opensearch-dashboards-3.0.0.yml", ) ) self.assertTrue(manifest in files) @patch("os.makedirs") @patch("os.chdir") + @patch("manifests_workflow.input_manifests.InputComponents") + @patch("manifests_workflow.input_manifests.InputManifest.from_file") @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron") @patch("manifests_workflow.input_manifests.InputManifest.from_path") @patch("manifests_workflow.input_manifests_opensearch_dashboards.ComponentOpenSearchDashboardsMin") @patch("manifests_workflow.input_manifests.InputManifest") def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_min: MagicMock, - mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, *mocks: MagicMock) -> None: + mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, + mock_input_manifest_from_file: MagicMock, mock_input_manifest_component: MagicMock, + *mocks: MagicMock) -> None: mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch-Dashboards") mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"] mock_component_opensearch_min.checkout.return_value = MagicMock(version="0.9.0") mock_input_manifest_from_path.return_value = MagicMock(components=[]) + manifests = InputManifestsOpenSearchDashboards() manifests.update() - self.assertEqual(mock_input_manifest().to_file.call_count, 1) + self.assertEqual(mock_input_manifest_from_file().to_file.call_count, 1) calls = [ call( os.path.join( @@ -51,7 +56,7 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_ ) ) ] - mock_input_manifest().to_file.assert_has_calls(calls) + mock_input_manifest_from_file().to_file.assert_has_calls(calls) mock_add_to_cron.assert_has_calls([ call('0.9.0') ]) diff --git a/tests/tests_sign_workflow/test_sign_artifacts.py b/tests/tests_sign_workflow/test_sign_artifacts.py index 0f37d07d1b..5dffa37489 100644 --- a/tests/tests_sign_workflow/test_sign_artifacts.py +++ b/tests/tests_sign_workflow/test_sign_artifacts.py @@ -10,19 +10,20 @@ class TestSignArtifacts(unittest.TestCase): @patch("sign_workflow.signer.GitRepository") - @patch("sign_workflow.signer.Signer", return_value=MagicMock()) + @patch("sign_workflow.signer_pgp.SignerPGP", return_value=MagicMock()) def test_from_path_method(self, mock_signer: Mock, *mocks: Any) -> None: components = ['maven'] artifact_type = 'dummy' sigtype = '.asc' + platform = 'linux' - klass = SignArtifacts.from_path(Path(r"/dummy/path/manifest.yml"), components, artifact_type, sigtype, mock_signer) + klass = SignArtifacts.from_path(Path(r"/dummy/path/manifest.yml"), components, artifact_type, sigtype, platform) self.assertEqual(type(SignWithBuildManifest), type(klass.__class__)) - klass = SignArtifacts.from_path(Path(os.path.dirname(__file__)), components, artifact_type, sigtype, mock_signer) + klass = SignArtifacts.from_path(Path(os.path.dirname(__file__)), components, artifact_type, sigtype, platform) self.assertEqual(type(SignExistingArtifactsDir), type(klass.__class__)) - klass = SignArtifacts.from_path(Path(r"/dummy/path/artifact.tar.gz"), components, artifact_type, sigtype, mock_signer) + klass = SignArtifacts.from_path(Path(r"/dummy/path/artifact.tar.gz"), components, artifact_type, sigtype, platform) self.assertEqual(type(SignArtifactsExistingArtifactFile), type(klass.__class__)) def test_signer_class(self) -> None: @@ -38,17 +39,20 @@ def test_signer_class(self) -> None: Path(r"/dummy/path/artifact.tar.gz")), SignArtifactsExistingArtifactFile) - def test_sign_with_build_manifest(self) -> None: + @patch("sign_workflow.signer.GitRepository") + def test_sign_with_build_manifest(self, mock_repo: Mock) -> None: manifest = Path(os.path.join(os.path.dirname(__file__), "data", "opensearch-build-1.1.0.yml")) sigtype = '.asc' - signer = MagicMock() + platform = 'windows' signer_with_manifest = SignWithBuildManifest( target=manifest, components=[], artifact_type="maven", signature_type=sigtype, - signer=signer + platform=platform ) + signer = MagicMock() + signer_with_manifest.signer = signer signer_with_manifest.sign() expected = [ 'maven/org/opensearch/opensearch-performance-analyzer/maven-metadata-local.xml', @@ -60,35 +64,42 @@ def test_sign_with_build_manifest(self) -> None: ] signer.sign_artifacts.assert_called_with(expected, manifest.parent, sigtype) - def test_sign_existing_artifacts_file(self) -> None: + @patch("sign_workflow.signer.GitRepository") + def test_sign_existing_artifacts_file(self, mock_repo: Mock) -> None: path = Path(r"/dummy/path/file.tar.gz") sigtype = '.sig' - signer = MagicMock() + platform = 'linux' signer_with_manifest = SignArtifactsExistingArtifactFile( target=path, components=['maven'], artifact_type='dummy', signature_type=sigtype, - signer=signer + platform=platform ) + signer = MagicMock() + signer_with_manifest.signer = signer signer_with_manifest.sign() - signer.sign_artifact.assert_called_with("file.tar.gz", path.parent, sigtype) + expected = 'file.tar.gz' + signer.sign_artifact.assert_called_with(expected, path.parent, sigtype) + @patch("sign_workflow.signer.GitRepository") @patch('os.walk') - def test_sign_existing_artifacts_folder(self, mock_os_walk: Mock) -> None: + def test_sign_existing_artifacts_folder(self, mock_os_walk: Mock, mock_repo: Mock) -> None: mock_os_walk.return_value = [ ('dummy', (), ['tar_dummy_artifact_1.0.0.tar.gz', 'zip_dummy_artifact_1.1.0.zip']) ] path = Path('dummy') sigtype = '.sig' - signer = MagicMock() + platform = 'linux' signer_with_manifest = SignExistingArtifactsDir( target=path, components=['maven'], artifact_type='dummy', signature_type=sigtype, - signer=signer + platform=platform ) + signer = MagicMock() + signer_with_manifest.signer = signer signer_with_manifest.sign() expected = ["tar_dummy_artifact_1.0.0.tar.gz", "zip_dummy_artifact_1.1.0.zip"] signer.sign_artifacts.assert_called_with(expected, path, sigtype) diff --git a/tests/tests_sign_workflow/test_signer.py b/tests/tests_sign_workflow/test_signer.py index 51d4b5a8b2..00ad5ad4c2 100644 --- a/tests/tests_sign_workflow/test_signer.py +++ b/tests/tests_sign_workflow/test_signer.py @@ -7,128 +7,46 @@ class TestSigner(unittest.TestCase): - @patch("sign_workflow.signer.GitRepository") - def test_accepted_file_types_asc(self, git_repo: Mock) -> None: - artifacts = [ - "bad-xml.xml", - "the-jar.jar", - "the-zip.zip", - "the-whl.whl", - "the-rpm.rpm", - "the-war.war", - "the-pom.pom", - "the-module.module", - "the-tar.tar.gz", - "random-file.txt", - "something-1.0.0.0.jar", - ] - expected = [ - call(os.path.join("path", "the-jar.jar"), ".asc"), - call(os.path.join("path", "the-zip.zip"), ".asc"), - call(os.path.join("path", "the-whl.whl"), ".asc"), - call(os.path.join("path", "the-rpm.rpm"), ".asc"), - call(os.path.join("path", "the-war.war"), ".asc"), - call(os.path.join("path", "the-pom.pom"), ".asc"), - call(os.path.join("path", "the-module.module"), ".asc"), - call(os.path.join("path", "the-tar.tar.gz"), ".asc"), - call(os.path.join("path", "something-1.0.0.0.jar"), ".asc"), - ] - signer = Signer() - signer.sign = MagicMock() # type: ignore - signer.sign_artifacts(artifacts, Path("path"), ".asc") - self.assertEqual(signer.sign.call_args_list, expected) + class DummySigner(Signer): + def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None: + pass - @patch("sign_workflow.signer.GitRepository") - def test_accepted_file_types_sig(self, git_repo: Mock) -> None: - artifacts = [ - "bad-xml.xml", - "the-jar.jar", - "the-zip.zip", - "the-whl.whl", - "the-rpm.rpm", - "the-war.war", - "the-pom.pom", - "the-module.module", - "the-tar.tar.gz", - "random-file.txt", - "something-1.0.0.0.jar", - "opensearch_sql_cli-1.0.0-py3-none-any.whl", - "cratefile.crate" - ] - expected = [ - call(os.path.join("path", "the-jar.jar"), ".sig"), - call(os.path.join("path", "the-zip.zip"), ".sig"), - call(os.path.join("path", "the-whl.whl"), ".sig"), - call(os.path.join("path", "the-rpm.rpm"), ".sig"), - call(os.path.join("path", "the-war.war"), ".sig"), - call(os.path.join("path", "the-pom.pom"), ".sig"), - call(os.path.join("path", "the-module.module"), ".sig"), - call(os.path.join("path", "the-tar.tar.gz"), ".sig"), - call(os.path.join("path", "something-1.0.0.0.jar"), ".sig"), - call(os.path.join("path", "opensearch_sql_cli-1.0.0-py3-none-any.whl"), ".sig"), - call(os.path.join("path", "cratefile.crate"), ".sig") - ] - signer = Signer() - signer.sign = MagicMock() # type: ignore - signer.sign_artifacts(artifacts, Path("path"), ".sig") - self.assertEqual(signer.sign.call_args_list, expected) + def is_valid_file_type(self, file_name: str) -> bool: + return file_name.endswith('zip') + + def sign(self, artifact: str, basepath: Path, signature_type: str) -> None: + pass @patch("sign_workflow.signer.GitRepository") def test_signer_checks_out_tool(self, mock_repo: Mock) -> None: - Signer() + self.DummySigner() self.assertEqual(mock_repo.return_value.execute.call_count, 2) mock_repo.return_value.execute.assert_has_calls([call("./bootstrap"), call("rm config.cfg")]) - @patch("sign_workflow.signer.GitRepository") - def test_signer_verify_asc(self, mock_repo: Mock) -> None: - signer = Signer() - signer.verify("/path/the-jar.jar.asc") - mock_repo.assert_has_calls([call().execute("gpg --verify-files /path/the-jar.jar.asc")]) - - @patch("sign_workflow.signer.GitRepository") - def test_signer_verify_sig(self, mock_repo: Mock) -> None: - signer = Signer() - signer.verify("/path/the-jar.jar.sig") - mock_repo.assert_has_calls([call().execute("gpg --verify-files /path/the-jar.jar.sig")]) - - @patch("sign_workflow.signer.GitRepository") - def test_signer_sign_asc(self, mock_repo: Mock) -> None: - signer = Signer() - signer.sign("/path/the-jar.jar", ".asc") - mock_repo.assert_has_calls( - [call().execute("./opensearch-signer-client -i /path/the-jar.jar -o /path/the-jar.jar.asc -p pgp")]) - - @patch("sign_workflow.signer.GitRepository") - def test_signer_sign_sig(self, mock_repo: Mock) -> None: - signer = Signer() - signer.sign("/path/the-jar.jar", ".sig") - mock_repo.assert_has_calls( - [call().execute("./opensearch-signer-client -i /path/the-jar.jar -o /path/the-jar.jar.sig -p pgp")]) - @patch("sign_workflow.signer.GitRepository") def test_sign_artifact_not_called(self, mock_repo: Mock) -> None: - signer = Signer() + signer = self.DummySigner() signer.generate_signature_and_verify = MagicMock() # type: ignore signer.sign_artifact("the-jar.notvalid", Path("/path"), ".sig") signer.generate_signature_and_verify.assert_not_called() @patch("sign_workflow.signer.GitRepository") def test_sign_artifact_called(self, mock_repo: Mock) -> None: - signer = Signer() + signer = self.DummySigner() signer.generate_signature_and_verify = MagicMock() # type: ignore signer.sign_artifact("the-jar.zip", Path("/path"), ".sig") signer.generate_signature_and_verify.assert_called_with("the-jar.zip", Path("/path"), ".sig") @patch("sign_workflow.signer.GitRepository") def test_remove_existing_signature_found(self, mock_repo: Mock) -> None: - signer = Signer() + signer = self.DummySigner() os.remove = MagicMock() - signer.sign("tests/tests_sign_workflow/data/signature/tar_dummy_artifact_1.0.0.tar.gz", ".sig") + signer.__remove_existing_signature__("tests/tests_sign_workflow/data/signature/tar_dummy_artifact_1.0.0.tar.gz.sig") os.remove.assert_called_with("tests/tests_sign_workflow/data/signature/tar_dummy_artifact_1.0.0.tar.gz.sig") @patch("sign_workflow.signer.GitRepository") def test_remove_existing_signature_not_found(self, mock_repo: Mock) -> None: - signer = Signer() + signer = self.DummySigner() os.remove = MagicMock() - signer.sign("tests/tests_sign_workflow/data/signature/not_found.tar.gz", ".sig") + signer.__remove_existing_signature__("tests/tests_sign_workflow/data/signature/not_found.tar.gz.sig") os.remove.assert_not_called() diff --git a/tests/tests_sign_workflow/test_signer_pgp.py b/tests/tests_sign_workflow/test_signer_pgp.py new file mode 100644 index 0000000000..08153fa114 --- /dev/null +++ b/tests/tests_sign_workflow/test_signer_pgp.py @@ -0,0 +1,104 @@ +import os +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, call, patch + +from sign_workflow.signer_pgp import SignerPGP + + +class TestSignerPGP(unittest.TestCase): + @patch("sign_workflow.signer.GitRepository") + def test_accepted_file_types_asc(self, git_repo: Mock) -> None: + artifacts = [ + "bad-xml.xml", + "the-jar.jar", + "the-zip.zip", + "the-whl.whl", + "the-rpm.rpm", + "the-war.war", + "the-pom.pom", + "the-module.module", + "the-tar.tar.gz", + "random-file.txt", + "something-1.0.0.0.jar", + ] + expected = [ + call("the-jar.jar", Path("path"), ".asc"), + call("the-zip.zip", Path("path"), ".asc"), + call("the-whl.whl", Path("path"), ".asc"), + call("the-rpm.rpm", Path("path"), ".asc"), + call("the-war.war", Path("path"), ".asc"), + call("the-pom.pom", Path("path"), ".asc"), + call("the-module.module", Path("path"), ".asc"), + call("the-tar.tar.gz", Path("path"), ".asc"), + call("something-1.0.0.0.jar", Path("path"), ".asc"), + ] + signer = SignerPGP() + signer.sign = MagicMock() # type: ignore + signer.verify = MagicMock() # type: ignore + signer.sign_artifacts(artifacts, Path("path"), ".asc") + self.assertEqual(signer.sign.call_args_list, expected) + + @patch("sign_workflow.signer.GitRepository") + def test_accepted_file_types_sig(self, git_repo: Mock) -> None: + artifacts = [ + "bad-xml.xml", + "the-jar.jar", + "the-zip.zip", + "the-whl.whl", + "the-rpm.rpm", + "the-war.war", + "the-pom.pom", + "the-module.module", + "the-tar.tar.gz", + "random-file.txt", + "something-1.0.0.0.jar", + "opensearch_sql_cli-1.0.0-py3-none-any.whl", + "cratefile.crate" + ] + expected = [ + call("the-jar.jar", Path("path"), ".sig"), + call("the-zip.zip", Path("path"), ".sig"), + call("the-whl.whl", Path("path"), ".sig"), + call("the-rpm.rpm", Path("path"), ".sig"), + call("the-war.war", Path("path"), ".sig"), + call("the-pom.pom", Path("path"), ".sig"), + call("the-module.module", Path("path"), ".sig"), + call("the-tar.tar.gz", Path("path"), ".sig"), + call("something-1.0.0.0.jar", Path("path"), ".sig"), + call("opensearch_sql_cli-1.0.0-py3-none-any.whl", Path("path"), ".sig"), + call("cratefile.crate", Path("path"), ".sig") + ] + signer = SignerPGP() + signer.sign = MagicMock() # type: ignore + signer.verify = MagicMock() # type: ignore + signer.sign_artifacts(artifacts, Path("path"), ".sig") + self.assertEqual(signer.sign.call_args_list, expected) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_verify_asc(self, mock_repo: Mock) -> None: + signer = SignerPGP() + signer.verify("/path/the-jar.jar.asc") + mock_repo.assert_has_calls([call().execute("gpg --verify-files /path/the-jar.jar.asc")]) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_verify_sig(self, mock_repo: Mock) -> None: + signer = SignerPGP() + signer.verify("/path/the-jar.jar.sig") + mock_repo.assert_has_calls([call().execute("gpg --verify-files /path/the-jar.jar.sig")]) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_sign_asc(self, mock_repo: Mock) -> None: + signer = SignerPGP() + signer.sign("the-jar.jar", Path("/path/"), ".asc") + command = "./opensearch-signer-client -i " + os.path.join(Path("/path/"), 'the-jar.jar') + " -o " + os.path.join(Path("/path/"), 'the-jar.jar.asc') + " -p pgp" + mock_repo.assert_has_calls( + [call().execute(command)]) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_sign_sig(self, mock_repo: Mock) -> None: + signer = SignerPGP() + signer.sign("the-jar.jar", Path("/path/"), ".sig") + command = "./opensearch-signer-client -i " + os.path.join(Path("/path/"), 'the-jar.jar') + " -o " + os.path.join(Path("/path/"), 'the-jar.jar.sig') + " -p pgp" + mock_repo.assert_has_calls( + [call().execute(command)]) diff --git a/tests/tests_sign_workflow/test_signer_windows.py b/tests/tests_sign_workflow/test_signer_windows.py new file mode 100644 index 0000000000..2a1ea77013 --- /dev/null +++ b/tests/tests_sign_workflow/test_signer_windows.py @@ -0,0 +1,49 @@ +import os +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, call, patch + +from sign_workflow.signer_windows import SignerWindows + + +class TestSignerWindows(unittest.TestCase): + + @patch("sign_workflow.signer.GitRepository") + def test_accepted_file_types(self, git_repo: Mock) -> None: + artifacts = [ + "bad-xml.xml", + "the-msi.msi", + "the-exe.exe", + "the-dll.dll", + "the-sys.sys", + "the-ps1.ps1", + "the-psm1.psm1", + "the-cat.cat", + "the-zip.zip", + "random-file.txt", + "something-1.0.0.0.jar", + ] + expected = [ + call("the-msi.msi", Path("path"), ".asc"), + call("the-exe.exe", Path("path"), ".asc"), + call("the-dll.dll", Path("path"), ".asc"), + call("the-sys.sys", Path("path"), ".asc"), + call("the-ps1.ps1", Path("path"), ".asc"), + call("the-psm1.psm1", Path("path"), ".asc"), + call("the-cat.cat", Path("path"), ".asc"), + call("the-zip.zip", Path("path"), ".asc"), + ] + signer = SignerWindows() + signer.sign = MagicMock() # type: ignore + signer.sign_artifacts(artifacts, Path("path"), ".asc") + self.assertEqual(signer.sign.call_args_list, expected) + + @patch("sign_workflow.signer.GitRepository") + @patch('os.rename') + @patch('os.mkdir') + def test_signer_sign(self, mock_os_mkdir: Mock, mock_os_rename: Mock, mock_repo: Mock) -> None: + signer = SignerWindows() + signer.sign("the-msi.msi", Path("/path/"), ".asc") + command = "./opensearch-signer-client -i " + os.path.join(Path("/path/"), 'the-msi.msi') + " -o " + os.path.join(Path("/path/"), 'signed_the-msi.msi') + " -p windows" + mock_repo.assert_has_calls( + [call().execute(command)]) diff --git a/tests/tests_sign_workflow/test_signers.py b/tests/tests_sign_workflow/test_signers.py new file mode 100644 index 0000000000..57b63b88ed --- /dev/null +++ b/tests/tests_sign_workflow/test_signers.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import unittest +from unittest.mock import Mock, patch + +from sign_workflow.signer_pgp import SignerPGP +from sign_workflow.signer_windows import SignerWindows +from sign_workflow.signers import Signers + + +class TestSigners(unittest.TestCase): + + @patch("sign_workflow.signer.GitRepository") + def test_signer_PGP(self, mock_repo: Mock) -> None: + signer = Signers.create("linux") + self.assertIs(type(signer), SignerPGP) + + @patch("sign_workflow.signer.GitRepository") + def test_signer_windows(self, mock_repo: Mock) -> None: + signer = Signers.create("windows") + self.assertIs(type(signer), SignerWindows) + + def test_signer_invalid(self) -> None: + with self.assertRaises(ValueError) as ctx: + Signers.create("mac") + self.assertEqual(str(ctx.exception), "Unsupported type of platform for signing: mac") diff --git a/vars/buildUploadManifestSHA.groovy b/vars/buildUploadManifestSHA.groovy index 805a044ccf..148f4016a5 100644 --- a/vars/buildUploadManifestSHA.groovy +++ b/vars/buildUploadManifestSHA.groovy @@ -3,7 +3,12 @@ void call(Map args = [:]) { def sha = getManifestSHA(args) - withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Upload(bucket: "${ARTIFACT_BUCKET_NAME}", file: sha.lock, path: sha.path) - } -} \ No newline at end of file + withCredentials([ + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME'), + string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC')]) { + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Upload(bucket: "${ARTIFACT_BUCKET_NAME}", file: sha.lock, path: sha.path) + } + } + +} diff --git a/vars/createReleaseTag.groovy b/vars/createReleaseTag.groovy index beb7357f2d..b3c5f4a64e 100644 --- a/vars/createReleaseTag.groovy +++ b/vars/createReleaseTag.groovy @@ -1,9 +1,9 @@ def call(Map args = [:]) { def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) - def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: args.distManifest)) + def bundleManifestObj = lib.jenkins.BundleManifest.new(readYaml(file: args.distManifest)) - def componentsName = buildManifestObj.getNames() + def componentsName = bundleManifestObj.getNames() def componetsNumber = componentsName.size() def version = args.tagVersion def untaggedRepoList = [] @@ -11,8 +11,8 @@ def call(Map args = [:]) { withCredentials([usernamePassword(credentialsId: "${GITHUB_BOT_TOKEN_NAME}", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { for (component in componentsName) { - def commitID = buildManifestObj.getCommitId(component) - def repo = buildManifestObj.getRepo(component) + def commitID = bundleManifestObj.getCommitId(component) + def repo = bundleManifestObj.getRepo(component) def push_url = "https://$GITHUB_TOKEN@" + repo.minus('https://') echo "Tagging $component at $commitID ..." diff --git a/vars/detectDockerAgent.groovy b/vars/detectDockerAgent.groovy index 9a7538beba..d378c08939 100644 --- a/vars/detectDockerAgent.groovy +++ b/vars/detectDockerAgent.groovy @@ -4,9 +4,18 @@ Map call(Map args = [:]) { def inputManifest = lib.jenkins.InputManifest.new(readYaml(file: manifest)) dockerImage = inputManifest.ci?.image?.name ?: 'opensearchstaging/ci-runner:ci-runner-centos7-v1' dockerArgs = inputManifest.ci?.image?.args + // Using default javaVersion as openjdk-17 + String javaVersion = 'openjdk-17' + java.util.regex.Matcher jdkMatch = (dockerArgs =~ /openjdk-\d+/) + if (jdkMatch.find()) { + def jdkMatchLine = jdkMatch[0] + javaVersion = jdkMatchLine + } echo "Using Docker image ${dockerImage} (${dockerArgs})" + echo "Using java version ${javaVersion}" return [ image: dockerImage, - args: dockerArgs + args: dockerArgs, + javaVersion: javaVersion ] } diff --git a/vars/downloadFromS3.groovy b/vars/downloadFromS3.groovy index 6ac31404be..1e0c04ffac 100644 --- a/vars/downloadFromS3.groovy +++ b/vars/downloadFromS3.groovy @@ -1,7 +1,7 @@ -void call(Map args = [:]){ - - withAWS(role: "${ARTIFACT_DOWNLOAD_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Download(file: args.destPath, bucket: args.bucket, path: args.path, force: args.force) +void call(Map args = [:]) { + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC')]) { + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(file: args.destPath, bucket: args.bucket, path: args.path, force: args.force) + } } - } diff --git a/vars/getManifestSHA.groovy b/vars/getManifestSHA.groovy index 23a810fde3..0ff2d7ea22 100644 --- a/vars/getManifestSHA.groovy +++ b/vars/getManifestSHA.groovy @@ -21,11 +21,14 @@ Map call(Map args = [:]) { echo "Manifest SHA path: ${manifestSHAPath}" Boolean manifestSHAExists = false - withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - if (s3DoesObjectExist(bucket: "${ARTIFACT_BUCKET_NAME}", path: manifestSHAPath)) { - manifestSHAExists = true + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + if (s3DoesObjectExist(bucket: "${ARTIFACT_BUCKET_NAME}", path: manifestSHAPath)) { + manifestSHAExists = true } } + } echo "Manifest SHA exists: ${manifestSHAExists}" diff --git a/vars/getRepositoryCommit.groovy b/vars/getRepositoryCommit.groovy new file mode 100644 index 0000000000..c93998eb3f --- /dev/null +++ b/vars/getRepositoryCommit.groovy @@ -0,0 +1,55 @@ +void call(Map args = [:]) { + + sh """#!/bin/bash + + set +x + set -e + + COMPONENT_LIST=() + COMPONENT_URL_LIST=() + COMPONENT_REF_LIST=() + + mkdir -p commits; cd commits + CURR_DIR=`pwd` + OUTPUT_FILE=${args.outputFile} + cp -v ${WORKSPACE}/${args.inputManifest} ${WORKSPACE}/\$OUTPUT_FILE + + if [ -z "${args.componentName}" ]; then + echo "Component list not specified so search the entire input manifest: ${WORKSPACE}/${args.inputManifest}" + read -r -a COMPONENT_LIST <<< `yq e '.components[].name' ${WORKSPACE}/${args.inputManifest} | tr '\n' ' '` + echo "Component list: \${COMPONENT_LIST[@]}" + else + echo "Specified component list: ${args.componentName}" + for comp in ${args.componentName}; do + comp_temp=`yq e ".components[] | select(.name == \\"\$comp\\") | .name" ${WORKSPACE}/${args.inputManifest} | tr '\n' ' ' | head -n 1` + if [ -z "\$comp_temp" ] || [ "\$comp_temp" = "null" ]; then + echo "ERROR: \$comp does not exist in manifest ${WORKSPACE}/${args.inputManifest}" + exit 1 + fi + done + read -r -a COMPONENT_LIST <<< "${args.componentName}" + echo "Component list: \${COMPONENT_LIST[@]}" + fi + + for entry in \${COMPONENT_LIST[@]}; do + COMPONENT_URL_LIST+=(`yq e ".components[] | select(.name == \\"\$entry\\") | .repository" ${WORKSPACE}/${args.inputManifest} | tr '\n' ' ' | head -n 1`) + COMPONENT_REF_LIST+=(`yq e ".components[] | select(.name == \\"\$entry\\") | .ref" ${WORKSPACE}/${args.inputManifest} | tr '\n' ' ' | head -n 1`) + done + + echo "Component url list: \${COMPONENT_URL_LIST[@]}" + echo "Component ref list: \${COMPONENT_REF_LIST[@]}" + + for index in \${!COMPONENT_LIST[@]}; do + cd \$CURR_DIR + mkdir -p \${COMPONENT_LIST[\$index]} + cd \${COMPONENT_LIST[\$index]} + git init -q + git remote add origin \${COMPONENT_URL_LIST[\$index]} + git fetch --depth 1 origin \${COMPONENT_REF_LIST[\$index]} + git checkout -q FETCH_HEAD + REAL_REF=`git rev-parse HEAD` + echo \$REAL_REF + yq -i ".components |= map(select(.name == \\"\${COMPONENT_LIST[\$index]}\\").ref=\\"\$REAL_REF\\")" ${WORKSPACE}/\$OUTPUT_FILE + done + """ +} diff --git a/vars/promoteArtifacts.groovy b/vars/promoteArtifacts.groovy index ea7af6c858..67f06873a9 100644 --- a/vars/promoteArtifacts.groovy +++ b/vars/promoteArtifacts.groovy @@ -20,90 +20,92 @@ void call(Map args = [:]) { String revision = version + qualifier println("Revision: ${revision}") - List distributionList = ["tar", "rpm"] - - for (distribution in distributionList) { - - // Must use local variable due to groovy for loop and closure scope - // Or the stage will fixed to the last item in return when trigger new stages - // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/ - def distribution_local = distribution - def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution_local}" - def prefixPath = "${WORKSPACE}/artifacts/${distribution_local}" - println("S3 download ${distribution_local} artifacts before creating signatures") - - withAWS(role: "${ARTIFACT_DOWNLOAD_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Download(bucket: "${ARTIFACT_BUCKET_NAME}", file: "${prefixPath}", path: "${artifactPath}/", force: true) - } - - String build_manifest = "$prefixPath/$artifactPath/builds/$filename/manifest.yml" - def buildManifest = readYaml(file: build_manifest) + List distributionList = ['tar', 'rpm'] + + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME'), + string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + for (distribution in distributionList) { + // Must use local variable due to groovy for loop and closure scope + // Or the stage will fixed to the last item in return when trigger new stages + // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/ + def distribution_local = distribution + def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution_local}" + def prefixPath = "${WORKSPACE}/artifacts/${distribution_local}" + println("S3 download ${distribution_local} artifacts before creating signatures") + + withAWS(role: 'opensearch-bundle', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(bucket: "${ARTIFACT_BUCKET_NAME}", file: "${prefixPath}", path: "${artifactPath}/", force: true) + } - print("Actions ${fileActions}") + String build_manifest = "$prefixPath/$artifactPath/builds/$filename/manifest.yml" + def buildManifest = readYaml(file: build_manifest) - argsMap = [:] - argsMap['sigtype'] = '.sig' + argsMap = [:] + argsMap['sigtype'] = '.sig' - String corePluginDir = "$prefixPath/$artifactPath/builds/$filename/core-plugins" - boolean corePluginDirExists = fileExists(corePluginDir) + String corePluginDir = "$prefixPath/$artifactPath/builds/$filename/core-plugins" + boolean corePluginDirExists = fileExists(corePluginDir) - //////////// Signing Artifacts - println("Signing Starts") + //////////// Signing Artifacts + println('Signing Starts') - if(corePluginDirExists && distribution_local.equals('tar')) { - println("Signing Core Plugins") - argsMap['artifactPath'] = corePluginDir - for (Closure action : fileActions) { - action(argsMap) + if (corePluginDirExists && distribution_local.equals('tar')) { + println('Signing Core Plugins') + argsMap['artifactPath'] = corePluginDir + for (Closure action : fileActions) { + action(argsMap) + } } - } - println("Signing Core/Bundle Artifacts") - String coreFullPath = ['core', filename, revision].join('/') - String bundleFullPath = ['bundle', filename, revision].join('/') - for (Closure action : fileActions) { - for (file in findFiles(glob: "**/${filename}-min-${revision}*.${distribution_local}*,**/${filename}-${revision}*.${distribution_local}*")) { - argsMap['artifactPath'] = "$WORKSPACE" + "/" + file.getPath() - action(argsMap) + println('Signing Core/Bundle Artifacts') + String coreFullPath = ['core', filename, revision].join('/') + String bundleFullPath = ['bundle', filename, revision].join('/') + for (Closure action : fileActions) { + for (file in findFiles(glob: "**/${filename}-min-${revision}*.${distribution_local}*,**/${filename}-${revision}*.${distribution_local}*")) { + argsMap['artifactPath'] = "$WORKSPACE" + '/' + file.getPath() + action(argsMap) + } } - } - //////////// Uploading Artifacts - withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - // Core Plugins only needs to be published once through Tar, ignore other distributions - if(corePluginDirExists && distribution_local.equals('tar')) { - List corePluginList = buildManifest.components.artifacts."core-plugins"[0] - for (String pluginSubPath : corePluginList) { - String pluginSubFolder = pluginSubPath.split('/')[0] - String pluginNameWithExt = pluginSubPath.split('/')[1] - String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '') - String pluginNameNoExt = pluginNameWithExt.replace('-' + revision, '') - String pluginFullPath = ['plugins', pluginName, revision].join('/') - s3Upload( + //////////// Uploading Artifacts + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + // Core Plugins only needs to be published once through Tar, ignore other distributions + if (corePluginDirExists && distribution_local.equals('tar')) { + List corePluginList = buildManifest.components.artifacts.'core-plugins'[0] + for (String pluginSubPath : corePluginList) { + String pluginSubFolder = pluginSubPath.split('/')[0] + String pluginNameWithExt = pluginSubPath.split('/')[1] + String pluginName = pluginNameWithExt.replace('-' + revision + '.zip', '') + String pluginNameNoExt = pluginNameWithExt.replace('-' + revision, '') + String pluginFullPath = ['plugins', pluginName, revision].join('/') + s3Upload( bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "releases/$pluginFullPath/", workingDir: "$prefixPath/$artifactPath/builds/$filename/core-plugins/", includePathPattern: "**/${pluginName}*" ) + } } - } - - // We will only publish min artifacts for Tar, ignore other distributions - if (distribution_local.equals('tar')) { - s3Upload( + + // We will only publish min artifacts for Tar, ignore other distributions + if (distribution_local.equals('tar')) { + s3Upload( bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "releases/$coreFullPath/", workingDir: "$prefixPath/$artifactPath/builds/$filename/dist/", includePathPattern: "**/${filename}-min-${revision}-${DISTRIBUTION_PLATFORM}-${DISTRIBUTION_ARCHITECTURE}*") - } + } - // We will publish bundle artifacts for all distributions - s3Upload( + // We will publish bundle artifacts for all distributions + s3Upload( bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "releases/$bundleFullPath/", workingDir: "$prefixPath/$artifactPath/dist/$filename/", includePathPattern: "**/${filename}-${revision}-${DISTRIBUTION_PLATFORM}-${DISTRIBUTION_ARCHITECTURE}*") - + } + } } - } } diff --git a/vars/promoteContainer.groovy b/vars/promoteContainer.groovy new file mode 100644 index 0000000000..2d3b8aa62d --- /dev/null +++ b/vars/promoteContainer.groovy @@ -0,0 +1,102 @@ +/**@ + * Promote image from staging docker to production docker hub or ECR repository. + * + * @param args A map of the following parameters + * @param args.imageRepository The repository of staging image. E.g.: opensearch:2.0.1.3910, opensearch-dashboards:2.0.1, data-prepper:2.0.1-1234 + * @param args.version The official version for release. E.g.: 2.0.1 + * @param args.dockerPromote The boolean argument if promote containers from staging to production docker repo. + * @param args.ecrPromote The boolean argument if promote containers from staging to production ECR repo. + * @param args.latestTag The boolean argument if promote containers from staging to production with latest tag. + * @param args.majorVersionTag The boolean argument if promote containers from staging to production with its major version tag. + */ +void call(Map args = [:]) { + + def imageRepo = args.imageRepository + def version = args.version + def imageProduct = imageRepo.split(':').first() + def sourceTag = imageRepo.split(':').last() + def dockerPromote = args.dockerPromote + def ecrPromote = args.ecrPromote + def latestBoolean = args.latestTag + def majorVersionBoolean = args.majorVersionTag + def majorVersion = version.split("\\.").first() + + def sourceReg = (imageProduct == 'data-prepper') ? "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}" : "opensearchstaging" + def dockerProduction = "opensearchproject" + def ecrProduction = "public.ecr.aws/opensearchproject" + + //Promoting docker images + if (dockerPromote.toBoolean()) { + println("Promoting $imageProduct to production docker hub with with $version tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: dockerProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:${version}") + ] + } + if (majorVersionBoolean.toBoolean()) { + println("Promoting to production docker hub with with $majorVersion tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: dockerProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:${majorVersion}") + ] + } + } + if (latestBoolean.toBoolean()) { + println("Promoting to production docker hub with with latest tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: dockerProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:latest") + ] + } + } + } + //Promoting image to ECR + if (ecrPromote.toBoolean()) { + println("Promoting to production ECR with with $version tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: ecrProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:${version}") + ] + } + if (majorVersionBoolean.toBoolean()) { + println("Promoting to production ECR with with $majorVersion tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: ecrProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:${majorVersion}") + ] + } + } + if (latestBoolean.toBoolean()) { + println("Promoting to production ECR with with latest tag.") + dockerCopy: { + build job: 'docker-copy', + parameters: [ + string(name: 'SOURCE_IMAGE_REGISTRY', value: sourceReg), + string(name: 'SOURCE_IMAGE', value: "${imageProduct}:${sourceTag}"), + string(name: 'DESTINATION_IMAGE_REGISTRY', value: ecrProduction), + string(name: 'DESTINATION_IMAGE', value: "${imageProduct}:latest") + ] + } + } + } +} diff --git a/vars/promoteYumRepos.groovy b/vars/promoteYumRepos.groovy index a18be4835e..f1e1614b8c 100644 --- a/vars/promoteYumRepos.groovy +++ b/vars/promoteYumRepos.groovy @@ -17,7 +17,7 @@ void call(Map args = [:]) { String buildnumber = args.buildNumber ?: 'none' if (buildnumber == 'none') { - println("User did not enter build number in jenkins parameter, exit 1") + println('User did not enter build number in jenkins parameter, exit 1') System.exit(1) } @@ -40,72 +40,73 @@ void call(Map args = [:]) { String yumRepoProdPath = "releases/bundle/${filename}/${yumRepoVersion}/yum" String artifactPath = "${localPath}/${yumRepoProdPath}" + withCredentials([string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME'), + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME')]) { + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println('Pulling Prod Yumrepo') + sh("aws s3 sync s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ ${artifactPath}/ --no-progress") + } - withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - println("Pulling Prod Yumrepo") - sh("aws s3 sync s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ ${artifactPath}/ --no-progress") - } - - sh """ - set -e - set +x - set +x + sh """ + set -e + set +x + set +x - echo "Pulling ${revision} rpms" - cd ${artifactPath} - curl -SLO ${stagingYumPathX64} - curl -SLO ${stagingYumPathARM64} + echo "Pulling ${revision} rpms" + cd ${artifactPath} + curl -SLO ${stagingYumPathX64} + curl -SLO ${stagingYumPathARM64} - ls -l + ls -l - rm -vf repodata/repomd.xml.asc + rm -vf repodata/repomd.xml.asc - echo "Update repo metadata" - createrepo --update . + echo "Update repo metadata" + createrepo --update . - # Rename .xml to .pom for signing - # Please do not add .xml to signer filter - # As maven have many .xml and we do not want to sign them - # This is an outlier case for yum repo only - mv -v repodata/repomd.xml repodata/repomd.pom + # Rename .xml to .pom for signing + # Please do not add .xml to signer filter + # As maven have many .xml and we do not want to sign them + # This is an outlier case for yum repo only + mv -v repodata/repomd.xml repodata/repomd.pom - echo "Complete metadata update, awaiting signing repomd.xml" + echo "Complete metadata update, awaiting signing repomd.xml" cd - """ - signArtifacts( - artifactPath: "${artifactPath}/repodata/repomd.pom", - sigtype: '.sig', - platform: 'linux' - ) + signArtifacts( + artifactPath: "${artifactPath}/repodata/repomd.pom", + sigtype: '.sig', + platform: 'linux' + ) - sh """ - set -e - set +x + sh """ + set -e + set +x - cd ${artifactPath}/repodata/ + cd ${artifactPath}/repodata/ - ls -l + ls -l - mv -v repomd.pom repomd.xml - mv -v repomd.pom.sig repomd.xml.sig + mv -v repomd.pom repomd.xml + mv -v repomd.pom.sig repomd.xml.sig - # This step is required as yum only accept .asc and signing workflow only support .sig - cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc + # This step is required as yum only accept .asc and signing workflow only support .sig + cat repomd.xml.sig | gpg --enarmor | sed 's@ARMORED FILE@SIGNATURE@g' > repomd.xml.asc - rm -vf repomd.xml.sig + rm -vf repomd.xml.sig - ls -l - - cd - + ls -l + cd - """ - withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { - println("Pushing Prod Yumrepo") - sh("aws s3 sync ${artifactPath}/ s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ --no-progress") - } - + withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') { + println('Pushing Prod Yumrepo') + sh("aws s3 sync ${artifactPath}/ s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${yumRepoProdPath}/ --no-progress") + } + } } diff --git a/vars/runGradleCheck.groovy b/vars/runGradleCheck.groovy index ae52a02c62..3705fbebd7 100644 --- a/vars/runGradleCheck.groovy +++ b/vars/runGradleCheck.groovy @@ -16,12 +16,11 @@ void call(Map args = [:]) { usernamePassword(credentialsId: "jenkins-gradle-check-s3-aws-resources", usernameVariable: 'amazon_s3_base_path', passwordVariable: 'amazon_s3_bucket')]) { sh """ + #!/bin/bash set -e set +x - env | grep JAVA | grep HOME - echo "Git clone: ${git_repo_url} with ref: ${git_reference}" rm -rf search git clone ${git_repo_url} search @@ -29,9 +28,24 @@ void call(Map args = [:]) { git checkout -f ${git_reference} git rev-parse HEAD - echo "Stop existing gradledaemon" + echo "Get Major Version" + OS_VERSION=`cat buildSrc/version.properties | grep opensearch | cut -d= -f2 | grep -oE '[0-9.]+'` + OS_MAJOR_VERSION=`echo \$OS_VERSION | grep -oE '[0-9]+' | head -n 1` + echo "Version: \$OS_VERSION, Major Version: \$OS_MAJOR_VERSION" + + if [ "\$OS_MAJOR_VERSION" -lt 2 ]; then + echo "Using JAVA 11" + export JAVA_HOME=\$JAVA11_HOME + else + echo "Using JAVA 17" + export JAVA_HOME=\$JAVA17_HOME + fi + + env | grep JAVA | grep HOME + + echo "Gradle clean cache and stop existing gradledaemon" ./gradlew --stop - find ~/.gradle -type f -name "*.lock" -delete + rm -rf ~/.gradle echo "Check existing dockercontainer" docker ps -a @@ -43,9 +57,15 @@ void call(Map args = [:]) { echo "Check docker-compose version" docker-compose version + echo "Check existing processes" + ps -ef | grep [o]pensearch | wc -l + echo "Cleanup existing processes" + kill -9 `ps -ef | grep [o]pensearch | awk '{print \$2}'` > /dev/null 2>&1 || echo + ps -ef | grep [o]pensearch | wc -l + echo "Start gradlecheck" GRADLE_CHECK_STATUS=0 - ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1 + ./gradlew clean && ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1 if [ "\$GRADLE_CHECK_STATUS" != 0 ]; then echo Gradle Check Failed! diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy index 550ad42c6e..6f31fa537f 100644 --- a/vars/runPerfTestScript.groovy +++ b/vars/runPerfTestScript.groovy @@ -4,9 +4,12 @@ void call(Map args = [:]) { install_opensearch_infra_dependencies() config_name = isNullOrEmpty(args.config) ? "config.yml" : args.config - withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/${config_name}", force: true) - } + withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), + string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { + withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/${config_name}", force: true) + } + } String stackNameSuffix = isNullOrEmpty(args.stackNameSuffix) ? 'perf-test' : args.stackNameSuffix diff --git a/vars/signArtifacts.groovy b/vars/signArtifacts.groovy index 491ddac51d..531eb51b2b 100644 --- a/vars/signArtifacts.groovy +++ b/vars/signArtifacts.groovy @@ -15,15 +15,23 @@ SignArtifacts signs the given artifacts and saves the signature in the same dire @param Map[platform] - The distribution platform for signing. */ void call(Map args = [:]) { - if (args.sigtype.equals('.rpm')) { - echo "RPM Add Sign" - - withAWS(role: "${SIGN_ASM_ROLE}", roleAccount: "${SIGN_ASM_ACCOUNT}", duration: 900, roleSessionName: 'jenkins-signing-session') { - withCredentials([ - string(credentialsId: 'jenkins-rpm-signing-asm-pass-id', variable: 'SIGNING_PASS_ID'), - string(credentialsId: 'jenkins-rpm-signing-asm-secret-id', variable: 'SIGNING_SECRET_ID')]) - { + withCredentials([ + string(credentialsId: 'jenkins-rpm-signing-account-number', variable: 'RPM_SIGNING_ACCOUNT_NUMBER'), + string(credentialsId: 'jenkins-rpm-signing-passphrase-secrets-arn', variable: 'RPM_SIGNING_PASSPHRASE_SECRETS_ARN'), + string(credentialsId: 'jenkins-rpm-signing-secret-key-secrets-arn', variable: 'RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN'), + string(credentialsId: 'jenkins-rpm-signing-key-id', variable: 'RPM_SIGNING_KEY_ID')]) { + echo 'RPM Add Sign' + +<<<<<<< HEAD +<<<<<<< HEAD + withAWS(role: 'jenkins-prod-rpm-signing-assume-role', roleAccount: "${RPM_SIGNING_ACCOUNT_NUMBER}", duration: 900, roleSessionName: 'jenkins-signing-session') { +======= + withAWS(role: 'jenkins-prod-rpm-signing-assume-role', roleAccount: "${signingAccount}", duration: 900, roleSessionName: 'jenkins-signing-session') { +>>>>>>> 21fe974 (Change rpm signing role with new one (#2352)) +======= + withAWS(role: 'jenkins-prod-rpm-signing-assume-role', roleAccount: "${RPM_SIGNING_ACCOUNT_NUMBER}", duration: 900, roleSessionName: 'jenkins-signing-session') { +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) sh """ set -e set +x @@ -61,8 +69,8 @@ void call(Map args = [:]) { echo "------------------------------------------------------------------------" echo "Import OpenSearch keys" - aws secretsmanager get-secret-value --region "${SIGN_ASM_REGION}" --secret-id "${SIGNING_PASS_ID}" | jq -r .SecretBinary | base64 --decode > passphrase - aws secretsmanager get-secret-value --region "${SIGN_ASM_REGION}" --secret-id "${SIGNING_SECRET_ID}" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - + aws secretsmanager get-secret-value --region us-west-2 --secret-id "${RPM_SIGNING_PASSPHRASE_SECRETS_ARN}" | jq -r .SecretBinary | base64 --decode > passphrase + aws secretsmanager get-secret-value --region us-west-2 --secret-id "${RPM_SIGNING_SECRET_KEY_ID_SECRETS_ARN}" | jq -r .SecretBinary | base64 --decode | gpg --quiet --import --pinentry-mode loopback --passphrase-file passphrase - echo "------------------------------------------------------------------------" echo "Start Signing Rpm" @@ -89,56 +97,124 @@ void call(Map args = [:]) { echo "------------------------------------------------------------------------" echo "Clean up gpg" - gpg --batch --yes --delete-secret-keys $SIGN_ASM_KEYID - gpg --batch --yes --delete-keys $SIGN_ASM_KEYID + gpg --batch --yes --delete-secret-keys ${RPM_SIGNING_KEY_ID} + gpg --batch --yes --delete-keys ${RPM_SIGNING_KEY_ID} rm -v passphrase """ - - } + } } - } else { - echo "PGP Signature Signing" - - if( !fileExists("$WORKSPACE/sign.sh")) { +<<<<<<< HEAD +<<<<<<< HEAD + echo 'PGP or Windows Signature Signing' +======= + echo "PGP or Windows Signature Signing" +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + echo 'PGP or Windows Signature Signing' +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + + if (!fileExists("$WORKSPACE/sign.sh")) { git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main' } importPGPKey() - + String arguments = generateArguments(args) // Sign artifacts - withCredentials([usernamePassword(credentialsId: "${GITHUB_BOT_TOKEN_NAME}", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN')]) { +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + // def configSecret = args.platform == "windows" ? "jenkins-signer-windows-config" : "jenkins-signer-client-creds" + if (args.platform == 'windows') { + withCredentials([usernamePassword(credentialsId: "${GITHUB_BOT_TOKEN_NAME}", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN'), + string(credentialsId: 'jenkins-signer-windows-role', variable: 'SIGNER_WINDOWS_ROLE'), + string(credentialsId: 'jenkins-signer-windows-external-id', variable: 'SIGNER_WINDOWS_EXTERNAL_ID'), + string(credentialsId: 'jenkins-signer-windows-unsigned-bucket', variable: 'SIGNER_WINDOWS_UNSIGNED_BUCKET'), + string(credentialsId: 'jenkins-signer-windows-signed-bucket', variable: 'SIGNER_WINDOWS_SIGNED_BUCKET'), + string(credentialsId: 'jenkins-signer-windows-profile-identifier', variable: 'SIGNER_WINDOWS_PROFILE_IDENTIFIER'), + string(credentialsId: 'jenkins-signer-windows-platform-identifier', variable: 'SIGNER_WINDOWS_PLATFORM_IDENTIFIER')]) { + sh """ + #!/bin/bash + set +x + export ROLE=$SIGNER_WINDOWS_ROLE + export EXTERNAL_ID=$SIGNER_WINDOWS_EXTERNAL_ID + export UNSIGNED_BUCKET=$SIGNER_WINDOWS_UNSIGNED_BUCKET + export SIGNED_BUCKET=$SIGNER_WINDOWS_SIGNED_BUCKET + export PROFILE_IDENTIFIER=$SIGNER_WINDOWS_PROFILE_IDENTIFIER + export PLATFORM_IDENTIFIER=$SIGNER_WINDOWS_PLATFORM_IDENTIFIER + + $WORKSPACE/sign.sh ${arguments} + """ + } + } + else { + withCredentials([usernamePassword(credentialsId: "${GITHUB_BOT_TOKEN_NAME}", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN'), + string(credentialsId: 'jenkins-signer-client-role', variable: 'SIGNER_CLIENT_ROLE'), + string(credentialsId: 'jenkins-signer-client-external-id', variable: 'SIGNER_CLIENT_EXTERNAL_ID'), + string(credentialsId: 'jenkins-signer-client-unsigned-bucket', variable: 'SIGNER_CLIENT_UNSIGNED_BUCKET'), + string(credentialsId: 'jenkins-signer-client-signed-bucket', variable: 'SIGNER_CLIENT_SIGNED_BUCKET')]) { + sh """ +<<<<<<< HEAD + #!/bin/bash + set +x + export ROLE=$SIGNER_CLIENT_ROLE + export EXTERNAL_ID=$SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=$SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=$SIGNER_CLIENT_SIGNED_BUCKET + + $WORKSPACE/sign.sh ${arguments} + """ + } +======= + def configSecret = args.platform == "windows" ? "signer-windows-config" : "signer-pgp-config" +======= + def configSecret = args.platform == "windows" ? "jenkins-signer-windows-config" : "jenkins-signer-client-creds" +>>>>>>> 932dbb4 (Add jenkins prefix to signer credentials (#2342)) + withCredentials([usernamePassword(credentialsId: "${GITHUB_BOT_TOKEN_NAME}", usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_TOKEN'), + string(credentialsId: configSecret, variable: 'configs')]) { + def creds = readJSON(text: configs) + def ROLE = creds['role'] + def EXTERNAL_ID = creds['external_id'] + def UNSIGNED_BUCKET = creds['unsigned_bucket'] + def SIGNED_BUCKET = creds['signed_bucket'] + def PROFILE_IDENTIFIER = creds['profile_identifier'] + def PLATFORM_IDENTIFIER = creds['platform_identifier'] sh """ - #!/bin/bash - set +x - export ROLE=${SIGNER_CLIENT_ROLE} - export EXTERNAL_ID=${SIGNER_CLIENT_EXTERNAL_ID} - export UNSIGNED_BUCKET=${SIGNER_CLIENT_UNSIGNED_BUCKET} - export SIGNED_BUCKET=${SIGNER_CLIENT_SIGNED_BUCKET} - - $WORKSPACE/sign.sh ${arguments} - """ +======= +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) + #!/bin/bash + set +x + export ROLE=$SIGNER_CLIENT_ROLE + export EXTERNAL_ID=$SIGNER_CLIENT_EXTERNAL_ID + export UNSIGNED_BUCKET=$SIGNER_CLIENT_UNSIGNED_BUCKET + export SIGNED_BUCKET=$SIGNER_CLIENT_SIGNED_BUCKET + + $WORKSPACE/sign.sh ${arguments} + """ +<<<<<<< HEAD +>>>>>>> 700c80d (Add signer to support signing windows artifacts (#2156)) +======= + } +>>>>>>> 528cdfc (Move clubbed secrets to individual secrets (#2356)) } - } } String generateArguments(args) { - String artifactPath = args.remove("artifactPath") + String artifactPath = args.remove('artifactPath') // artifactPath is mandatory and the first argument String arguments = artifactPath // generation command line arguments - args.each{key, value -> arguments += " --${key}=${value}"} + args.each { key, value -> arguments += " --${key }=${value }" } return arguments } -void importPGPKey(){ - - sh "curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -" - +void importPGPKey() { + sh 'curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -' } - diff --git a/vars/uploadArtifacts.groovy b/vars/uploadArtifacts.groovy index f573b7a150..74ef0981cf 100644 --- a/vars/uploadArtifacts.groovy +++ b/vars/uploadArtifacts.groovy @@ -12,7 +12,8 @@ void call(Map args = [:]) { withCredentials([ string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME'), string(credentialsId: 'jenkins-artifact-production-bucket-name', variable: 'ARTIFACT_PRODUCTION_BUCKET_NAME'), - string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'),]) { + string(credentialsId: 'jenkins-aws-production-account', variable: 'AWS_ACCOUNT_ARTIFACT'), + string(credentialsId: 'jenkins-artifact-promotion-role', variable: 'ARTIFACT_PROMOTION_ROLE_NAME')]) { echo "Uploading to s3://${ARTIFACT_BUCKET_NAME}/${artifactPath}" uploadToS3(