test

Signed-off-by: Oliver Koenig <okoenig@nvidia.com>
NVIDIA · Sep 20, 2024 · e23faa4 · e23faa4
1 parent 447eb32
commit e23faa4
Show file tree

Hide file tree

Showing 2 changed files with 118 additions and 68 deletions.
diff --git a/.github/workflows/_build_container.yml b/.github/workflows/_build_container.yml
@@ -14,19 +14,26 @@
 name: Build NeMo Aligner container
 on: 
   workflow_call:
+    inputs:
+      ref: 
+        description: Git ref to checkout
+        default: ${{ github.sha }}
+        required: false
+        type: string
 
 defaults:
   run:
     shell: bash -x -e -u -o pipefail {0}
 
 jobs:
-  build:
+  main:
     runs-on: self-hosted-builder
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: ${{ github.run_id }}
+          ref: ${{ inputs.sha }}
 
       - name: Clean runner cache
         run: | 

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -24,67 +24,36 @@ on:
         description: Branch to release
         required: true
         type: string
+  # to be removed later:
+  pull_request:
+
 jobs: 
-  main:
-    if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+  build-container:
+    if: ${{ github.event.label.name == 'Run CICD' || github.ref == 'refs/heads/main' }}
+    uses: ./.github/workflows/_build_container.yml
+    with: 
+      ref: ${{ inputs.branch || github.sha }}
+
+  build-and-test-wheel:
+    needs: build-container
     runs-on: ubuntu-latest
-    environment: 
-      name: main
-    steps:  
+    container: nemoci.azurecr.io/nemo_aligner_container:${{ github.run_id }}
+    steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           path: ${{ github.run_id }}
-          ref: ${{ inputs.branch }}
+          ref: ${{ inputs.branch || github.sha }}
 
-      - name: Create release
-        id: version-number
-        run: |
-          cd ${{ github.run_id }}
-          VERSION=$(python -c "import nemo_aligner; print(nemo_aligner.__version__)")
-          
-          NAME="NVIDIA NeMo-Aligner ${VERSION}"
-          CHANGELOG=$(awk '/^## '"$NAME"'/{flag=1; next} /^## /{flag=0} flag' CHANGELOG.md)
-          CHANGELOG=$(echo "$CHANGELOG" | sed '/./,$!d' | sed ':a;N;$!ba;s/\n$//')
-
-          PAYLOAD=$(jq \
-                      -n \
-                      -c \
-                      --arg TAG_NAME "v${VERSION}" \
-                      --arg CI_COMMIT_BRANCH "${{ inputs.branch }}" \
-                      --arg NAME "$NAME" \
-                      --arg BODY "$CHANGELOG" \
-                      '{
-                        "tag_name": $TAG_NAME,
-                        "target_commitish": $CI_COMMIT_BRANCH,
-                        "name": $NAME,
-                        "body": $BODY,
-                        "draft": false,
-                        "prerelease": false,
-                        "generate_release_notes": false
-                      }'
-                  )
-
-          curl -L \
-            -X POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${{ secrets.PAT }}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://github.com/gitapi/repos/NVIDIA/NeMo-Aligner/releases \
-            -d "$PAYLOAD"
-      
-      - name: Build, test, and release wheel
-        env:
-          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      - name: Build and test wheel
         run: |
           cd ${{ github.run_id }}
           EXPECTED_VERSION=$(python -c 'import nemo_aligner; print(nemo_aligner.__version__)')
 
           python3 -m pip install --upgrade build
           python3 -m build
 
-          pip install --no-deps dist/*.whl
+          pip install dist/*.whl
 
           cd ../
 
@@ -94,26 +63,100 @@ jobs:
             echo 'Wheel has an outdated version, mission abort immediately!'
             exit 1
           fi
+        
+      - name: Upload wheel 
+        uses: actions/upload-artifact@v4
+        with:
+          name: pip-wheel
+          path: ${{ github.run_id }}/dist/
 
-          echo Proceed with uploading wheel...
-          cd ${{ github.run_id }}
-          python3 -m pip install --upgrade twine
-          python3 -m twine upload --repository pypi dist/*
+  # publish-wheel:
+  #   needs: build-and-test-wheel
+  #   if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+  #   runs-on: ubuntu-latest
+  #   environment: 
+  #     name: main
+  #   steps:  
+  #     - name: Download wheel
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: pip-wheel
+  #         path: ${{ github.run_id }}/dist/
+
+  #     - name: Release wheel
+  #       env:
+  #         TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+  #         TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+  #       run: |
+  #         python3 -m pip install --upgrade twine
+  #         python3 -m twine upload --repository pypi ${{ github.run_id }}/dist/*
 
-      - name: notify
-        run: |
+  # create-gh-release:
+  #   needs: publish-wheel
+  #   runs-on: ubuntu-latest
+  #   if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
+  #       with:
+  #         path: ${{ github.run_id }}
+  #         ref: ${{ inputs.branch }}
+
+  #     - name: Create release
+  #       id: version-number
+  #       run: |
+  #         cd ${{ github.run_id }}
+  #         VERSION=$(python -c "import nemo_aligner; print(nemo_aligner.__version__)")
+
+  #         NAME="NVIDIA NeMo-Aligner ${VERSION}"
+  #         CHANGELOG=$(awk '/^## '"$NAME"'/{flag=1; next} /^## /{flag=0} flag' CHANGELOG.md)
+  #         CHANGELOG=$(echo "$CHANGELOG" | sed '/./,$!d' | sed ':a;N;$!ba;s/\n$//')
+
+  #         PAYLOAD=$(jq \
+  #                     -n \
+  #                     -c \
+  #                     --arg TAG_NAME "v${VERSION}" \
+  #                     --arg CI_COMMIT_BRANCH "${{ inputs.branch }}" \
+  #                     --arg NAME "$NAME" \
+  #                     --arg BODY "$CHANGELOG" \
+  #                     '{
+  #                       "tag_name": $TAG_NAME,
+  #                       "target_commitish": $CI_COMMIT_BRANCH,
+  #                       "name": $NAME,
+  #                       "body": $BODY,
+  #                       "draft": false,
+  #                       "prerelease": false,
+  #                       "generate_release_notes": false
+  #                     }'
+  #                 )
+
+  #         curl -L \
+  #           -X POST \
+  #           -H "Accept: application/vnd.github+json" \
+  #           -H "Authorization: Bearer ${{ secrets.PAT }}" \
+  #           -H "X-GitHub-Api-Version: 2022-11-28" \
+  #           https://github.com/gitapi/repos/NVIDIA/NeMo-Aligner/releases \
+  #           -d "$PAYLOAD"
+
+  # notify:
+  #   needs: publish-wheel
+  #   runs-on: ubuntu-latest
+  #   if: contains(fromJSON('["terryk", "gshennvm", "ko3n1g"]'), github.actor)
+  #   steps:
+  #     - name: notify
+  #       run: |
+
+  #         URL="https://github.com/NVIDIA/NeMo-Aligner/releases/tag/v${{ steps.version-number.outputs.VERSION }}"
+  #         MESSAGE='{
+  #           "blocks": [
+  #             {
+  #               "type": "section",
+  #               "text": {
+  #                 "type": "mrkdwn",
+  #                 "text": "Releasebot 🤖: NeMo-Aligner released <'$URL'|`${{ steps.version-number.outputs.VERSION }}>` 🚀"
+  #               }
+  #             }
+  #           ]
+  #         }'
 
-          URL="https://github.com/NVIDIA/NeMo-Aligner/releases/tag/v${{ steps.version-number.outputs.VERSION }}"
-          MESSAGE='{
-            "blocks": [
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "Releasebot 🤖: NeMo-Aligner released <'$URL'|`${{ steps.version-number.outputs.VERSION }}>` 🚀"
-                }
-              }
-            ]
-          }'
-
-          curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }}
+  #         curl -X POST -H "Content-type: application/json" --data "$MESSAGE" ${{ secrets.SLACK_RELEASE_ENDPOINT }}