install the onnxruntime explicitly to support ORT 1.15 and nightly bu…

…ild (#593)

## Describe your changes
Remove call for make in CI pipelines to support install different ORT
including nightly

## Checklist before requesting a review
- [ ] Add unit tests for this change.
- [ ] Make sure all tests can pass.
- [ ] Update documents if necessary.
- [ ] Format your code by running `pre-commit run --all-files`
- [ ] Is this a user-facing change? If yes, give a description of this
change to be included in the release notes.

## (Optional) Issue link

---------

Co-authored-by: Jambay Kinley <jambaykinley@microsoft.com>

.azure_pipelines/job_templates/olive-build-doc-template.yaml

@@ -27,7 +27,7 @@ jobs:
       displayName: Checkout release branch
       condition: ne('${{parameters.doc_version}}', 'latest')
 
-    - script: make install-olive INSTALL_EXTRAS=[cpu]
+    - script: python -m pip install .[cpu]
       displayName: Install Olive
 
     - script: |

.azure_pipelines/job_templates/olive-build-template.yaml

@@ -4,88 +4,23 @@ parameters:
   name: ''
   pool: ''
   device: 'cpu'
+  onnxruntime: 'onnxruntime'
 
 jobs:
 - template: olive-test-template.yaml
   parameters:
     name: ${{parameters.name}}_Unit_Test_Olive
     pool: ${{parameters.pool}}
     device: ${{parameters.device}}
-    WINDOWS: ${{parameters.windows}}
+    windows: ${{parameters.windows}}
     test_type: 'unit_test'
+    onnxruntime: ${{parameters.onnxruntime}}
 
 - template: olive-test-template.yaml
   parameters:
     name: ${{parameters.name}}_Integration_Test_Olive
     pool: ${{parameters.pool}}
     device: ${{parameters.device}}
-    WINDOWS: ${{parameters.windows}}
+    windows: ${{parameters.windows}}
     test_type: 'integ_test'
-
-- job: ${{parameters.name}}_Test_Examples
-  timeoutInMinutes: 300
-  pool:
-    name: ${{ parameters.pool}}
-  strategy:
-    matrix:
-      ${{ insert }}: ${{ parameters.examples }}
-  variables:
-    WINDOWS: ${{ parameters.windows }}
-    runCodesignValidationInjection: false
-    device: ${{ parameters.device }}
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: 3.8
-    displayName: Use Python 3.8
-
-  - script: make install-olive INSTALL_EXTRAS=[$(device)]
-    displayName: Install Olive
-
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
-      scriptLocation: 'inlineScript'
-      inlineScript: make test-examples
-    displayName: Test Examples
-    env:
-      OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
-      WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
-      WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
-      WORKSPACE_NAME: $(workspace-name)
-      AZURE_TENANT_ID: $(azure-tenant-id)
-      AZURE_CLIENT_ID: $(olive-rg-sp-id)
-      AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
-      EXAMPLE_FOLDER: $(exampleFolder)
-      EXAMPLE_NAME: $(exampleName)
-
-  - task: ComponentGovernanceComponentDetection@0
-    inputs:
-      scanType: 'Register'
-      verbosity: 'Verbose'
-      alertWarningLevel: 'High'
-    displayName: Component Detection
-
-  - task: PublishTestResults@2
-    condition: succeededOrFailed()
-    inputs:
-      testResultsFiles: '**/*TestOlive*.xml'
-      testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
-    displayName: Upload pipeline run test results
-
-  - script: make clean WINDOWS=$(WINDOWS)
-    condition: always()
-    displayName: Clean remaining artifacts
-
-- template: olive-test-template.yaml
-  parameters:
-    name: ${{parameters.name}}_Multiple_EP_Test_Olive
-    pool: ${{parameters.pool}}
-    device: ${{parameters.device}}
-    WINDOWS: ${{parameters.windows}}
-    test_type: 'multiple_ep'
-    ${{ if eq(parameters.windows, 'True') }}:
-      python_version: '3.10'
-    ${{ else }}:
-      python_version: '3.8'
+    onnxruntime: ${{parameters.onnxruntime}}

.azure_pipelines/job_templates/olive-example-template.yaml

@@ -0,0 +1,74 @@
+# Olive Build and Test Pipeline template for examples on Azure DevOps
+
+parameters:
+  name: ''
+  pool: ''
+  python_version: '3.8'
+  onnxruntime: 'onnxruntime'
+
+jobs:
+  - job: ${{parameters.name}}_Test_Examples
+    timeoutInMinutes: 300
+    pool:
+      name: ${{ parameters.pool}}
+    strategy:
+      matrix:
+        ${{ insert }}: ${{ parameters.examples }}
+    variables:
+      runCodesignValidationInjection: false
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.python_version }}
+      displayName: Use Python ${{ parameters.python_version }}
+
+    - script: python -m pip install .
+      displayName: Install Olive
+
+    - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}:
+      - script: |
+          pip install onnxruntime
+          pip uninstall -y onnxruntime
+          pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+        displayName: Install ${{ parameters.onnxruntime }}
+    - ${{ else }}:
+      - script: |
+          pip install ${{ parameters.onnxruntime }}
+        displayName: Install ${{ parameters.onnxruntime }}
+
+    - task: AzureCLI@1
+      inputs:
+        azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
+        scriptLocation: 'inlineScript'
+        inlineScript: |
+          python -m pip install pytest
+          python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/requirements.txt
+          python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py
+      displayName: Test Examples
+      env:
+        OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
+        WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+        WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+        WORKSPACE_NAME: $(workspace-name)
+        AZURE_TENANT_ID: $(azure-tenant-id)
+        AZURE_CLIENT_ID: $(olive-rg-sp-id)
+        AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
+
+    - task: ComponentGovernanceComponentDetection@0
+      inputs:
+        scanType: 'Register'
+        verbosity: 'Verbose'
+        alertWarningLevel: 'High'
+      displayName: Component Detection
+
+    - task: PublishTestResults@2
+      condition: succeededOrFailed()
+      inputs:
+        testResultsFiles: '**/*TestOlive*.xml'
+        testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
+      displayName: Upload pipeline run test results
+
+    - script: git clean -dfX
+      condition: always()
+      displayName: Clean remaining artifacts

.azure_pipelines/job_templates/olive-test-template.yaml

@@ -4,17 +4,17 @@ parameters:
   test_type: ''
   device: 'cpu'
   python_version: '3.8'
+  onnxruntime: 'onnxruntime'
 
 jobs:
 - ${{ if eq(parameters.device, 'cpu') }}:
-  - job: ${{parameters.name}}_Test_Olive
+  - job: ${{parameters.name}}
     timeoutInMinutes: 300
     pool:
       name: ${{ parameters.pool}}
     variables:
       WINDOWS: ${{ parameters.windows}}
       runCodesignValidationInjection: false
-      device: ${{ parameters.device }}
       testType: ${{ parameters.test_type }}
       python_version: ${{ parameters.python_version }}
 
@@ -29,23 +29,64 @@ jobs:
       condition: and(eq(variables.WINDOWS, 'False'), eq(variables.testType, 'integ_test'))
       continueOnError: true
 
-    - script: make install-olive INSTALL_EXTRAS=[$(device)]
+    - script: |
+        python -m pip install .
       displayName: Install Olive
 
-    - task: AzureCLI@1
-      inputs:
-        azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
-        scriptLocation: 'inlineScript'
-        inlineScript: make $(testType)
-      displayName: Test Olive
-      env:
-        OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
-        WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
-        WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
-        WORKSPACE_NAME: $(workspace-name)
-        AZURE_TENANT_ID: $(azure-tenant-id)
-        AZURE_CLIENT_ID: $(olive-rg-sp-id)
-        AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
+    - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}:
+      - script: |
+          pip install onnxruntime
+          pip uninstall -y onnxruntime
+          pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+        displayName: Install ${{ parameters.onnxruntime }}
+    - ${{ else }}:
+      - script: |
+          pip install ${{ parameters.onnxruntime }}
+        displayName: Install ${{ parameters.onnxruntime }}
+
+    - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            call python -m pip install pytest
+            call curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip
+            call 7z x openvino_toolkit.zip
+            call w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat
+            call python -m pip install numpy psutil coverage protobuf==3.20.3
+            call coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType)
+            call coverage xml
+        displayName: Test Olive
+        env:
+          OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
+          WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+          WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+          WORKSPACE_NAME: $(workspace-name)
+          AZURE_TENANT_ID: $(azure-tenant-id)
+          AZURE_CLIENT_ID: $(olive-rg-sp-id)
+          AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
+    - ${{ else }}:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            python -m pip install pytest
+            python -m pip install -r $(Build.SourcesDirectory)/test/requirements-test.txt
+
+            coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType)
+            coverage xml
+
+        displayName: Test Olive
+        env:
+          OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
+          WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+          WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+          WORKSPACE_NAME: $(workspace-name)
+          AZURE_TENANT_ID: $(azure-tenant-id)
+          AZURE_CLIENT_ID: $(olive-rg-sp-id)
+          AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
 
     - task: CredScan@3
       displayName: 'Run CredScan'
@@ -72,6 +113,6 @@ jobs:
         codeCoverageTool: 'Cobertura'
         summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
 
-    - script: make clean WINDOWS=$(WINDOWS)
+    - script: git clean -dfX
       condition: always()
       displayName: Clean remaining artifacts

.azure_pipelines/olive-ci.yaml

@@ -27,11 +27,28 @@ variables:
     ComponentDetection.Timeout: 2400
 
 jobs:
+# Linux unit test and integration test
 - template: job_templates/olive-build-template.yaml
   parameters:
     name: Linux_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
     windows: False
+    onnxruntime: onnxruntime
+
+# Windows unit test and integration test
+- template: job_templates/olive-build-template.yaml
+  parameters:
+    name: Windows_CI
+    pool: $(OLIVE_POOL_WIN2019)
+    windows: True
+    onnxruntime: onnxruntime
+
+# Linux examples test
+- template: job_templates/olive-example-template.yaml
+  parameters:
+    name: Linux_CI
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    onnxruntime: onnxruntime
     examples:
       bert_ptq_cpu:
         exampleFolder: bert
@@ -58,11 +75,12 @@ jobs:
         exampleFolder: whisper
         exampleName: whisper
 
-- template: job_templates/olive-build-template.yaml
+# # Windows examples test
+- template: job_templates/olive-example-template.yaml
   parameters:
     name: Windows_CI
     pool: $(OLIVE_POOL_WIN2019)
-    windows: True
+    onnxruntime: onnxruntime
     examples:
       bert_ptq_cpu:
         exampleFolder: bert
@@ -83,17 +101,40 @@ jobs:
         exampleFolder: whisper
         exampleName: whisper
 
-- template: job_templates/olive-build-template.yaml
+# Linux GPU examples testing.
+- template: job_templates/olive-example-template.yaml
   parameters:
     name: Linux_GPU_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
-    windows: False
-    device: gpu
+    onnxruntime: onnxruntime-gpu
     examples:
       bert_cuda_gpu:
         exampleFolder: bert
         exampleName: bert_cuda_gpu
 
+# Multiple EP Linux testing
+- template: job_templates/olive-test-template.yaml
+  parameters:
+    name: Linux_CI_Multiple_EP_Test_Olive
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    device: 'cpu'
+    windows: 'False'
+    test_type: 'multiple_ep'
+    onnxruntime: onnxruntime
+    python_version: '3.8'
+
+# Multiple EP Windows testing
+- template: job_templates/olive-test-template.yaml
+  parameters:
+    name: Windows_CI_Multiple_EP_Test_Olive
+    pool: $(OLIVE_POOL_WIN2019)
+    device: 'cpu'
+    windows: 'True'
+    test_type: 'multiple_ep'
+    onnxruntime: onnxruntime
+    python_version: '3.10'
+
+# build documents
 - template: job_templates/olive-build-doc-template.yaml
   parameters:
     job_name: Test_BuildDocs