Add Onnxruntime parity checks to CI

Dockerfile

@@ -117,7 +117,9 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR
     /bin/sh /onnxruntime/dockerfiles/scripts/install_common_deps.sh
 
 
-ADD tools/build_and_test_onnxrt.sh /onnxruntime/build_and_test_onnxrt.sh
+ADD tools/build_and_install_onnxrt.sh /onnxruntime/build_and_install_onnxrt.sh
+ADD tools/test_onnxrt_unit_tests.sh /onnxruntime/test_onnxrt_unit_tests.sh
+ADD tools/test_onnxrt_parity_checks.sh /onnxruntime/test_onnxrt_parity_checks.sh
 
 ENV MIOPEN_FIND_DB_PATH=/tmp/miopen/find-db
 ENV MIOPEN_USER_DB_PATH=/tmp/miopen/user-db
@@ -127,3 +129,8 @@ ENV LD_LIBRARY_PATH=$PREFIX/lib
 ENV UBSAN_OPTIONS=print_stacktrace=1
 ENV ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
 RUN ln -s /opt/rocm/llvm/bin/llvm-symbolizer /usr/bin/llvm-symbolizer
+
+#install dependancies used for parity checks
+RUN pip3 install psutil==5.9.5 onnx==1.10.2 coloredlogs==15.0.1 packaging==23.1 transformers==4.29.2 sympy==1.12
+RUN pip3 install -U numpy==1.21.6
+

Jenkinsfile

@@ -159,15 +159,46 @@ def onnxnode(name, body) {
     }
 }
 
-rocmtest onnx: onnxnode('mi100+') { cmake_build ->
-    stage("Onnx runtime") {
+rocmtest onnxbuild: onnxnode('anygpu') { cmake_build ->
+    stage("Onnxruntime Build and Install") {
         sh '''
             apt install half
             #ls -lR
             md5sum ./build/*.deb
             dpkg -i ./build/*.deb
             env
-            cd /onnxruntime && ./build_and_test_onnxrt.sh
+            cd /onnxruntime && ./build_and_install_onnxrt.sh
         '''
+        stash includes:'/onnxruntime/build/Linux/Release/dist/*.whl', name:'onnxruntime-wheel'
     }
 }
+
+def onnxtestnode(name, body) {
+    return { label ->
+        rocmtestnode(variant: label, node: rocmnodename(name), docker_args: '-u root', body: body, pre: {
+           sh ''' apt install half
+                  md5sum ./build/*.deb
+                  dpkg -i ./build/*.deb 
+                  pip3 install /onnxruntime/build/Linux/Release/dist/*.whl '''
+            unstash 'migraphx-package'
+            unstash 'onnxruntime-wheel'
+        })
+    }
+}
+
+rocmtest onnxtests: onnxtestnode('anygpu') {cmake_build ->
+    stage("Onnxruntime Unit tests") {
+        sh '''
+            env
+            cd /onnxruntime && ./test_onnxrt_unit_tests.sh 
+        '''
+    }
+}, onnx_parity: onnxtestnode('anygpu') {cmake_build ->
+    stage("Parity tests Stable Pytorch") {
+        sh '''
+            env
+            cd /onnxruntime && ./test_onnxrt_parity_tests.sh 
+        '''
+    }
+}
+

tools/build_and_install_onnxrt.sh

@@ -0,0 +1,37 @@
+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+set -e
+
+ulimit -c unlimited
+
+cd /onnxruntime
+pip3 install -r requirements-dev.txt
+# Add newer cmake to the path
+export PATH="/opt/cmake/bin:$PATH"
+export CXXFLAGS="-D__HIP_PLATFORM_AMD__=1 -w"
+./build.sh --config Release  --cmake_extra_defines CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --update --build --build_wheel --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --skip_tests --rocm_home /opt/rocm --use_migraphx --migraphx_home /opt/rocm --rocm_version=`cat /opt/rocm/.info/version-dev` --allow_running_as_root
+
+#install onnxruntime wheel for parity tests
+cd build/Linux/Release
+pip3 install dist/*.whl

tools/docker/ubuntu_2204.dockerfile

@@ -125,3 +125,7 @@ ENV UBSAN_OPTIONS=print_stacktrace=1
 ENV ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1
 RUN ln -s /opt/rocm/llvm/bin/llvm-symbolizer /usr/bin/llvm-symbolizer
 
+#install dependancies used for onnxrt parity checks
+RUN pip3 install psutil==5.9.5 onnx==1.10.2 coloredlogs==15.0.1 packaging==23.1 transformers==4.29.2 sympy==1.12
+RUN pip3 install -U numpy==1.21.6
+

tools/test_onnxrt_parity_checks.sh

@@ -0,0 +1,37 @@
+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+set -e
+
+#Go to folder with parity checks
+cd /workspace/onnxruntime/onnxruntime/test/python/transformers/
+
+#Install latest stable torch version
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm5.4.2
+
+#Parity checks asked by microsoft. Ensure these are running and work with MIGraphX changes
+python test_parity_gelu.py --no_optimize 
+
+python test_parity_layernorm.py --no_optimize 
+
+python test_parity_huggingface_gpt_attention.py

tools/build_and_test_onnxrt.sh → tools/test_onnxrt_unit_tests.sh

@@ -1,7 +1,7 @@
 #####################################################################################
 # The MIT License (MIT)
 #
-# Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -21,21 +21,9 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-
 set -e
 
-ulimit -c unlimited
-
-cd /onnxruntime
-pip3 install -r requirements-dev.txt
-# Add newer cmake to the path
-export PATH="/opt/cmake/bin:$PATH"
-export CXXFLAGS="-D__HIP_PLATFORM_AMD__=1 -w"
-./build.sh --config Release  --cmake_extra_defines CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --update --build --build_wheel --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --skip_tests --rocm_home /opt/rocm --use_migraphx --migraphx_home /opt/rocm --rocm_version=`cat /opt/rocm/.info/version-dev` --allow_running_as_root
-
-cd build/Linux/Release
 #Add test launcher for onnxrt tests
-
 echo 'InferenceSessionTests.CheckRunProfilerWithSessionOptions' >> ../../../tools/ci_build/github/pai/migraphx-excluded-tests.txt
 echo 'InferenceSessionTests.CheckRunProfilerWithSessionOptions2' >> ../../../tools/ci_build/github/pai/migraphx-excluded-tests.txt
 echo 'InferenceSessionTests.Test3LayerNestedSubgraph' >> ../../../tools/ci_build/github/pai/migraphx-excluded-tests.txt