From 14d371631b3fe8e7f8f92df18eba7fa379d0538c Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Fri, 19 May 2023 12:02:35 -0400 Subject: [PATCH 01/10] Update instal_prereqs.sh to handle 22.04 defines Needed to run containers with 22.04 --- tools/install_prereqs.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index a2d6f9f0718..75853ca3df3 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -56,8 +56,14 @@ echo "Dependencies are installed at $PREFIX" # Install deps with rbuild rbuild prepare -d $PREFIX -s develop -# install onnx package for unit tests -pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 +if ! python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then + python3.8 -m pip install protobuf==3.20.3 onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 + export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" python3.10 -m pip install onnx==1.10.2 + python3.10 -m pip install protobuf==3.20.3 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 +else + # install onnx package for unit tests + pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 -# pin version of protobuf in Python for onnx runtime unit tests -pip3 install protobuf==3.20.0 + # pin version of protobuf in Python for onnx runtime unit tests + pip3 install protobuf==3.20.0 +fi \ No newline at end of file From 557e6dbea0fd5a69886afef47cb9fc6246685be7 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Thu, 25 May 2023 11:10:31 -0400 Subject: [PATCH 02/10] Add Dockerfile for Ubuntu 22.04 and ROCm 5.5 Updated dockerfile to use ROCm 5.5 and Ubuntu 22.04 for use with building MIGraphX Able to run make -j$(nproc) check successfully with this --- Dockerfile_Ubuntu_2204 | 127 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 Dockerfile_Ubuntu_2204 diff --git a/Dockerfile_Ubuntu_2204 b/Dockerfile_Ubuntu_2204 new file mode 100644 index 00000000000..06440bf50e9 --- /dev/null +++ b/Dockerfile_Ubuntu_2204 @@ -0,0 +1,127 @@ +FROM ubuntu:22.04 + +ARG PREFIX=/usr/local + +# Support multiarch +RUN dpkg --add-architecture i386 + +# Install rocm key +RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl && \ + curl -fsSL http://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg + +# Add rocm repository +RUN sh -c "echo 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] http://repo.radeon.com/rocm/apt/5.5 jammy main' > /etc/apt/sources.list.d/rocm.list" + +# From docs.amd.com for installing rocm. Needed to install properly +RUN sh -c "echo 'Package: *\nPin: release o=repo.radeon.com\nPin-priority: 600' > /etc/apt/preferences.d/rocm-pin-600" + + +# Install dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ + apt-utils \ + build-essential \ + #clang-format-10 \ + cmake \ + curl \ + doxygen \ + #g++-7 \ + gdb \ + git \ + lcov \ + locales \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + software-properties-common \ + wget \ + rocm-device-libs \ + hip-base \ + libnuma-dev \ + miopen-hip \ + rocblas \ + hipfft \ + rocthrust \ + rocrand \ + hipsparse \ + rccl \ + rccl-dev \ + rocm-smi-lib \ + rocm-dev \ + roctracer-dev \ + hipcub \ + hipblas \ + hipify-clang \ + half \ + libssl-dev \ + zlib1g-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + + +# add this for roctracer dependancies +RUN pip3 install CppHeaderParser + +# Workaround broken rocm packages +RUN ln -s /opt/rocm-* /opt/rocm +RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf +RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf +RUN ldconfig + +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +# Install dependencies +ADD dev-requirements.txt /dev-requirements.txt +ADD requirements.txt /requirements.txt +ADD rbuild.ini /rbuild.ini + +COPY ./tools/install_prereqs.sh / +RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh +RUN test -f /usr/local/hash || exit 1 + +# Install yapf +RUN pip3 install yapf==0.28.0 + +# Install doc requirements +ADD doc/requirements.txt /doc-requirements.txt +RUN pip3 install -r /doc-requirements.txt + +# Download real models to run onnx unit tests +ENV ONNX_HOME=/.onnx +COPY ./tools/download_models.sh / +RUN /download_models.sh && rm /download_models.sh + +# Install latest ccache version +RUN cget -p $PREFIX install facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake +RUN cget -p $PREFIX install ccache@v4.1 -DENABLE_TESTING=OFF +RUN cget -p /opt/cmake install kitware/cmake@v3.24.3 + +COPY ./test/onnx/.onnxrt-commit / + +ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime +ARG ONNXRUNTIME_BRANCH=main +ARG ONNXRUNTIME_COMMIT + +RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ + cd onnxruntime && \ + if [ -z "$ONNXRUNTIME_COMMIT" ] ; then git checkout $(cat /.onnxrt-commit) ; else git checkout ${ONNXRUNTIME_COMMIT} ; fi && \ + /bin/sh /onnxruntime/dockerfiles/scripts/install_common_deps.sh + + +ADD tools/build_and_test_onnxrt.sh /onnxruntime/build_and_test_onnxrt.sh + +RUN cget -p /usr/local install ROCmSoftwarePlatform/rocMLIR@a997d5f51314b45d7a4c04f1599966dcf53f9b4d -DBUILD_MIXR_TARGET=On -DLLVM_ENABLE_ZSTD=Off -DLLVM_ENABLE_THREADS=Off + +ENV MIOPEN_FIND_DB_PATH=/tmp/miopen/find-db +ENV MIOPEN_USER_DB_PATH=/tmp/miopen/user-db +ENV LD_LIBRARY_PATH=$PREFIX/lib + +# Setup ubsan environment to printstacktrace +ENV UBSAN_OPTIONS=print_stacktrace=1 +ENV ASAN_OPTIONS=detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1 +RUN ln -s /opt/rocm/llvm/bin/llvm-symbolizer /usr/bin/llvm-symbolizer + From 5fabcc4a279fce79cf93c015afb24dfe695f5aa9 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Thu, 25 May 2023 14:22:32 -0400 Subject: [PATCH 03/10] Clean this up since its breaking CI --- tools/install_prereqs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index 75853ca3df3..66c0f035561 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -58,7 +58,8 @@ rbuild prepare -d $PREFIX -s develop if ! python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then python3.8 -m pip install protobuf==3.20.3 onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 - export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" python3.10 -m pip install onnx==1.10.2 + export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" + python3.10 -m pip install onnx==1.10.2 python3.10 -m pip install protobuf==3.20.3 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 else # install onnx package for unit tests From 5ffca18982850ff4bcf3afaa11a5a0f5d27c5397 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Thu, 25 May 2023 15:28:39 -0400 Subject: [PATCH 04/10] cleanup install preq some more. -use one protobuf version -remove extra python3.8 installs from 3.10 case --- tools/install_prereqs.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index 66c0f035561..db407ab8eea 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -56,15 +56,14 @@ echo "Dependencies are installed at $PREFIX" # Install deps with rbuild rbuild prepare -d $PREFIX -s develop -if ! python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then - python3.8 -m pip install protobuf==3.20.3 onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 +if python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" - python3.10 -m pip install onnx==1.10.2 - python3.10 -m pip install protobuf==3.20.3 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 + python3.10 -m pip onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 else # install onnx package for unit tests pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 # pin version of protobuf in Python for onnx runtime unit tests - pip3 install protobuf==3.20.0 -fi \ No newline at end of file +fi + +pip3 install protobuf==3.20.0 From 12f33b48779600e99f4057e6834b09fe9e1beba6 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Wed, 31 May 2023 11:12:34 -0400 Subject: [PATCH 05/10] Move comment for protobuf comment --- tools/install_prereqs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index db407ab8eea..4ec707b61f9 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -63,7 +63,7 @@ else # install onnx package for unit tests pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 - # pin version of protobuf in Python for onnx runtime unit tests fi +# pin version of protobuf in Python for onnx runtime unit tests between dist versions pip3 install protobuf==3.20.0 From d3275a01e2b83f1444bdf70e9de5921c2c7151f9 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Wed, 31 May 2023 11:19:27 -0400 Subject: [PATCH 06/10] Move Dockerfile for 22.04 to Dockerfiles/ folder --- .../Dockerfile_Ubuntu_2204.dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfile_Ubuntu_2204 => Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile (100%) diff --git a/Dockerfile_Ubuntu_2204 b/Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile similarity index 100% rename from Dockerfile_Ubuntu_2204 rename to Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile From f000cd5da913b62352aa8a920f97ece8f753b806 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Wed, 31 May 2023 12:26:27 -0400 Subject: [PATCH 07/10] Move and rename 2204 docker file remove Docker_** from the name. Move these to tools/docker --- .../docker/ubuntu_2204.dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile => tools/docker/ubuntu_2204.dockerfile (100%) diff --git a/Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile b/tools/docker/ubuntu_2204.dockerfile similarity index 100% rename from Dockerfiles/Dockerfile_Ubuntu_2204.dockerfile rename to tools/docker/ubuntu_2204.dockerfile From e228416878c18e52d2611638bcd55f6f6a65c5ed Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Wed, 31 May 2023 13:11:15 -0400 Subject: [PATCH 08/10] Add pip3 installs to be shared between python versions --- tools/install_prereqs.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index 4ec707b61f9..d466ba11b60 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -58,12 +58,9 @@ rbuild prepare -d $PREFIX -s develop if python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" - python3.10 -m pip onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 -else - # install onnx package for unit tests - pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 - fi +pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0 + # pin version of protobuf in Python for onnx runtime unit tests between dist versions pip3 install protobuf==3.20.0 From 2f2a38c0551d75ed854106da4ba097acccbb4d30 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Mon, 12 Jun 2023 11:57:47 -0400 Subject: [PATCH 09/10] Add Package pin from repo.radeon.com --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 7e8d4571edd..d76976fda8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,9 @@ RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl && # Add rocm repository RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/5.4.2/ ubuntu main > /etc/apt/sources.list.d/rocm.list' +# From docs.amd.com for installing rocm. Needed to install properly +RUN sh -c "echo 'Package: *\nPin: release o=repo.radeon.com\nPin-priority: 600' > /etc/apt/preferences.d/rocm-pin-600" + # Install dependencies RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ apt-utils \ From 527cdbe3ea88ddf48f553d0891587a04b2a535d2 Mon Sep 17 00:00:00 2001 From: Ted Themistokleous Date: Mon, 12 Jun 2023 11:58:56 -0400 Subject: [PATCH 10/10] Add CMAKE_ARG ONNX_USE_PROTOBUF_SHARED_LIBS for every default python dist Set this to be default as part of installing prereqs --- tools/install_prereqs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/install_prereqs.sh b/tools/install_prereqs.sh index d466ba11b60..9d1a321f118 100755 --- a/tools/install_prereqs.sh +++ b/tools/install_prereqs.sh @@ -56,9 +56,7 @@ echo "Dependencies are installed at $PREFIX" # Install deps with rbuild rbuild prepare -d $PREFIX -s develop -if python3 -c 'import sys; assert sys.version_info >= (3, 10)' > /dev/null; then - export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" -fi +export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON" pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0