-
Notifications
You must be signed in to change notification settings - Fork 741
/
action.yml
246 lines (219 loc) · 14.8 KB
/
action.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
name: Deploy on CentOS
runs:
using: composite
steps:
- name: Install environment
shell: bash
env:
GITHUB_EVENT_HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
cd /root
mkdir -p .ccache
echo "max_size = 2.0G" > .ccache/ccache.conf
echo "hash_dir = false" >> .ccache/ccache.conf
echo "sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros" >> .ccache/ccache.conf
SCL_ENABLE="devtoolset-9 rh-python38"
SCL_INSTALL="rh-python38-python-urllib3"
CENTOS_VERSION=$(rpm --eval '%{centos_ver}')
if [[ "$CENTOS_VERSION" == "6" ]]; then
find /etc/yum.repos.d/ -name *.repo | xargs -i sed -i 's/mirror\.centos\.org\/centos/vault.centos.org/g;s/$releasever/6.10/g;s/mirrorlist/#mirrorlist/g;s/#baseurl/baseurl/g' {}
SCL_ENABLE="devtoolset-9 rh-python36 python27"
SCL_INSTALL=""
fi
echo "SCL_ENABLE=$SCL_ENABLE" >> $GITHUB_ENV
echo "SCL_INSTALL=$SCL_INSTALL" >> $GITHUB_ENV
yum -y update
yum -y install centos-release-scl-rh epel-release
if [[ "$CENTOS_VERSION" == "6" ]]; then
sed -i 's/mirror\.centos\.org\/centos/vault.centos.org/g;s/6\/sclo/6.10\/sclo/g;s/mirrorlist/#mirrorlist/g;s/#baseurl/baseurl/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
fi
curl -L "https://negativo17.org/repos/epel-multimedia.repo" | sed -e '/^\[[a-z\-]*\]/a priority=99' > /etc/yum.repos.d/epel-multimedia.repo
yum -y install $SCL_ENABLE $SCL_INSTALL rh-java-common-ant boost-devel ccache clang gcc-c++ gcc-gfortran java-1.8.0-openjdk-devel ant python python3-devel python3-pip swig file which wget unzip tar bzip2 gzip xz patch autoconf-archive automake make libtool bison flex perl-core nasm alsa-lib-devel freeglut-devel gtk2-devel libusb-devel libusb1-devel curl-devel expat-devel gettext-devel openssl-devel bzip2-devel zlib-devel SDL2-devel libva-devel libxkbcommon-devel libxkbcommon-x11-devel xcb-util* fontconfig-devel libffi-devel ragel ocl-icd-devel GeoIP-devel pcre-devel ssdeep-devel yajl-devel
# https://gcc.gnu.org/legacy-ml/gcc-patches/2018-01/msg01962.html
sed -i 's/_mm512_abs_pd (__m512 __A)/_mm512_abs_pd (__m512d __A)/g' /opt/rh/devtoolset-9/root/usr/lib/gcc/x86_64-redhat-linux/9/include/avx512fintrin.h
source scl_source enable $SCL_ENABLE || true
curl -LO https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.tar.gz
curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
curl -LO https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.18.3.tar.gz
curl -LO https://services.gradle.org/distributions/gradle-7.1.1-bin.zip
tar -xzf cmake-3.24.2-linux-x86_64.tar.gz -C /opt/
tar -xzf apache-maven-3.6.3-bin.tar.gz -C /opt/
tar -xzf git-2.18.3.tar.gz
unzip gradle-7.1.1-bin.zip -d /opt/
pushd git-2.18.3; make -j2 prefix=/usr/local/; make -j2 prefix=/usr/local/ install; popd
ln -sf /usr/bin/python3.6 /usr/bin/python3
ln -sf /opt/cmake-3.24.2-linux-x86_64/bin/* /usr/bin/
ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/bin/mvn
ln -sf /opt/gradle-7.1.1/bin/gradle /usr/bin/gradle
python3 -m pip install gdown || python3 -m pip install gdown
if [[ $CI_DEPLOY_PLATFORM == android-* ]]; then
curl -LO https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip
unzip -qq android-ndk-r21e-linux-x86_64.zip
ln -sf android-ndk-r21e android-ndk
export MAVEN_OPTIONS="-Djavacpp.platform.root=$(pwd)/android-ndk/ -Djava.library.path="
fi
if [[ $CI_DEPLOY_PLATFORM == "linux-x86" ]]; then
rpm -qa | sed s/.x86_64$/.i686/ | xargs yum -q -y install
wget --no-directories --no-parent -r https://www.repo.cloudlinux.com/cloudlinux/$CENTOS_VERSION/sclo/devtoolset-9/i386/
rpm -i --force --nodeps *.rpm
fi
if [[ -n ${CI_DEPLOY_NEED_BAZEL:-} ]]; then
echo Installing Bazel
curl -LO https://github.com/bazelbuild/bazel/releases/download/0.25.3/bazel-0.25.3-installer-linux-x86_64.sh
bash bazel-0.25.3-installer-linux-x86_64.sh
fi
if [[ "$CI_DEPLOY_PLATFORM" == "linux-x86_64" ]] && [[ -n ${CI_DEPLOY_NEED_CUDA:-} ]]; then
echo Installing CUDA, cuDNN, nvCOMP, etc
curl -LO https://developer.download.nvidia.com/compute/cuda/12.1.1/local_installers/cuda-repo-rhel7-12-1-local-12.1.1_530.30.02-1.x86_64.rpm
curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libcudnn8-8.9.1.23-1.cuda12.1.x86_64.rpm
curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libcudnn8-devel-8.9.1.23-1.cuda12.1.x86_64.rpm
curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libnccl-2.18.1-1+cuda12.1.x86_64.rpm
curl -LO https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/libnccl-devel-2.18.1-1+cuda12.1.x86_64.rpm
rpm -i --force --ignorearch --nodeps cuda-repo-rhel7-* libcudnn*.rpm libnccl*.rpm
rm -f *.rpm *.tgz *.txz *.tar.*
pushd /var/cuda-repo-rhel7-12-1-local/; rpm -i --force --ignorearch --nodeps cuda*.rpm libc*.rpm libn*.rpm; rm *.rpm; popd
ln -sf /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/libcuda.so
ln -sf /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/local/cuda/lib64/libnvidia-ml.so
mv /usr/include/cudnn* /usr/include/nccl* /usr/local/cuda/include/
mv /usr/lib64/libcudnn* /usr/lib64/libnccl* /usr/local/cuda/lib64/
curl -LO https://developer.download.nvidia.com/compute/nvcomp/3.0.4/local_installers/nvcomp_3.0.4_x86_64_12.x.tgz
tar -xvf nvcomp_3.0.4_x86_64_12.x.tgz -C /usr/local/cuda/lib64/ --strip-components=1 lib/
tar -xvf nvcomp_3.0.4_x86_64_12.x.tgz -C /usr/local/cuda/include/ --strip-components=1 include/
rm -f nvcomp_3.0.4_x86_64_12.x.tgz
# Work around issues with CUDA 10.2/11.x
mv /usr/include/cublas* /usr/include/nvblas* /usr/local/cuda/include/ || true
mv /usr/lib64/libcublas* /usr/lib64/libnvblas* /usr/local/cuda/lib64/ || true
for f in /usr/local/cuda/include/*_v8.h; do ln -sf $f ${f:0:${#f}-5}.h; done
for f in /usr/local/cuda/lib64/libcudnn*so.8.*; do ln -sf $f ${f:0:${#f}-4}; ln -sf $f ${f:0:${#f}-6}; done
for f in /usr/local/cuda/lib64/*.so.10; do ln -s $f $f.2; done
for f in /usr/local/cuda/lib64/*.so.10; do ln -s $f ${f:0:${#f}-1}1; done
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.1
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.2
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.3
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.4
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.5
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.6
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.7
ln -s libcudart.so.11.0 /usr/local/cuda/lib64/libcudart.so.11.8
cp /usr/local/cuda/lib64/stubs/libcuda.so /usr/lib64/libcuda.so
cp /usr/local/cuda/lib64/stubs/libcuda.so /usr/lib64/libcuda.so.1
cp /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/lib64/libnvidia-ml.so
cp /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/lib64/libnvidia-ml.so.1
sed -i s/cublas_v2.h/cublas_api.h/g /usr/local/cuda/include/cublasXt.h /usr/local/cuda/include/cusolverDn.h /usr/local/cuda/include/cusolverSp.h
sed -i /block_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
sed -i /device_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
sed -i /device_segmented_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
sed -i /warp_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
# Remove downloaded archives and unused libraries to avoid running out of disk space
rm -f $(find /usr/local/cuda/ -name '*.a' -and -not -name libcudart_static.a -and -not -name libcudadevrt.a -and -not -name libnvcomp_device.a)
fi
if [[ "$CI_DEPLOY_MODULE" == "nvcodec" ]]; then
echo Installing NVIDIA Video Codec SDK
python3 -m gdown 13KyYoq8ZOLIA7jugDkdFma34mnyxnz4x
unzip Video_Codec_SDK_12.1.14.zip -d /usr/local
ln -sf /usr/local/Video_Codec_SDK_* /usr/local/videocodecsdk
fi
if [[ "$CI_DEPLOY_MODULE" == "tensorrt" ]]; then
echo Installing TensorRT
python3 -m gdown 1dVhD-DEYY42QbZe1GXl-vxe3k6KqWGsL
tar -hxvf TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-12.0.tar.gz -C /usr/local/
ln -sf /usr/local/TensorRT* /usr/local/tensorrt
fi
if [[ "$CI_DEPLOY_MODULE" == "mkl" ]]; then
echo Installing MKL
curl -LO https://registrationcenter-download.intel.com/akdlm/IRC_NAS/cd17b7fe-500e-4305-a89b-bd5b42bfd9f8/l_onemkl_p_2023.1.0.46342_offline.sh
bash l_onemkl_p_2023.1.0.46342_offline.sh -s -a -s --eula accept
fi
if [[ "$CI_DEPLOY_PLATFORM" == "linux-x86" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then
echo Installing FlyCapture i386
python3 -m gdown 1BOpSik1Fndagzjf4ykwzermt2qlTzsWI
tar -xzvf flycapture2-2.13.3.31-i386-pkg_xenial.tgz
find -name *.deb | while read f; do ar -xv $f; tar --totals -xf data.tar.xz -C /; done;
fi
if [[ "$CI_DEPLOY_PLATFORM" == "linux-x86_64" ]] && [[ "$CI_DEPLOY_MODULE" == "flycapture" ]]; then
echo Installing FlyCapture amd64
python3 -m gdown 1YtVjdnbQLZHX_ocQ6xAmiq6pjftuPOPd
tar -xzvf flycapture2-2.13.3.31-amd64-pkg_xenial.tgz
find -name *.deb | while read f; do ar -xv $f; tar --totals -xf data.tar.xz -C /; done;
fi
if [[ "$CI_DEPLOY_PLATFORM" == "linux-x86_64" ]] && [[ "$CI_DEPLOY_MODULE" == "spinnaker" ]]; then
echo Installing Spinnaker amd64
python3 -m gdown 1Xapzvxrgx1w3RCbAOFILaNCZwpusB_VA
tar -xzvf spinnaker-4.0.0.116-amd64-pkg-22.04.tar.gz
find -name *.deb | while read f; do ar -xv $f; tar --totals -xvf data.tar.xz -C /; done;
fi
if [[ -n ${CI_DEPLOY_NEED_GCC:-} ]]; then
rm -f /usr/lib/libgfortran.so.3* /usr/lib64/libgfortran.so.3* # not required for GCC 7+
rm -f /usr/lib/libgfortran.so.4* /usr/lib64/libgfortran.so.4* # not required for GCC 9+
else
rm -f /usr/lib/libgfortran.so.5* /usr/lib64/libgfortran.so.5* # remove library from GCC 9+
fi
GRADLE_TASK=publishToMavenLocal
MAVEN_PHASE=install
if [[ -n ${CI_DEPLOY_USERNAME:-} ]] && [[ ! "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
GRADLE_TASK=publish
MAVEN_PHASE=deploy
if [[ "$GITHUB_EVENT_HEAD_COMMIT_MESSAGE" == Release* ]] || [[ "${GITHUB_REF#refs/heads/}" == "release" ]]; then
python3 -m gdown -q -O settings.tar.gz $CI_DEPLOY_SETTINGS
tar -xzf settings.tar.gz
MAVEN_OPTIONS="$MAVEN_OPTIONS -Dgpg.homedir=$(pwd)/.gnupg/ -DperformRelease -DstagingRepositoryId=$STAGING_REPOSITORY"
fi
fi
echo "GPG_PASSPHRASE=***" >> $GITHUB_ENV
echo "GRADLE_TASK=$GRADLE_TASK" >> $GITHUB_ENV
echo "MAVEN_PHASE=$MAVEN_PHASE" >> $GITHUB_ENV
echo "MAVEN_OPTIONS=$MAVEN_OPTIONS" >> $GITHUB_ENV
- name: Build project
shell: bash
run: |
source scl_source enable $SCL_ENABLE || true
git --version
gcc --version
cmake --version
gradle --version
mvn -version
gpg --version
python3 --version
ccache -s --version
df -h
export MAKEJ=$(getconf _NPROCESSORS_ONLN)
echo Fetching $GITHUB_REPOSITORY@$GITHUB_SHA
git init
git fetch --depth 1 https://github.com/$GITHUB_REPOSITORY $GITHUB_SHA
git checkout $GITHUB_SHA
git submodule update --init --recursive
git submodule foreach --recursive 'git reset --hard'
echo "Fixing HOME to /root (was '$HOME')"
export HOME=/root
if [[ -n ${CI_DEPLOY_NEED_GRADLE:-} ]]; then
echo Executing Gradle $GRADLE_TASK ${{ matrix.options }} $CI_DEPLOY_OPTIONS on $MAKEJ processors
export GRADLE_OPTIONS="-Dorg.gradle.jvmargs=-Xmx2048m -PjavacppPlatform=$CI_DEPLOY_PLATFORM -PjavacppPlatformExtension=${{ matrix.ext }} ${{ matrix.options }} $CI_DEPLOY_OPTIONS --info"
gradle clean build $GRADLE_OPTIONS -b $CI_DEPLOY_MODULE/build.gradle
echo Publishing only now to help avoid race conditions with Gradle
gradle $GRADLE_TASK $GRADLE_OPTIONS -b $CI_DEPLOY_MODULE/build.gradle --refresh-dependencies
else
mkdir -p $HOME/.m2
echo "<settings>" > $HOME/.m2/settings.xml
echo "<mirrors><mirror><id>google-maven-central</id><name>GCS Maven Central mirror</name><url>https://maven-central.storage-download.googleapis.com/maven2/</url><mirrorOf>central</mirrorOf></mirror></mirrors>" >> $HOME/.m2/settings.xml
echo "<servers><server><id>sonatype-nexus-snapshots</id><username>$CI_DEPLOY_USERNAME</username><password>$CI_DEPLOY_PASSWORD</password></server><server><id>sonatype-nexus-staging</id><username>$CI_DEPLOY_USERNAME</username><password>$CI_DEPLOY_PASSWORD</password></server></servers>" >> $HOME/.m2/settings.xml
echo "</settings>" >> $HOME/.m2/settings.xml
SECONDS=$(( RANDOM % 300 ))
echo Sleeping $SECONDS seconds to help avoid race conditions with Maven
sleep $SECONDS
echo Executing Maven $MAVEN_PHASE $CI_DEPLOY_OPTIONS on $MAKEJ processors
export EXT=${{ matrix.ext }}
export EXT2=${EXT:1}
export MAVEN_OPTS="-Xss2m -Xmx4g"
export MAVEN_OPTIONS="clean $MAVEN_PHASE -B -U -e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Djavacpp.platform=$CI_DEPLOY_PLATFORM -Djavacpp.platform.extension=$EXT $MAVEN_OPTIONS $CI_DEPLOY_OPTIONS"
for i in {1..5}; do STATUS=0; mvn $MAVEN_OPTIONS -pl .,$CI_DEPLOY_MODULE && break || STATUS=$? && sleep 60; done; (exit $STATUS)
for i in {1..5}; do STATUS=0; mvn $MAVEN_OPTIONS -f $CI_DEPLOY_MODULE/platform/$EXT2/pom.xml && break || STATUS=$? && sleep 60; done; (exit $STATUS)
if [[ -e $CI_DEPLOY_MODULE/platform/redist/pom.xml ]]; then
for i in {1..5}; do STATUS=0; mvn $MAVEN_OPTIONS -f $CI_DEPLOY_MODULE/platform/redist/pom.xml && break || STATUS=$? && sleep 60; done; (exit $STATUS)
fi
fi
df -h
- name: Clean up
shell: bash
run: |
cd /root
rm -Rf $(find .m2/repository/ -name '*SNAPSHOT*')