From 3f6709b3120563f62169513a82f4de1d5840f502 Mon Sep 17 00:00:00 2001 From: Nico Hezel Date: Wed, 30 May 2018 00:26:16 +0200 Subject: [PATCH 1/3] add tensorflow with gpu support for windows --- tensorflow/cppbuild.sh | 12 ++++++--- .../bytedeco/javacpp/presets/tensorflow.java | 26 +++++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/tensorflow/cppbuild.sh b/tensorflow/cppbuild.sh index ea146ed66c9..33eb2fd4bbe 100755 --- a/tensorflow/cppbuild.sh +++ b/tensorflow/cppbuild.sh @@ -69,7 +69,7 @@ if [[ "$EXTENSION" == *gpu ]]; then export TF_NEED_CUDA=1 export TF_NEED_TENSORRT=1 export GPU_FLAGS="--config=cuda" - export CMAKE_GPU_FLAGS="-Dtensorflow_ENABLE_GPU=ON -Dtensorflow_CUDA_VERSION=$TF_CUDA_VERSION -Dtensorflow_CUDNN_VERSION=$TF_CUDNN_VERSION" + export CMAKE_GPU_FLAGS="-Dtensorflow_ENABLE_GPU=ON -Dtensorflow_CUDA_VERSION=$TF_CUDA_VERSION -Dtensorflow_CUDNN_VERSION=$TF_CUDNN_VERSION -DCUDNN_HOME=$CUDNN_INSTALL_PATH" fi if [[ "$TF_NEED_CUDA" == 0 ]] || [[ ! -d "$TENSORRT_INSTALL_PATH" ]]; then @@ -139,11 +139,15 @@ case $PLATFORM in export PATH=$DYLD_LIBRARY_PATH:$PATH patch -Np1 < ../../../tensorflow-macosx.patch ;; - windows-x86_64) + windows-x86_64) + # help cmake's findCuda-method to find the right cuda version + export CUDA_PATH=$CUDA_TOOLKIT_PATH + mkdir -p tensorflow/contrib/cmake/build cd tensorflow/contrib/cmake/build - "$CMAKE" -A x64 -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE="C:/Python27/python.exe" -Dtensorflow_BUILD_PYTHON_BINDINGS=OFF -Dtensorflow_BUILD_SHARED_LIB=ON -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX $CMAKE_GPU_FLAGS .. - MSBuild.exe tensorflow_static.vcxproj //p:Configuration=Release + "$CMAKE" -A x64 -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE="C:/Python27/python.exe" -Dtensorflow_BUILD_PYTHON_BINDINGS=OFF -Dtensorflow_BUILD_SHARED_LIB=ON -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX -G"Visual Studio 14" $CMAKE_GPU_FLAGS .. + MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tf_core_gpu_kernels.vcxproj + MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tensorflow_static.vcxproj cd ../../../.. ;; *) diff --git a/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java b/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java index 8ded52835cb..afcf49b3e84 100644 --- a/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java +++ b/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java @@ -195,12 +195,28 @@ preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x86/Microsoft.VC140.CRT/", "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x86/"}), @Platform( - value = "windows-x86_64", - preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.CRT/", - "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x64/"}), - @Platform( - value = {"linux-x86_64", "macosx-x86_64", "windows-x86_64"}, + value = "windows-x86_64", + preloadpath = { "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.CRT/", + "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x64/"}), + + @Platform( + value = {"windows-x86_64"}, + extension = "-gpu", + link = {"Advapi32#", "zlibstatic", "gpr", "grpc_unsecure", "grpc++_unsecure", "farmhash", "fft2d", + "lmdb", "giflib", "libjpeg", "libpng12_static", "nsync", "libprotobuf", "re2", "snappy", "sqlite", + + "cuda@.9.2", "cudnn@.7", "cublas@.9.2", "cublas_device@.9.2", "cufft@.9.2", "cufftw@.9.2", + "curand@.9.2", "cusolver@.9.2", "cusparse@.9.2", "cudart@.9.2", "cudart_static@.9.2", "cupti@.9.2", + + "tensorflow_static", "tf_core_gpu_kernels", "tf_protos_cc", "tf_cc_op_gen_main"}, + includepath = { "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/include/"}, + linkpath = { "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/lib/x64/", + "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/extras/CUPTI/libx64/"}), + + @Platform( + value = {"linux-x86_64", "macosx-x86_64"}, extension = "-gpu"), + @Platform( value = {"android"}, compiler = {"cpp11"}, From 074154e3abf6a0eabc4e70fff3e9b8e49b14d4bb Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Sat, 2 Jun 2018 10:41:12 +0900 Subject: [PATCH 2/3] Fix formatting and shorten build directory --- tensorflow/README.md | 6 +++ tensorflow/cppbuild.sh | 21 +++++----- tensorflow/pom.xml | 40 +++++++++--------- .../bytedeco/javacpp/presets/tensorflow.java | 41 ++++++++----------- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/tensorflow/README.md b/tensorflow/README.md index 3d1a98c53da..ecde2b54bd7 100644 --- a/tensorflow/README.md +++ b/tensorflow/README.md @@ -60,6 +60,12 @@ We can use [Maven 3](http://maven.apache.org/) to download and install automatic 1.8.0-1.4.2-SNAPSHOT macosx-x86_64-gpu + + org.bytedeco.javacpp-presets + tensorflow + 1.8.0-1.4.2-SNAPSHOT + windows-x86_64-gpu + diff --git a/tensorflow/cppbuild.sh b/tensorflow/cppbuild.sh index 33eb2fd4bbe..3a809df9a92 100755 --- a/tensorflow/cppbuild.sh +++ b/tensorflow/cppbuild.sh @@ -69,7 +69,7 @@ if [[ "$EXTENSION" == *gpu ]]; then export TF_NEED_CUDA=1 export TF_NEED_TENSORRT=1 export GPU_FLAGS="--config=cuda" - export CMAKE_GPU_FLAGS="-Dtensorflow_ENABLE_GPU=ON -Dtensorflow_CUDA_VERSION=$TF_CUDA_VERSION -Dtensorflow_CUDNN_VERSION=$TF_CUDNN_VERSION -DCUDNN_HOME=$CUDNN_INSTALL_PATH" + export CMAKE_GPU_FLAGS="-Dtensorflow_ENABLE_GPU=ON -Dtensorflow_CUDA_VERSION=$TF_CUDA_VERSION -Dtensorflow_CUDNN_VERSION=$TF_CUDNN_VERSION" fi if [[ "$TF_NEED_CUDA" == 0 ]] || [[ ! -d "$TENSORRT_INSTALL_PATH" ]]; then @@ -139,16 +139,15 @@ case $PLATFORM in export PATH=$DYLD_LIBRARY_PATH:$PATH patch -Np1 < ../../../tensorflow-macosx.patch ;; - windows-x86_64) - # help cmake's findCuda-method to find the right cuda version - export CUDA_PATH=$CUDA_TOOLKIT_PATH - - mkdir -p tensorflow/contrib/cmake/build - cd tensorflow/contrib/cmake/build - "$CMAKE" -A x64 -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE="C:/Python27/python.exe" -Dtensorflow_BUILD_PYTHON_BINDINGS=OFF -Dtensorflow_BUILD_SHARED_LIB=ON -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX -G"Visual Studio 14" $CMAKE_GPU_FLAGS .. - MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tf_core_gpu_kernels.vcxproj - MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tensorflow_static.vcxproj - cd ../../../.. + windows-x86_64) + # help cmake's findCuda-method to find the right cuda version + export CUDA_PATH="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v$TF_CUDA_VERSION" + mkdir -p ../build + cd ../build + "$CMAKE" -A x64 -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE="C:/Python27/python.exe" -Dtensorflow_BUILD_PYTHON_BINDINGS=OFF -Dtensorflow_BUILD_SHARED_LIB=ON -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX -G"Visual Studio 14" $CMAKE_GPU_FLAGS -DCUDNN_HOME="$CUDA_PATH" ../tensorflow-$TENSORFLOW_VERSION/tensorflow/contrib/cmake + MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tf_core_gpu_kernels.vcxproj + MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tensorflow_static.vcxproj + cd ../tensorflow-$TENSORFLOW_VERSION ;; *) echo "Error: Platform \"$PLATFORM\" is not supported" diff --git a/tensorflow/pom.xml b/tensorflow/pom.xml index 9a1b46d528b..87c881d46a7 100644 --- a/tensorflow/pom.xml +++ b/tensorflow/pom.xml @@ -81,30 +81,30 @@ ${tensorflow.path}/bazel-tensorflow-${tensorflow.version}/external/protobuf_archive/src/ ${tensorflow.path}/bazel-tensorflow-${tensorflow.version}/external/eigen_archive/ ${tensorflow.path}/bazel-tensorflow-${tensorflow.version}/external/eigen_archive/eigen-eigen-d02e6a705c30/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/zlib/src/zlib/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/nsync/src/nsync/public/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/protobuf/src/protobuf/src/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/eigen/src/eigen/ + ${tensorflow.path}/../build/ + ${tensorflow.path}/../build/zlib/src/zlib/ + ${tensorflow.path}/../build/nsync/src/nsync/public/ + ${tensorflow.path}/../build/protobuf/src/protobuf/src/ + ${tensorflow.path}/../build/eigen/src/eigen/ ${basedir}/target/classes/org/bytedeco/javacpp/include/ ${tensorflow.path}/bazel-bin/tensorflow/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/zlib/src/zlib/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/grpc/src/grpc/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/farmhash/src/farmhash/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/fft2d/src/fft2d/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/lmdb/src/lmdb-build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/gif/src/gif-build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/jpeg/src/jpeg-build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/png/src/png-build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/nsync/src/nsync/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/protobuf/src/protobuf/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/re2/src/re2/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/snappy/src/snappy/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/sqlite/src/sqlite-build/Release/ - ${tensorflow.path}/tensorflow/contrib/cmake/build/tf_cc_op_gen_main.dir/Release/ + ${tensorflow.path}/../build/Release/ + ${tensorflow.path}/../build/zlib/src/zlib/Release/ + ${tensorflow.path}/../build/grpc/src/grpc/Release/ + ${tensorflow.path}/../build/farmhash/src/farmhash/Release/ + ${tensorflow.path}/../build/fft2d/src/fft2d/Release/ + ${tensorflow.path}/../build/lmdb/src/lmdb-build/Release/ + ${tensorflow.path}/../build/gif/src/gif-build/Release/ + ${tensorflow.path}/../build/jpeg/src/jpeg-build/Release/ + ${tensorflow.path}/../build/png/src/png-build/Release/ + ${tensorflow.path}/../build/nsync/src/nsync/Release/ + ${tensorflow.path}/../build/protobuf/src/protobuf/Release/ + ${tensorflow.path}/../build/re2/src/re2/Release/ + ${tensorflow.path}/../build/snappy/src/snappy/Release/ + ${tensorflow.path}/../build/sqlite/src/sqlite-build/Release/ + ${tensorflow.path}/../build/tf_cc_op_gen_main.dir/Release/ ${basedir}/cppbuild/${javacpp.platform}${javacpp.platform.extension}/java/ diff --git a/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java b/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java index afcf49b3e84..de8ad8ed6dd 100644 --- a/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java +++ b/tensorflow/src/main/java/org/bytedeco/javacpp/presets/tensorflow.java @@ -181,11 +181,14 @@ "tensorflow/cc/ops/training_ops.h", "tensorflow/cc/ops/user_ops.h"}, link = "tensorflow_cc", preload = "tensorflow_framework"), + @Platform( + value = {"linux-x86_64", "macosx-x86_64"}, + extension = "-gpu"), @Platform( value = "windows", link = {"Advapi32#", "zlibstatic", "gpr", "grpc_unsecure", "grpc++_unsecure", "farmhash", "fft2d", - "lmdb", "giflib", "libjpeg", "libpng12_static", "nsync", "libprotobuf", "re2", "snappy", - "sqlite", "tensorflow_static", "tf_protos_cc", "tf_cc_op_gen_main"}, + "lmdb", "giflib", "libjpeg", "libpng12_static", "nsync", "libprotobuf", "re2", "snappy", "sqlite", + "tensorflow_static", "tf_protos_cc", "tf_cc_op_gen_main"}, preload = {"concrt140", "msvcp140", "vcruntime140", "api-ms-win-crt-locale-l1-1-0", "api-ms-win-crt-string-l1-1-0", "api-ms-win-crt-stdio-l1-1-0", "api-ms-win-crt-math-l1-1-0", "api-ms-win-crt-heap-l1-1-0", "api-ms-win-crt-runtime-l1-1-0", "api-ms-win-crt-convert-l1-1-0", "api-ms-win-crt-environment-l1-1-0", @@ -195,28 +198,20 @@ preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x86/Microsoft.VC140.CRT/", "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x86/"}), @Platform( - value = "windows-x86_64", - preloadpath = { "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.CRT/", - "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x64/"}), - - @Platform( - value = {"windows-x86_64"}, + value = "windows-x86_64", + preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.CRT/", + "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x64/"}), + @Platform( + value = "windows-x86_64", extension = "-gpu", - link = {"Advapi32#", "zlibstatic", "gpr", "grpc_unsecure", "grpc++_unsecure", "farmhash", "fft2d", - "lmdb", "giflib", "libjpeg", "libpng12_static", "nsync", "libprotobuf", "re2", "snappy", "sqlite", - - "cuda@.9.2", "cudnn@.7", "cublas@.9.2", "cublas_device@.9.2", "cufft@.9.2", "cufftw@.9.2", - "curand@.9.2", "cusolver@.9.2", "cusparse@.9.2", "cudart@.9.2", "cudart_static@.9.2", "cupti@.9.2", - - "tensorflow_static", "tf_core_gpu_kernels", "tf_protos_cc", "tf_cc_op_gen_main"}, - includepath = { "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/include/"}, - linkpath = { "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/lib/x64/", - "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/extras/CUPTI/libx64/"}), - - @Platform( - value = {"linux-x86_64", "macosx-x86_64"}, - extension = "-gpu"), - + link = {"Advapi32#", "zlibstatic", "gpr", "grpc_unsecure", "grpc++_unsecure", "farmhash", "fft2d", + "lmdb", "giflib", "libjpeg", "libpng12_static", "nsync", "libprotobuf", "re2", "snappy", "sqlite", + "cudart", "cudart_static", "cuda", "cublas", "cublas_device", "cudnn", + "cufft", "cufftw", "curand", "cusolver", "cusparse", "cupti", + "tf_core_gpu_kernels", "tensorflow_static", "tf_protos_cc", "tf_cc_op_gen_main"}, + includepath = {"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/include/"}, + linkpath = {"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/lib/x64/", + "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.2/extras/CUPTI/libx64/"}), @Platform( value = {"android"}, compiler = {"cpp11"}, From 4e6a95760947c9226643d4a9e287a2cb17233cfb Mon Sep 17 00:00:00 2001 From: Samuel Audet Date: Sat, 2 Jun 2018 12:51:02 +0900 Subject: [PATCH 3/3] Fix non-CUDA build --- tensorflow/cppbuild.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/cppbuild.sh b/tensorflow/cppbuild.sh index 3a809df9a92..15978bf7e29 100755 --- a/tensorflow/cppbuild.sh +++ b/tensorflow/cppbuild.sh @@ -145,7 +145,9 @@ case $PLATFORM in mkdir -p ../build cd ../build "$CMAKE" -A x64 -DCMAKE_BUILD_TYPE=Release -DPYTHON_EXECUTABLE="C:/Python27/python.exe" -Dtensorflow_BUILD_PYTHON_BINDINGS=OFF -Dtensorflow_BUILD_SHARED_LIB=ON -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX -G"Visual Studio 14" $CMAKE_GPU_FLAGS -DCUDNN_HOME="$CUDA_PATH" ../tensorflow-$TENSORFLOW_VERSION/tensorflow/contrib/cmake - MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tf_core_gpu_kernels.vcxproj + if [[ "$EXTENSION" == *gpu ]]; then + MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tf_core_gpu_kernels.vcxproj + fi MSBuild.exe //p:Configuration=Release /maxcpucount:$MAKEJ tensorflow_static.vcxproj cd ../tensorflow-$TENSORFLOW_VERSION ;;