* Add support for Windows to presets for MXNet and enable MKL-DNN (p…

…ull #309)
bytedeco · Oct 2, 2018 · 99b82ac · 99b82ac
1 parent b3524ea
commit 99b82ac
Show file tree

Hide file tree

Showing 10 changed files with 108 additions and 37 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -104,6 +104,8 @@ environment:
       OS: windows-x86_64
     - PROJ: "cuda"
       OS: windows-x86_64
+    - PROJ: "mxnet"
+      OS: windows-x86_64
     - PROJ: "tensorflow"
       OS: windows-x86_64
     - PROJ: "ale"
@@ -116,6 +118,9 @@ environment:
     - PROJ: "opencv"
       OS: windows-x86_64
       EXT: -gpu
+    - PROJ: "mxnet"
+      OS: windows-x86_64
+      EXT: -gpu
     - PROJ: "tensorflow"
       OS: windows-x86_64
       EXT: -gpu

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,5 @@
 
- * Add support for Windows to presets for LiquidFun ([pull #536](https://github.com/bytedeco/javacpp-presets/pull/536))
+ * Add support for Windows to presets for LiquidFun ([pull #536](https://github.com/bytedeco/javacpp-presets/pull/536)) and MXNet ([pull #309](https://github.com/bytedeco/javacpp-presets/pull/309))
  * Add CUDA-enabled build for MXNet via `-gpu` extension ([pull #609](https://github.com/bytedeco/javacpp-presets/pull/609))
  * Prevent MKL-DNN from compiling code with `-march=native` ([pull #618](https://github.com/bytedeco/javacpp-presets/pull/618))
  * Add an RPATH to `libmkldnn.so.0` to avoid loading issues on Linux ([issue deeplearning4j/deeplearning4j#6366](https://github.com/deeplearning4j/deeplearning4j/issues/6366))
@@ -15,7 +15,7 @@
  * Add presets for Spinnaker 1.15.x ([pull #553](https://github.com/bytedeco/javacpp-presets/pull/553)), CPython 3.6.x, ONNX 1.3.0 ([pull #547](https://github.com/bytedeco/javacpp-presets/pull/547))
  * Define `std::vector<tensorflow::OpDef>` type to `OpDefVector` for TensorFlow
  * Link HDF5 with zlib on Windows also ([issue deeplearning4j/deeplearning4j#6017](https://github.com/deeplearning4j/deeplearning4j/issues/6017))
- * Enable MKL-DNN for TensorFlow
+ * Enable MKL-DNN for MXNet and TensorFlow
  * Upgrade presets for OpenCV 3.4.3, FFmpeg 4.0.2, HDF5 1.10.3, MKL 2019.0, MKL-DNN 0.16, OpenBLAS 0.3.3, ARPACK-NG 3.6.3, LLVM 7.0.0, Tesseract 4.0.0-beta.4, CUDA 10.0, cuDNN 7.3, MXNet 1.3.0, TensorFlow 1.11.0, TensorRT 5.0, and their dependencies
  * Fix loading issue with `opencv_cudaobjdetect` and `opencv_cudaoptflow` on Windows ([issue #592](https://github.com/bytedeco/javacpp-presets/issues/592))
 

diff --git a/mxnet/cppbuild.sh b/mxnet/cppbuild.sh
@@ -14,39 +14,14 @@ export CUDA_ARCH=-arch=sm_30
 export USE_CUDA=0
 export USE_CUDNN=0
 export USE_CUDA_PATH=
+export USE_MKLDNN=1
 if [[ "$EXTENSION" == *gpu ]]; then
     export ADD_CFLAGS="$ADD_CFLAGS -DMXNET_USE_CUDA=1"
     export USE_CUDA=1
     export USE_CUDNN=1
     export USE_CUDA_PATH="/usr/local/cuda"
 fi
 
-case $PLATFORM in
-    linux-x86)
-        export CC="gcc -m32"
-        export CXX="g++ -m32"
-        export BLAS="openblas"
-        ;;
-    linux-x86_64)
-        export CC="gcc -m64"
-        export CXX="g++ -m64"
-        if which g++-6 &> /dev/null; then
-            export CC="gcc-6 -m64"
-            export CXX="g++-6 -m64"
-        fi
-        export BLAS="openblas"
-        ;;
-    macosx-*)
-        export CC="clang"
-        export CXX="clang++"
-        export BLAS="openblas"
-        ;;
-    *)
-        echo "Error: Platform \"$PLATFORM\" is not supported"
-        return 0
-        ;;
-esac
-
 MXNET_VERSION=1.3.0
 download http://apache.org/dist/incubator/mxnet/$MXNET_VERSION/apache-mxnet-src-$MXNET_VERSION-incubating.tar.gz apache-mxnet-src-$MXNET_VERSION-incubating.tar.gz
 
@@ -78,15 +53,70 @@ cd apache-mxnet-src-$MXNET_VERSION-incubating
 sedinplace 's/kCPU/Context::kCPU/g' src/operator/tensor/elemwise_binary_scalar_op_basic.cc
 sedinplace 's:../../src/operator/tensor/:./:g' src/operator/tensor/cast_storage-inl.h
 
+case $PLATFORM in
+    linux-x86)
+        export CC="gcc -m32"
+        export CXX="g++ -m32"
+        export BLAS="openblas"
+        export USE_MKLDNN=0
+        ;;
+    linux-x86_64)
+        export CC="gcc -m64"
+        export CXX="g++ -m64"
+        if which g++-6 &> /dev/null; then
+            export CC="gcc-6 -m64"
+            export CXX="g++-6 -m64"
+        fi
+        export BLAS="openblas"
+        ;;
+    macosx-*)
+        export CC="clang"
+        export CXX="clang++"
+        export BLAS="openblas"
+        ;;
+    windows-x86_64)
+        # copy include files
+        mkdir -p ../include
+        cp -r include/mxnet 3rdparty/dmlc-core/include/dmlc 3rdparty/mshadow/mshadow ../include
+
+        # configure the build
+        mkdir -p ../build
+        cd ../build
+        USE_X="-DCUDA_ARCH_LIST=3.0+PTX -DUSE_CUDA=$USE_CUDA -DUSE_CUDNN=$USE_CUDNN -DUSE_OPENCV=ON -DUSE_MKLDNN=$USE_MKLDNN"
+        OPENCV="-DOpenCV_DIR=$OPENCV_PATH/ -DOpenCV_CONFIG_PATH=$OPENCV_PATH/"
+        OPENBLAS="-DOpenBLAS_INCLUDE_DIR=$OPENBLAS_PATH/include/ -DOpenBLAS_LIB=$OPENBLAS_PATH/lib/openblas.lib"
+        "$CMAKE" -G "Visual Studio 14 2015 Win64" $USE_X $OPENCV $OPENBLAS ../apache-mxnet-src-$MXNET_VERSION-incubating
+
+        # build the project
+        MSBuild.exe ALL_BUILD.vcxproj //p:Configuration=Release //maxcpucount:$MAKEJ
+
+        # copy binary files
+        mkdir -p ../bin
+        cp Release/*.dll 3rdparty/mkldnn/src/Release/*.dll ../bin
+
+        # copy library files
+        mkdir -p ../lib
+        cp Release/libmxnet.lib ../lib/mxnet.lib
+
+        # finish
+        cd ../..
+        return 0
+        ;;
+    *)
+        echo "Error: Platform \"$PLATFORM\" is not supported"
+        return 0
+        ;;
+esac
+
 export C_INCLUDE_PATH="$OPENBLAS_PATH/include/:$OPENCV_PATH/include/"
 export CPLUS_INCLUDE_PATH="$C_INCLUDE_PATH"
 export LIBRARY_PATH="$OPENBLAS_PATH/:$OPENBLAS_PATH/lib/:$OPENCV_PATH/:$OPENCV_PATH/lib/"
 
 sed -i="" 's/$(shell pkg-config --cflags opencv)//' Makefile
 sed -i="" 's/$(shell pkg-config --libs opencv)/-lopencv_highgui -lopencv_imgcodecs -lopencv_imgproc -lopencv_core/' Makefile
-make -j $MAKEJ CC="$CC" CXX="$CXX" USE_BLAS="$BLAS" USE_OPENMP="$USE_OPENMP" CUDA_ARCH="$CUDA_ARCH" USE_CUDA="$USE_CUDA" USE_CUDNN="$USE_CUDNN" USE_CUDA_PATH="$USE_CUDA_PATH" USE_F16C=0 ADD_CFLAGS="$ADD_CFLAGS" ADD_LDFLAGS="$ADD_LDFLAGS" lib/libmxnet.a lib/libmxnet.so
-cp -a include lib 3rdparty/dmlc-core/include ..
-cp -a 3rdparty/mshadow/mshadow ../include
+make -j $MAKEJ CC="$CC" CXX="$CXX" USE_BLAS="$BLAS" USE_OPENMP="$USE_OPENMP" CUDA_ARCH="$CUDA_ARCH" USE_CUDA="$USE_CUDA" USE_CUDNN="$USE_CUDNN" USE_CUDA_PATH="$USE_CUDA_PATH" USE_MKLDNN="$USE_MKLDNN" USE_F16C=0 ADD_CFLAGS="$ADD_CFLAGS" ADD_LDFLAGS="$ADD_LDFLAGS" lib/libmxnet.a lib/libmxnet.so
+cp -r include lib 3rdparty/dmlc-core/include ..
+cp -r 3rdparty/mshadow/mshadow ../include
 unset CC
 unset CXX
 

diff --git a/mxnet/platform/pom.xml b/mxnet/platform/pom.xml
@@ -30,6 +30,11 @@
       <artifactId>openblas-platform</artifactId>
       <version>0.3.3-${project.parent.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.bytedeco.javacpp-presets</groupId>
+      <artifactId>mkl-dnn-platform</artifactId>
+      <version>0.16-${project.parent.version}</version>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>${javacpp.moduleId}</artifactId>
@@ -53,6 +58,12 @@
       <version>${project.version}</version>
       <classifier>${javacpp.platform.macosx-x86_64}</classifier>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>${javacpp.moduleId}</artifactId>
+      <version>${project.version}</version>
+      <classifier>${javacpp.platform.windows-x86_64}</classifier>
+    </dependency>
   </dependencies>
 
   <build>
@@ -65,7 +76,7 @@
             <configuration>
               <archive>
                 <manifestEntries>
-                  <Class-Path>${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-x86.jar ${javacpp.moduleId}-linux-x86_64.jar ${javacpp.moduleId}-macosx-x86_64.jar</Class-Path>
+                  <Class-Path>${javacpp.moduleId}.jar ${javacpp.moduleId}-linux-x86.jar ${javacpp.moduleId}-linux-x86_64.jar ${javacpp.moduleId}-macosx-x86_64.jar ${javacpp.moduleId}-windows-x86_64.jar</Class-Path>
                 </manifestEntries>
               </archive>
             </configuration>

diff --git a/mxnet/pom.xml b/mxnet/pom.xml
@@ -25,6 +25,11 @@
       <artifactId>openblas</artifactId>
       <version>0.3.3-${project.parent.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.bytedeco.javacpp-presets</groupId>
+      <artifactId>mkl-dnn</artifactId>
+      <version>0.16-${project.parent.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.bytedeco</groupId>
       <artifactId>javacpp</artifactId>
@@ -110,6 +115,18 @@
       </plugin>
       <plugin>
         <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>${javacpp.platform}${javacpp.platform.extension}</id>
+            <phase>package</phase>
+            <configuration>
+              <excludes>
+                <exclude>org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/*mklml*</exclude>
+                <exclude>org/bytedeco/javacpp/${javacpp.platform}${javacpp.platform.extension}/*omp*</exclude>
+              </excludes>
+            </configuration>
+          </execution>
+        </executions>
       </plugin>
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>

diff --git a/mxnet/src/main/java/org/bytedeco/javacpp/presets/mxnet.java b/mxnet/src/main/java/org/bytedeco/javacpp/presets/mxnet.java
@@ -40,12 +40,12 @@
  * @author Samuel Audet
  */
 @Properties(inherit = {openblas.class, opencv_imgcodecs.class, opencv_highgui.class}, target = "org.bytedeco.javacpp.mxnet", value = {
-    @Platform(value = {"linux-x86", "macosx"}, compiler = "cpp11", define = {"DMLC_USE_CXX11 1", "MSHADOW_USE_CBLAS 1", "MSHADOW_IN_CXX11 1"},
+    @Platform(value = {"linux-x86", "macosx", "windows"}, compiler = "cpp11", define = {"DMLC_USE_CXX11 1", "MSHADOW_USE_CBLAS 1", "MSHADOW_IN_CXX11 1"},
         include = {"mxnet/c_api.h", "mxnet/c_predict_api.h", /*"dmlc/base.h", "dmlc/io.h", "dmlc/logging.h", "dmlc/type_traits.h",
                    "dmlc/parameter.h", "mshadow/base.h", "mshadow/expression.h", "mshadow/tensor.h", "mxnet/base.h",*/},
-        link = "mxnet", /*resource = {"include", "lib"},*/ includepath = {"/usr/local/cuda/include/",
+        link = "mxnet", preload = {"iomp5", "libiomp5md", "mklml", "mklml_intel", "mkldnn@.0", "libmxnet"}, /*resource = {"include", "lib"},*/ includepath = {"/usr/local/cuda/include/",
         "/System/Library/Frameworks/vecLib.framework/", "/System/Library/Frameworks/Accelerate.framework/"}, linkpath = "/usr/local/cuda/lib/"),
-    @Platform(value = {"linux-x86_64", "macosx-x86_64"}, define = {"DMLC_USE_CXX11 1", "MSHADOW_USE_CBLAS 1", "MSHADOW_IN_CXX11 1", "MSHADOW_USE_CUDA 1"}, extension = "-gpu") })
+    @Platform(value = {"linux-x86_64", "macosx-x86_64", "windows-x86_64"}, define = {"DMLC_USE_CXX11 1", "MSHADOW_USE_CBLAS 1", "MSHADOW_IN_CXX11 1", "MSHADOW_USE_CUDA 1"}, extension = "-gpu") })
 public class mxnet implements LoadEnabled, InfoMapper {
 
     @Override public void init(ClassProperties properties) {

diff --git a/openblas/cppbuild.sh b/openblas/cppbuild.sh
@@ -212,4 +212,9 @@ unset CC
 unset FC
 unset LDFLAGS
 
+if [[ -f ../lib/libopenblas.dll.a ]]; then
+    # bundle the import library for Windows under a friendly name for MSVC
+    cp ../lib/libopenblas.dll.a ../lib/openblas.lib
+fi
+
 cd ../..
diff --git a/openblas/pom.xml b/openblas/pom.xml
@@ -86,7 +86,7 @@
                 <exclude>org/bytedeco/javacpp/${javacpp.platform}/*omp*</exclude>
                 <exclude>org/bytedeco/javacpp/${javacpp.platform}/*libopenblas_nolapack*</exclude>
                 <exclude>org/bytedeco/javacpp/${javacpp.platform}/bin/*openblas*</exclude>
-                <exclude>org/bytedeco/javacpp/${javacpp.platform}/lib/*openblas*</exclude>
+                <exclude>org/bytedeco/javacpp/${javacpp.platform}/lib/*libopenblas*</exclude>
                 <exclude>${javacpp.platform.library.path}/*libopenblas_nolapack*</exclude>
                 <exclude>${javacpp.platform.library.path}/lib/*openblas*</exclude>
               </excludes>

diff --git a/opencv/src/main/java/org/bytedeco/javacpp/presets/opencv_core.java b/opencv/src/main/java/org/bytedeco/javacpp/presets/opencv_core.java
@@ -65,10 +65,12 @@
         "api-ms-win-crt-locale-l1-1-0", "api-ms-win-crt-string-l1-1-0", "api-ms-win-crt-stdio-l1-1-0", "api-ms-win-crt-math-l1-1-0",
         "api-ms-win-crt-heap-l1-1-0", "api-ms-win-crt-runtime-l1-1-0", "api-ms-win-crt-convert-l1-1-0", "api-ms-win-crt-environment-l1-1-0",
         "api-ms-win-crt-time-l1-1-0", "api-ms-win-crt-filesystem-l1-1-0", "api-ms-win-crt-utility-l1-1-0", "api-ms-win-crt-multibyte-l1-1-0",
-        "vcruntime140", "msvcp140", "concrt140", "opencv_cudev343"}),
+        "vcruntime140", "msvcp140", "concrt140", "vcomp140", "opencv_cudev343"}),
     @Platform(value = "windows-x86", preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x86/Microsoft.VC140.CRT/",
+                                                    "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x86/Microsoft.VC140.OpenMP/",
                                                     "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x86/"}),
     @Platform(value = "windows-x86_64", preloadpath = {"C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.CRT/",
+                                                       "C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/redist/x64/Microsoft.VC140.OpenMP/",
                                                        "C:/Program Files (x86)/Windows Kits/10/Redist/ucrt/DLLs/x64/"}),
     @Platform(value = {"linux-x86_64", "macosx-x86_64", "windows-x86_64"}, extension = "-gpu")},
         target = "org.bytedeco.javacpp.opencv_core", helper = "org.bytedeco.javacpp.helper.opencv_core")

diff --git a/pom.xml b/pom.xml
@@ -1345,6 +1345,7 @@
         <module>leptonica</module>
         <module>tesseract</module>
         <module>cuda</module>
+        <module>mxnet</module>
         <module>tensorflow</module>
         <module>ale</module>
         <module>liquidfun</module>