Skip to content

Commit

Permalink
[MXNET-710] Change POM files to be able to regularly publish to Apach…
Browse files Browse the repository at this point in the history
…e Release & Maven Central Repo (apache#11862)

* pom file changes for maven builds
  • Loading branch information
nswamy authored Jul 25, 2018
1 parent 21b4d2f commit a0c1179
Show file tree
Hide file tree
Showing 18 changed files with 402 additions and 53 deletions.
23 changes: 22 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,7 @@ scalaclean:
scalapkg:
(cd $(ROOTDIR)/scala-package; \
mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dcurrent_libdir="$(ROOTDIR)/lib" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")
Expand All @@ -608,12 +609,32 @@ scalaintegrationtest:
scalainstall:
(cd $(ROOTDIR)/scala-package; \
mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

scalarelease-dryrun:
(cd $(ROOTDIR)/scala-package; \
mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scalarelease-prepare:
(cd $(ROOTDIR)/scala-package; \
mvn release:clean release:prepare -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scalarelease-perform:
(cd $(ROOTDIR)/scala-package; \
mvn release:perform -DautoVersionSubmodules=true \
-Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \
-Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""")

scaladeploy:
(cd $(ROOTDIR)/scala-package; \
mvn deploy -Prelease,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \
mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests -Dcxx="$(CXX)" \
-Dbuild.platform="$(SCALA_PKG_PROFILE)" \
-Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \
-Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a")

Expand Down
36 changes: 35 additions & 1 deletion scala-package/assembly/linux-x86_64-cpu/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,49 @@
<version>1.3.0-SNAPSHOT</version>
<type>so</type>
</dependency>
<dependency>
<groupId>org.apache.mxnet</groupId>
<artifactId>mxnet-infer_${scala.binary.version}</artifactId>
<version>1.3.0-SNAPSHOT</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<!--<version>1.1.0-SNAPSHOT</version>-->
<configuration>
<pomElements>
<dependencies>remove</dependencies>
</pomElements>
</configuration>
<executions>
<!-- enable flattening -->
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<!-- ensure proper cleanup -->
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<skip>true</skip>
<skip>false</skip>
</configuration>
</plugin>
<plugin>
Expand Down
36 changes: 35 additions & 1 deletion scala-package/assembly/linux-x86_64-gpu/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,49 @@
<version>1.3.0-SNAPSHOT</version>
<type>so</type>
</dependency>
<dependency>
<groupId>org.apache.mxnet</groupId>
<artifactId>mxnet-infer_${scala.binary.version}</artifactId>
<version>1.3.0-SNAPSHOT</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<!--<version>1.1.0-SNAPSHOT</version>-->
<configuration>
<pomElements>
<dependencies>remove</dependencies>
</pomElements>
</configuration>
<executions>
<!-- enable flattening -->
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<!-- ensure proper cleanup -->
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<skip>true</skip>
<skip>false</skip>
</configuration>
</plugin>
<plugin>
Expand Down
41 changes: 41 additions & 0 deletions scala-package/assembly/osx-x86_64-cpu/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,51 @@
<version>1.3.0-SNAPSHOT</version>
<type>jnilib</type>
</dependency>
<dependency>
<groupId>org.apache.mxnet</groupId>
<artifactId>mxnet-infer_${scala.binary.version}</artifactId>
<version>1.3.0-SNAPSHOT</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<!--<version>1.1.0-SNAPSHOT</version>-->
<configuration>
<pomElements>
<dependencies>remove</dependencies>
</pomElements>
</configuration>
<executions>
<!-- enable flattening -->
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<!-- ensure proper cleanup -->
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
Expand Down
9 changes: 8 additions & 1 deletion scala-package/assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,13 @@
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
Expand All @@ -47,7 +54,7 @@
<goal>jar-no-fork</goal>
</goals>
<configuration>
<includePom>true</includePom>>
<includePom>true</includePom>
</configuration>
</execution>
</executions>
Expand Down
7 changes: 7 additions & 0 deletions scala-package/core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
Expand Down
108 changes: 108 additions & 0 deletions scala-package/dev/compile-mxnet-backend.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/bin/bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (Yizhi) This is mainly inspired by the script in apache/spark.
# I did some modificaiton to get it with our project.
#

set -e
echo "Compiling MXNet Backend, Hang tight!....."

if [[ ($# -ne 2) || ( $1 == "--help") || $1 == "-h" ]]; then
echo "Usage: [-h|--help] <osx-x86_64-cpu/linux-x86_64-cpu/linux-x86_64-gpu> <project.basedir>" 1>&2
exit 1
fi
PLATFORM=$1
MXNETDIR=$2


# below routine shamelessly copied from
# https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh
# This routine executes a command,
# prints error message on the console on non-zero exit codes and
# returns the exit code to the caller.
chkret() {
cmd=$*
echo "$cmd"
$cmd
ret=$?
if [[ ${ret} != 0 ]]; then
echo " "
echo "ERROR: Return value non-zero for: $cmd"
echo " "
exit 1
fi
} # chkret()

UNAME=`uname -s`
chkret pushd $MXNETDIR
chkret git submodule update --init --recursive

# don't want to overwrite an existing config file
cp make/config.mk ./config.mk

if [[ $PLATFORM == "osx-x86_64-cpu" ]];
then
echo "Building MXNet Backend on MAC OS"
echo "ADD_CFLAGS += -I/usr/local/opt/opencv/include" >> ./config.mk
echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk
echo "ADD_LDFLAGS += -L/usr/local/opt/opencv/lib" >> ./config.mk
echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk
echo "USE_OPENMP = 0" >> ./config.mk
echo "USE_LAPACK_PATH = /usr/local/opt/lapack/lib" >> ./config.mk
make -j$(sysctl -n hw.ncpu)
elif [[ $PLATFORM == "linux-x86_64-cpu" ]];
then
echo "Building MXNet Backend on Linux CPU"
echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk
echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk
echo "USE_OPENCV=1" >> ./config.mk
echo "USE_OPENMP=1" >> ./config.mk
echo "USE_BLAS=openblas" >> ./config.mk
echo "USE_LAPACK=1" >> ./config.mk
echo "USE_DIST_KVSTORE=1" >> ./config.mk
echo "USE_S3=1" >> ./config.mk
make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1)
elif [[ $PLATFORM == "linux-x86_64-gpu" ]]
then
echo "Building MXNet Backend on Linux GPU"
echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk
echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk
echo "USE_OPENCV=1" >> ./config.mk
echo "USE_OPENMP=1" >> ./config.mk
echo "USE_BLAS=openblas" >> ./config.mk
echo "USE_LAPACK=1" >> ./config.mk
echo "USE_DIST_KVSTORE=1" >> ./config.mk
echo "USE_S3=1" >> ./config.mk
echo "USE_CUDA=1" >> ./config.mk
echo "USE_CUDNN=1" >> ./config.mk
echo "ADD_CFLAGS += -I/usr/local/cuda/include" >> ./config.mk
echo "ADD_LDFLAGS += -L/usr/local/cuda/lib64/ " >> ./config.mk
#update th nccl version approriately
echo "ADD_LDFLAGS += -L/lib/nccl/cuda-9.0/lib " >> ./config.mk
eval "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/lib/nccl/cuda-9.0/lib"
eval "export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH"
make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1)
echo "Building MXNet Backend on Linux GPU"
else
echo "MY ALIEN OVERLOADS HAVE NOT TOLD WHAT TO DO FOR INVALID INPUT !!!"
echo "Currently supported platforms: osx-x86_64-cpu or linux-x86_64-cpu or linux-x86_64-gpu"
fi
chkret popd
echo "done building MXNet Backend"
exit 0
7 changes: 7 additions & 0 deletions scala-package/infer/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
Expand Down
9 changes: 8 additions & 1 deletion scala-package/init-native/linux-x86_64/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
Expand Down Expand Up @@ -55,7 +62,7 @@
<compilerStartOption>-std=c++0x</compilerStartOption>
</compilerStartOptions>
<compilerEndOptions>
<compilerEndOption>-I../../../include</compilerEndOption>
<compilerEndOption>-I${project.basedir}/../../../include</compilerEndOption>
<compilerEndOption>${all_includes}</compilerEndOption>
<compilerEndOption>${cflags}</compilerEndOption>
</compilerEndOptions>
Expand Down
Loading

0 comments on commit a0c1179

Please sign in to comment.