Skip to content

Commit

Permalink
Refactored Caffe's cppbuild, added Mac support
Browse files Browse the repository at this point in the history
  • Loading branch information
cypof committed Aug 15, 2015
1 parent dcc438f commit 43ed2f8
Showing 1 changed file with 200 additions and 102 deletions.
302 changes: 200 additions & 102 deletions caffe/cppbuild.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,108 +7,194 @@ if [[ -z "$PLATFORM" ]]; then
exit
fi

mkdir -p $PLATFORM
cd $PLATFORM
mkdir -p include
mkdir -p lib

if [ $PLATFORM == linux-x86_64_ ]
then
MAKEJ="${MAKEJ:-4}"

# glog
download https://github.com/google/glog/archive/v0.3.4.tar.gz ../v0.3.4.tar.gz
tar xzvf ../v0.3.4.tar.gz
cd glog-0.3.4
./configure --prefix=`pwd`/tmp
make -j $MAKEJ && make install
mv tmp/include/glog ../include/
mv tmp/lib/libglog.so.0.0.0 ../lib/libglog.so
cd ..

# gflags
download https://github.com/gflags/gflags/archive/v2.1.2.tar.gz ../v2.1.2.tar.gz
tar xzvf ../v2.1.2.tar.gz
cd gflags-2.1.2
mkdir build && cd build
export CXXFLAGS="-fPIC" && cmake -DBUILD_SHARED_LIBS=ON .. && make
mv include/gflags ../../include/
mv lib/libgflags.so.2.1.2 ../../lib/libgflags.so
cd ../..

# protobuf
download https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz ../protobuf-2.6.1.tar.gz
tar xzvf ../protobuf-2.6.1.tar.gz
cd protobuf-2.6.1
./configure --prefix=`pwd`/tmp
make -j $MAKEJ && make install
mv tmp/include/google ../include/
# only cp as lib will still be needed by protoc
cp tmp/lib/libprotobuf.so.9.0.1 ../lib/libprotobuf.so
mv tmp/bin ..
cd ..

# leveldb
download https://github.com/google/leveldb/archive/v1.18.tar.gz ../v1.18.tar.gz
tar xzvf ../v1.18.tar.gz
cd leveldb-1.18
make -j $MAKEJ
mv include/leveldb ../include/
mv libleveldb.so.1.18 ../lib/libleveldb.so
cd ..

# snappy
download https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz ../snappy-1.1.3.tar.gz
tar xzvf ../snappy-1.1.3.tar.gz
cd snappy-1.1.3
./configure --prefix=`pwd`/tmp
make -j $MAKEJ && make install
mv tmp/include/* ../include/
mv tmp/lib/libsnappy.so.1.3.0 ../lib/libsnappy.so
cd ..

# lmdb
download https://github.com/LMDB/lmdb/archive/LMDB_0.9.15.tar.gz ../LMDB_0.9.15.tar.gz
tar xzvf ../LMDB_0.9.15.tar.gz
cd lmdb-LMDB_0.9.15/libraries/liblmdb
make -j $MAKEJ
mv lmdb.h ../../../include/
mv liblmdb.so ../../../lib/
cd ../../..

# boost
download http://iweb.dl.sourceforge.net/project/boost/boost/1.58.0/boost_1_58_0.tar.gz ../boost_1_58_0.tar.gz
tar xzvf ../boost_1_58_0.tar.gz
cd boost_1_58_0
./bootstrap.sh --with-libraries=system,thread
./b2 stage
mv boost ../include
mv stage/lib/libboost_system.so.1.58.0 ../lib/libboost_system.so
mv stage/lib/libboost_thread.so.1.58.0 ../lib/libboost_thread.so
cd ..

# hdf5
download http://www.hdfgroup.org/ftp/HDF5/current/bin/linux-centos6-x86_64/hdf5-1.8.15-patch1-linux-centos6-x86_64-shared.tar.gz ../hdf5-1.8.15-patch1-linux-centos6-x86_64-shared.tar.gz
tar xzvf ../hdf5-1.8.15-patch1-linux-centos6-x86_64-shared.tar.gz
cd hdf5-1.8.15-patch1-linux-centos6-x86_64-shared
mv include/* ../include/
mv lib/libhdf5_hl.so.10.0.1 ../lib/libhdf5_hl.so
mv lib/libhdf5.so.10.0.1 ../lib/libhdf5.so
cd ..

# blas (requires fortran, e.g. sudo yum install gcc-gfortran)
download http://github.com/xianyi/OpenBLAS/archive/v0.2.14.tar.gz ../v0.2.14.tar.gz
tar xzvf ../v0.2.14.tar.gz
cd OpenBLAS-0.2.14
# CentOS compiler version can't compile AVX2 instructions, TODO update compiler
make -j $MAKEJ NO_AVX2=1
mv *.h ../include/
mv libopenblas_sandybridgep-r0.2.14.so ../lib/libopenblas.so
if [[ $PLATFORM == windows* ]]; then
echo TODO
else
GLOG=0.3.4
GFLAGS=2.1.2
PROTO=2.6.1
LEVELDB=1.18
SNAPPY=1.1.3
LMDB=0.9.15
BOOST=1_58_0 # also change the download link
HDF5=1.8.15-patch1
BLAS=0.2.14
MAKEJ="${MAKEJ:-4}"

download https://github.com/google/glog/archive/v$GLOG.tar.gz v$GLOG.tar.gz
download https://github.com/gflags/gflags/archive/v$GFLAGS.tar.gz v$GFLAGS.tar.gz
download https://github.com/google/protobuf/releases/download/v$PROTO/protobuf-$PROTO.tar.gz protobuf-$PROTO.tar.gz
download https://github.com/google/leveldb/archive/v$LEVELDB.tar.gz v$LEVELDB.tar.gz
download https://github.com/google/snappy/releases/download/$SNAPPY/snappy-$SNAPPY.tar.gz snappy-$SNAPPY.tar.gz
download https://github.com/LMDB/lmdb/archive/LMDB_$LMDB.tar.gz LMDB_$LMDB.tar.gz
download http://iweb.dl.sourceforge.net/project/boost/boost/1.58.0/boost_$BOOST.tar.gz boost_$BOOST.tar.gz
download http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-$HDF5.tar.bz2 hdf5-$HDF5.tar.bz2
download https://github.com/xianyi/OpenBLAS/archive/v$BLAS.tar.gz v$BLAS.tar.gz

mkdir -p $PLATFORM
cd $PLATFORM
INSTALL_PATH=`pwd`
mkdir -p include
mkdir -p lib

tar -xzvf ../v$GLOG.tar.gz
tar -xzvf ../v$GFLAGS.tar.gz
tar -xzvf ../protobuf-$PROTO.tar.gz
tar -xzvf ../v$LEVELDB.tar.gz
tar -xzvf ../snappy-$SNAPPY.tar.gz
tar -xzvf ../LMDB_$LMDB.tar.gz
tar -xzvf ../boost_$BOOST.tar.gz
tar -xjvf ../hdf5-$HDF5.tar.bz2
tar -xzvf ../v$BLAS.tar.gz
fi

case $PLATFORM in
android-arm)
# todo
;;
linux-x86_64)
cd glog-$GLOG
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/glog ../include/
mv tmp/lib/libglog.so.0.0.0 ../lib/libglog.so
cd ..

cd gflags-$GFLAGS
mkdir build
cd build
export CXXFLAGS="-fPIC" && cmake -DBUILD_SHARED_LIBS=ON .. && make
mv include/gflags ../../include/
mv lib/libgflags.so.$GFLAGS ../../lib/libgflags.so
cd ../..

cd protobuf-$PROTO
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/google ../include/
# only cp as lib will still be needed by protoc
cp tmp/lib/libprotobuf.so.9.0.1 ../lib/libprotobuf.so
mv tmp/bin ..
cd ..

cd leveldb-$LEVELDB
make -j $MAKEJ
mv include/leveldb ../include/
mv libleveldb.so.$LEVELDB ../lib/libleveldb.so
cd ..

cd snappy-$SNAPPY
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/* ../include/
mv tmp/lib/libsnappy.so.1.3.0 ../lib/libsnappy.so
cd ..

cd lmdb-LMDB_$LMDB/libraries/liblmdb
make -j $MAKEJ
mv lmdb.h ../../../include/
mv liblmdb.so ../../../lib/
cd ../../..

cd boost_$BOOST
./bootstrap.sh --with-libraries=system,thread
./b2 stage
mv boost ../include
mv stage/lib/libboost_system.so.1.58.0 ../lib/libboost_system.so
mv stage/lib/libboost_thread.so.1.58.0 ../lib/libboost_thread.so
cd ..

# x86-64 only binaries
download http://www.hdfgroup.org/ftp/HDF5/current/bin/linux-centos6-x86_64/hdf5-$HDF5-linux-centos6-x86_64-shared.tar.gz ../hdf5-$HDF5-linux-centos6-x86_64-shared.tar.gz
tar xzvf ../hdf5-$HDF5-linux-centos6-x86_64-shared.tar.gz
cd hdf5-$HDF5-linux-centos6-x86_64-shared
mv include/* ../include/
mv lib/libhdf5_hl.so.10.0.1 ../lib/libhdf5_hl.so
mv lib/libhdf5.so.10.0.1 ../lib/libhdf5.so
cd ..

# blas (requires fortran, e.g. sudo yum install gcc-gfortran)
cd OpenBLAS-$BLAS
# CentOS compiler version can't compile AVX2 instructions, TODO update compiler
make -j $MAKEJ NO_AVX2=1
mv *.h ../include/
mv libopenblas_sandybridgep-r$BLAS.so ../lib/libopenblas.so
cd ..
;;
macosx-*)
cd glog-$GLOG
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/glog ../include/
mv tmp/lib/libglog.0.dylib ../lib/libglog.dylib
cd ..

cd gflags-$GFLAGS
mkdir build
cd build
export CXXFLAGS="-fPIC" && cmake -DBUILD_SHARED_LIBS=ON .. && make
mv include/gflags ../../include/
mv lib/libgflags.$GFLAGS.dylib ../../lib/libgflags.dylib
cd ../..

cd protobuf-$PROTO
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/google ../include/
# only cp as lib will still be needed by protoc
cp tmp/lib/libprotobuf.9.dylib ../lib/libprotobuf.dylib
mv tmp/bin ..
cd ..

cd leveldb-$LEVELDB
make -j $MAKEJ
mv include/leveldb ../include/
mv libleveldb.dylib.$LEVELDB ../lib/libleveldb.dylib
cd ..

cd snappy-$SNAPPY
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/* ../include/
mv tmp/lib/libsnappy.1.dylib ../lib/libsnappy.dylib
cd ..

cd lmdb-LMDB_$LMDB/libraries/liblmdb
make -j $MAKEJ
mv lmdb.h ../../../include/
mv liblmdb.so ../../../lib/liblmdb.dylib
cd ../../..

cd boost_$BOOST
./bootstrap.sh --with-libraries=system,thread
./b2 stage
mv boost ../include
mv stage/lib/*.dylib ../lib/
cd ..

cd hdf5-$HDF5
./configure --prefix=`pwd`/tmp
make -j $MAKEJ
make install
mv tmp/include/* ../include/
mv tmp/lib/libhdf5_hl.10.dylib ../lib/libhdf5_hl.dylib
mv tmp/lib/libhdf5.10.dylib ../lib/libhdf5.dylib
cd ..
;;
*)
# todo
;;
esac

CAFFE_VERSION=master
download https://github.com/BVLC/caffe/archive/master.zip ../caffe-$CAFFE_VERSION.zip
#unzip ../caffe-$CAFFE_VERSION.zip
unzip ../caffe-$CAFFE_VERSION.zip
cd caffe-$CAFFE_VERSION
cp Makefile.config.example Makefile.config

Expand All @@ -120,17 +206,29 @@ case $PLATFORM in
printf "\n" >> Makefile.config
printf "INCLUDE_DIRS += ../include ../../../../opencv/cppbuild/linux-x86_64/include\n" >> Makefile.config
printf "LIBRARY_DIRS += ../lib ../../../../opencv/cppbuild/linux-x86_64/lib\n" >> Makefile.config
printf "BLAS := open\n" >> Makefile.config
export PATH=../bin:$PATH
export LD_LIBRARY_PATH=../lib:../protobuf-2.6.1/tmp/lib:$LD_LIBRARY_PATH
CC="gcc -m64" CXX="g++ -m64" BLAS=open DISTRIBUTE_DIR=.. make -j $MAKEJ
export LD_LIBRARY_PATH=../lib:../protobuf-$PROTO/tmp/lib:$LD_LIBRARY_PATH
CC="gcc -m64" CXX="g++ -m64" BLAS=open DISTRIBUTE_DIR=.. make -j $MAKEJ lib
# Manual deploy to avoid python build
mkdir -p ../include/caffe/proto
cp include/caffe/* ../include/caffe/
cp -r include/caffe/* ../include/caffe/
cp build/src/caffe/proto/caffe.pb.h ../include/caffe/proto
cp build/lib/libcaffe.so ../lib
;;
macosx-*)
LDFLAGS="-undefined dynamic_lookup" DISTRIBUTE_DIR=.. make --ignore-errors -j4 -e distribute
# LDFLAGS="-undefined dynamic_lookup" DISTRIBUTE_DIR=.. make --ignore-errors -j4 -e distribute
printf "\n" >> Makefile.config
printf "INCLUDE_DIRS += ../include ../../../../opencv/cppbuild/linux-x86_64/include\n" >> Makefile.config
printf "LIBRARY_DIRS += ../lib ../../../../opencv/cppbuild/linux-x86_64/lib\n" >> Makefile.config
export PATH=../bin:$PATH
export LD_LIBRARY_PATH=../lib:../protobuf-$PROTO/tmp/lib:$LD_LIBRARY_PATH
CC="gcc -m64" CXX="g++ -m64" BLAS=open DISTRIBUTE_DIR=.. make -j $MAKEJ lib
# Manual deploy to avoid python build
mkdir -p ../include/caffe/proto
cp -r include/caffe/* ../include/caffe/
cp build/src/caffe/proto/caffe.pb.h ../include/caffe/proto
cp build/lib/libcaffe.so ../lib/libcaffe.dylib
;;
*)
echo "Error: Platform \"$PLATFORM\" is not supported"
Expand Down

0 comments on commit 43ed2f8

Please sign in to comment.