Rebuild (#24)

* Reworked build process to be easier and more flexible * arm32 and arm64 builds renamed * arm32 and arm64 docker images are all built on x86 via cross-compiling
snowzach · Jun 19, 2020 · dccf0a3 · dccf0a3
1 parent 2a850c9
commit dccf0a3
Show file tree

Hide file tree

Showing 17 changed files with 830 additions and 1,341 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -12,4 +12,5 @@ doods
 private
 models/**
 Dockerfile.*.*
-
+Dockerfile.*
+README.md
diff --git a/Dockerfile b/Dockerfile
@@ -1,25 +1,150 @@
-ARG BUILDER_TAG="latest"
-FROM registry.prozach.org/doods-builder:$BUILDER_TAG as builder
+FROM ubuntu:18.04 as base
+
+# Install reqs with cross compile support
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config zip zlib1g-dev unzip wget bash-completion git curl \
+    build-essential patch g++ python python-future python-numpy python-six python3 \
+    cmake ca-certificates \
+    libc6-dev libstdc++6 libusb-1.0-0
+
+# Install protoc
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip && \
+    unzip protoc-3.12.3-linux-x86_64.zip -d /usr/local && \
+    rm /usr/local/readme.txt && \
+    rm protoc-3.12.3-linux-x86_64.zip
+
+# Version Configuration
+ARG BAZEL_VERSION="0.26.1"
+ARG TF_VERSION="v1.15.3"
+ARG OPENCV_VERSION="4.3.0"
+ARG GO_VERSION="1.14.3"
+
+# Install bazel
+ENV BAZEL_VERSION $BAZEL_VERSION
+RUN wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb && \
+    dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb && \
+    rm bazel_${BAZEL_VERSION}-linux-x86_64.deb
+
+# Download tensorflow sources
+ENV TF_VERSION $TF_VERSION
+RUN cd /opt && git clone https://github.com/tensorflow/tensorflow.git --branch $TF_VERSION --single-branch
+
+# Configure tensorflow
+ENV TF_NEED_GDR=0 TF_NEED_AWS=0 TF_NEED_GCP=0 TF_NEED_CUDA=0 TF_NEED_HDFS=0 TF_NEED_OPENCL_SYCL=0 TF_NEED_VERBS=0 TF_NEED_MPI=0 TF_NEED_MKL=0 TF_NEED_JEMALLOC=1 TF_ENABLE_XLA=0 TF_NEED_S3=0 TF_NEED_KAFKA=0 TF_NEED_IGNITE=0 TF_NEED_ROCM=0
+RUN cd /opt/tensorflow && yes '' | ./configure
+
+# Tensorflow build flags for rpi
+ENV BAZEL_COPT_FLAGS="--local_resources 16000,16,1 -c opt --config monolithic --copt=-march=native --copt=-O3 --copt=-fomit-frame-pointer --incompatible_no_support_tools_in_action_inputs=false --config=noaws --config=nohdfs"
+ENV BAZEL_EXTRA_FLAGS=""
+
+# Compile and build tensorflow lite
+RUN cd /opt/tensorflow && \
+    bazel build -c opt $BAZEL_COPT_FLAGS --verbose_failures $BAZEL_EXTRA_FLAGS //tensorflow/lite:libtensorflowlite.so && \
+    install bazel-bin/tensorflow/lite/libtensorflowlite.so /usr/local/lib/libtensorflowlite.so && \
+    bazel build -c opt $BAZEL_COPT_FLAGS --verbose_failures $BAZEL_EXTRA_FLAGS //tensorflow/lite/experimental/c:libtensorflowlite_c.so && \
+    install bazel-bin/tensorflow/lite/experimental/c/libtensorflowlite_c.so /usr/local/lib/libtensorflowlite_c.so && \
+    mkdir -p /usr/local/include/flatbuffers && cp bazel-tensorflow/external/flatbuffers/include/flatbuffers/* /usr/local/include/flatbuffers
+
+# Compile and install tensorflow shared library
+RUN cd /opt/tensorflow && \
+    bazel build -c opt $BAZEL_COPT_FLAGS --verbose_failures $BAZEL_EXTRA_FLAGS //tensorflow:libtensorflow.so && \
+    install bazel-bin/tensorflow/libtensorflow.so /usr/local/lib/libtensorflow.so && \
+    ln -rs /usr/local/lib/libtensorflow.so /usr/local/lib/libtensorflow.so.1
+
+# cleanup so the cache directory isn't huge
+RUN cd /opt/tensorflow && \
+    bazel clean && rm -Rf /root/.cache
+
+# Install GOCV
+ENV OPENCV_VERSION $OPENCV_VERSION
+RUN cd /tmp && \
+    curl -Lo opencv.zip https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
+    unzip -q opencv.zip && \
+    curl -Lo opencv_contrib.zip https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip && \
+    unzip -q opencv_contrib.zip && \
+    rm opencv.zip opencv_contrib.zip && \
+    cd opencv-${OPENCV_VERSION} && \
+    mkdir build && cd build && \
+    cmake -D CMAKE_BUILD_TYPE=RELEASE \
+    -D CMAKE_INSTALL_PREFIX=/usr/local \
+    -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-${OPENCV_VERSION}/modules \
+    -D WITH_JASPER=OFF \
+    -D WITH_QT=OFF \
+    -D WITH_GTK=OFF \
+    -D BUILD_DOCS=OFF \
+    -D BUILD_EXAMPLES=OFF \
+    -D BUILD_TESTS=OFF \
+    -D BUILD_PERF_TESTS=OFF \
+    -D BUILD_opencv_java=NO \
+    -D BUILD_opencv_python=NO \
+    -D BUILD_opencv_python2=NO \
+    -D BUILD_opencv_python3=NO \
+    -D OPENCV_GENERATE_PKGCONFIG=ON .. && \
+    make -j $(nproc --all) && \
+    make preinstall && make install && \
+    cd /tmp && rm -rf opencv*
+
+# Download the edgetpu library and install it
+RUN cd /tmp && git clone https://github.com/google-coral/edgetpu.git && \
+    install edgetpu/libedgetpu/throttled/k8/libedgetpu.so.1.0 /usr/local/lib/libedgetpu.so.1.0 && \
+    ln -rs /usr/local/lib/libedgetpu.so.1.0 /usr/local/lib/libedgetpu.so.1 && \
+    ln -rs /usr/local/lib/libedgetpu.so.1.0 /usr/local/lib/libedgetpu.so && \
+    mkdir -p /usr/local/include/libedgetpu && \
+    install edgetpu/libedgetpu/edgetpu.h /usr/local/include/libedgetpu/edgetpu.h && \
+    install edgetpu/libedgetpu/edgetpu_c.h /usr/local/include/libedgetpu/edgetpu_c.h && \
+    rm -Rf edgetpu
+
+# Configure the Go version to be used
+ENV GO_ARCH "amd64"
+ENV GOARCH=amd64
+
+# Install Go
+ENV GO_VERSION $GO_VERSION
+RUN curl -kLo go${GO_VERSION}.linux-${GO_ARCH}.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \
+    tar -C /usr/local -xzf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \
+    rm go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
+
+FROM ubuntu:18.04 as builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config zip zlib1g-dev unzip wget bash-completion git curl \
+    build-essential patch g++ python python-future python3 ca-certificates \
+    libc6-dev libstdc++6 libusb-1.0-0
+
+# Copy all libraries, includes and go
+COPY --from=base /usr/local/. /usr/local/.
+COPY --from=base /opt/tensorflow /opt/tensorflow 
+
+ENV GOOS=linux
+ENV CGO_ENABLED=1
+ENV CGO_CFLAGS=-I/opt/tensorflow
+ENV PATH /usr/local/go/bin:/go/bin:${PATH}
+ENV GOPATH /go
 
 # Create the build directory
+RUN mkdir /build
 WORKDIR /build
 ADD . .
 RUN make
 
-FROM debian:buster-slim
+FROM ubuntu:18.04
+
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends libusb-1.0 libc++-7-dev wget unzip ca-certificates libdc1394-22 libavcodec58 libavformat58 && \
+    apt-get install -y --no-install-recommends libusb-1.0 libc++-7-dev wget unzip ca-certificates libdc1394-22 libavcodec57 libavformat57 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 RUN mkdir -p /opt/doods
 WORKDIR /opt/doods
 COPY --from=builder /usr/local/lib/. /usr/local/lib/.
 COPY --from=builder /build/doods /opt/doods/doods
-ADD config.yaml /opt/doods/config.yaml
 RUN ldconfig
 
+# Download sample models
 RUN mkdir models
 RUN wget https://storage.googleapis.com/download.tensorflow.org/models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && unzip coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && rm coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && mv detect.tflite models/coco_ssd_mobilenet_v1_1.0_quant.tflite && rm labelmap.txt
 RUN wget https://dl.google.com/coral/canned_models/coco_labels.txt && mv coco_labels.txt models/coco_labels0.txt
+RUN wget http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_v2_coco_2018_01_28.tar.gz && tar -zxvf faster_rcnn_inception_v2_coco_2018_01_28.tar.gz faster_rcnn_inception_v2_coco_2018_01_28/frozen_inference_graph.pb --strip=1 && mv frozen_inference_graph.pb models/faster_rcnn_inception_v2_coco_2018_01_28.pb && rm faster_rcnn_inception_v2_coco_2018_01_28.tar.gz
+RUN wget https://github.com/raw/amikelive/coco-labels/master/coco-labels-2014_2017.txt && mv coco-labels-2014_2017.txt models/coco_labels1.txt
+ADD config.yaml config.yaml
 
 CMD ["/opt/doods/doods", "-c", "/opt/doods/config.yaml", "api"]
diff --git a/Dockerfile.base.amd64 → Dockerfile.amd64 b/Dockerfile.base.amd64 → Dockerfile.amd64
@@ -1,4 +1,4 @@
-FROM debian:buster as builder
+FROM ubuntu:18.04 as base
 
 # Install reqs with cross compile support
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -8,18 +8,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libc6-dev libstdc++6 libusb-1.0-0
 
 # Install protoc
-RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.9.1/protoc-3.9.1-linux-x86_64.zip && \
-    unzip protoc-3.9.1-linux-x86_64.zip -d /usr/local && \
+RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v3.12.3/protoc-3.12.3-linux-x86_64.zip && \
+    unzip protoc-3.12.3-linux-x86_64.zip -d /usr/local && \
     rm /usr/local/readme.txt && \
-    rm protoc-3.9.1-linux-x86_64.zip
+    rm protoc-3.12.3-linux-x86_64.zip
+
+# Version Configuration
+ARG BAZEL_VERSION="0.26.1"
+ARG TF_VERSION="v1.15.3"
+ARG OPENCV_VERSION="4.3.0"
+ARG GO_VERSION="1.14.3"
 
 # Install bazel
-RUN wget https://github.com/bazelbuild/bazel/releases/download/0.27.1/bazel_0.27.1-linux-x86_64.deb && \
-    dpkg -i bazel_0.27.1-linux-x86_64.deb && \
-    rm bazel_0.27.1-linux-x86_64.deb
+ENV BAZEL_VERSION $BAZEL_VERSION
+RUN wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb && \
+    dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb && \
+    rm bazel_${BAZEL_VERSION}-linux-x86_64.deb
 
 # Download tensorflow sources
-ARG TF_VERSION="v2.1.0-rc1"
 ENV TF_VERSION $TF_VERSION
 RUN cd /opt && git clone https://github.com/tensorflow/tensorflow.git --branch $TF_VERSION --single-branch
 
@@ -50,7 +56,6 @@ RUN cd /opt/tensorflow && \
     bazel clean && rm -Rf /root/.cache
 
 # Install GOCV
-ARG OPENCV_VERSION="4.1.2"
 ENV OPENCV_VERSION $OPENCV_VERSION
 RUN cd /tmp && \
     curl -Lo opencv.zip https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip && \
@@ -92,3 +97,54 @@ RUN cd /tmp && git clone https://github.com/google-coral/edgetpu.git && \
 # Configure the Go version to be used
 ENV GO_ARCH "amd64"
 ENV GOARCH=amd64
+
+# Install Go
+ENV GO_VERSION $GO_VERSION
+RUN curl -kLo go${GO_VERSION}.linux-${GO_ARCH}.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \
+    tar -C /usr/local -xzf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz && \
+    rm go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
+
+FROM ubuntu:18.04 as builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config zip zlib1g-dev unzip wget bash-completion git curl \
+    build-essential patch g++ python python-future python3 ca-certificates \
+    libc6-dev libstdc++6 libusb-1.0-0
+
+# Copy all libraries, includes and go
+COPY --from=base /usr/local/. /usr/local/.
+COPY --from=base /opt/tensorflow /opt/tensorflow 
+
+ENV GOOS=linux
+ENV CGO_ENABLED=1
+ENV CGO_CFLAGS=-I/opt/tensorflow
+ENV PATH /usr/local/go/bin:/go/bin:${PATH}
+ENV GOPATH /go
+
+# Create the build directory
+RUN mkdir /build
+WORKDIR /build
+ADD . .
+RUN make
+
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends libusb-1.0 libc++-7-dev wget unzip ca-certificates libdc1394-22 libavcodec57 libavformat57 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN mkdir -p /opt/doods
+WORKDIR /opt/doods
+COPY --from=builder /usr/local/lib/. /usr/local/lib/.
+COPY --from=builder /build/doods /opt/doods/doods
+RUN ldconfig
+
+# Download sample models
+RUN mkdir models
+RUN wget https://storage.googleapis.com/download.tensorflow.org/models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && unzip coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && rm coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip && mv detect.tflite models/coco_ssd_mobilenet_v1_1.0_quant.tflite && rm labelmap.txt
+RUN wget https://dl.google.com/coral/canned_models/coco_labels.txt && mv coco_labels.txt models/coco_labels0.txt
+RUN wget http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_v2_coco_2018_01_28.tar.gz && tar -zxvf faster_rcnn_inception_v2_coco_2018_01_28.tar.gz faster_rcnn_inception_v2_coco_2018_01_28/frozen_inference_graph.pb --strip=1 && mv frozen_inference_graph.pb models/faster_rcnn_inception_v2_coco_2018_01_28.pb && rm faster_rcnn_inception_v2_coco_2018_01_28.tar.gz
+RUN wget https://github.com/raw/amikelive/coco-labels/master/coco-labels-2014_2017.txt && mv coco-labels-2014_2017.txt models/coco_labels1.txt
+ADD config.yaml config.yaml
+
+CMD ["/opt/doods/doods", "-c", "/opt/doods/config.yaml", "api"]