From 8c80a444c50a303f4fd8a92977d092136e7f6f38 Mon Sep 17 00:00:00 2001 From: jay-m-dev Date: Thu, 27 Jul 2023 13:34:22 -0700 Subject: [PATCH 1/2] 1dockerfile multimach, unittest fix, prod/rpi dockerfile removed --- .env | 6 +- docker-compose-int-test.yml | 2 + docker-compose-multi-machine.yml | 21 ++ docker-compose-production.yml | 6 +- docker-compose.yml | 2 + docker/lab/Dockerfile | 80 ++++++-- docker/lab/Dockerfile_production | 132 ------------ docker/lab/rpi.Dockerfile | 242 ---------------------- docker/lab/rpi_production.Dockerfile | 249 ----------------------- docker/machine/Dockerfile | 40 +++- docker/machine/Dockerfile_production | 64 ------ docker/machine/rpi.Dockerfile | 95 --------- docker/machine/rpi_production.Dockerfile | 99 --------- docs/guides/developerGuide.md | 6 +- lab/package.json | 2 +- machine/learn/skl_utils.py | 1 + machine/test/learn_tests.py | 88 ++++---- 17 files changed, 180 insertions(+), 955 deletions(-) delete mode 100644 docker/lab/Dockerfile_production delete mode 100644 docker/lab/rpi.Dockerfile delete mode 100644 docker/lab/rpi_production.Dockerfile delete mode 100644 docker/machine/Dockerfile_production delete mode 100644 docker/machine/rpi.Dockerfile delete mode 100644 docker/machine/rpi_production.Dockerfile diff --git a/.env b/.env index a7cb8b073..ec535b6b4 100644 --- a/.env +++ b/.env @@ -7,5 +7,9 @@ # Leave this set to 0 on the GitHub repo so the unit and # integration tests do not need to have wheels (until we # find a convenient way to use wheels on GitHub) -TAG=0.20.a0 +# DOCKER_BUILD_ENV - the environment to use for the docker build +# Set to dev for development and prod for production +# The corresponding docker image will be built using this variable +TAG=1.0.a0 USE_WHEELS=1 +DOCKER_BUILD_ENV=dev diff --git a/docker-compose-int-test.yml b/docker-compose-int-test.yml index a30310fca..46734e895 100644 --- a/docker-compose-int-test.yml +++ b/docker-compose-int-test.yml @@ -22,6 +22,7 @@ services: build: context: . dockerfile: docker/lab/Dockerfile + target: ${DOCKER_BUILD_ENV} args: # - USE_WHEELS=${USE_WHEELS} - USE_WHEELS=0 @@ -47,6 +48,7 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} args: # - USE_WHEELS=${USE_WHEELS} - USE_WHEELS=0 diff --git a/docker-compose-multi-machine.yml b/docker-compose-multi-machine.yml index de2fdd07d..8cc3887dd 100644 --- a/docker-compose-multi-machine.yml +++ b/docker-compose-multi-machine.yml @@ -5,6 +5,9 @@ services: build: context: . dockerfile: docker/lab/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -24,6 +27,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -43,6 +49,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -62,6 +71,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -81,6 +93,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -99,6 +114,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: @@ -118,6 +136,9 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: ${DOCKER_BUILD_ENV} + args: + - USE_WHEELS=${USE_WHEELS} tty: true stdin_open: true volumes: diff --git a/docker-compose-production.yml b/docker-compose-production.yml index 964101164..08e473a55 100644 --- a/docker-compose-production.yml +++ b/docker-compose-production.yml @@ -5,7 +5,8 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile_production + dockerfile: docker/lab/Dockerfile + target: prod image: "aliro_lab:${TAG}" tty: true stdin_open: true @@ -25,7 +26,8 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile_production + dockerfile: docker/machine/Dockerfile + target: prod image: "aliro_machine:${TAG}" tty: true stdin_open: true diff --git a/docker-compose.yml b/docker-compose.yml index f411eb5ab..254a5a06f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ services: build: context: . dockerfile: docker/lab/Dockerfile + target: dev args: - USE_WHEELS=${USE_WHEELS} tty: true @@ -25,6 +26,7 @@ services: build: context: . dockerfile: docker/machine/Dockerfile + target: dev args: - USE_WHEELS=${USE_WHEELS} tty: true diff --git a/docker/lab/Dockerfile b/docker/lab/Dockerfile index 03998df11..f70963810 100644 --- a/docker/lab/Dockerfile +++ b/docker/lab/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: Build and compile dependencies -FROM python:3.7.16-slim-buster AS builder +FROM python:3.7.16-slim-buster as base ARG USE_WHEELS ARG wheel_directory=docker/wheels @@ -42,10 +42,6 @@ RUN if [ ${USE_WHEELS} -eq 1 ] ; \ fi RUN rm /root/wheel/*.whl -# # These requirements need to be built, using wheels throws errors -# COPY ${docker_filepath}/requirements-build.txt /root/ -# RUN pip install --no-cache-dir -r /root/requirements-build.txt - ## Webserver RUN rm /etc/apache2/sites-enabled/* COPY ${docker_filepath}/ports.conf /etc/apache2/ @@ -75,19 +71,19 @@ COPY lab/package-lock.json /appsrc/lab/ RUN dos2unix /appsrc/lab/*.json RUN npm install --silent --progress=false -# Stage 2: Final runtime image -FROM python:3.7.16-slim-buster +# Stage 2a: build dev version of lab +FROM python:3.7.16-slim-buster as dev ARG docker_filepath=docker/lab/files # Copy installed packages from the builder stage -COPY --from=builder /usr/local /usr/local +COPY --from=base /usr/local /usr/local # Copy Apache configuration -COPY --from=builder /etc/apache2 /etc/apache2 +COPY --from=base /etc/apache2 /etc/apache2 # Copy webapp and lab source code -COPY --from=builder /appsrc/lab /appsrc/lab +COPY --from=base /appsrc/lab /appsrc/lab # Copy scripts and other files COPY ${docker_filepath}/start.sh /root/ @@ -113,13 +109,6 @@ RUN apt-get update && apt-get install -y dos2unix git build-essential && \ apt-get autoremove -y && \ apt-get clean && \ chmod +x /root/*.sh - # rm -rf /src/scikit-surprise -# RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' -# RUN dos2unix /etc/profile.d/tag.sh -# RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh - -# Clean up -# RUN apt-get remove -y wget xz-utils # Expose ports and set working directory EXPOSE 5080 @@ -131,3 +120,60 @@ CMD ["/bin/bash", "/root/start.sh"] # Entrypoint ENTRYPOINT ["/root/entrypoint.sh"] +# Stage 2b: build prod version of lab +FROM python:3.7.16-slim-buster as prod +ARG docker_filepath=docker/lab/files + +# Copy src for lab +COPY lab /appsrc/lab +COPY ai /appsrc/ai + +# Copy installed packages from the builder stage +COPY --from=base /usr/local /usr/local + +# Copy Apache configuration +COPY --from=base /etc/apache2 /etc/apache2 + +# Copy webapp and lab source code +COPY --from=base /appsrc/lab /appsrc/lab + +# Copy scripts and other files +COPY ${docker_filepath}/start.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ +COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd +COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ +COPY .env /etc/profile.d/ + +# These requirements need to be built, using wheels throws errors +COPY ${docker_filepath}/requirements-build.txt /root/ + +# Copy knowledgebases +COPY 'data/knowledgebases/sklearn-benchmark-data-knowledgebase-r6.tsv.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_classification_metafeatures.csv.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_regression_results.pkl.gz' /appsrc/data/knowledgebases/ +COPY 'data/knowledgebases/pmlb_regression_metafeatures.csv.gz' /appsrc/data/knowledgebases/ + +# Set version and build environment +ENV BUILD_ENV='prod' +RUN apt-get update && apt-get install -y dos2unix git build-essential && \ + pip install --no-cache-dir -r /root/requirements-build.txt && \ + cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' && \ + dos2unix /etc/profile.d/tag.sh && \ + sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get remove -y dos2unix git build-essential && \ + apt-get autoremove -y && \ + apt-get clean && \ + chmod +x /root/*.sh + +# Expose ports and set working directory +EXPOSE 5080 +WORKDIR /appsrc/lab/ + +# Start the webserver +CMD ["/bin/bash", "/root/start.sh"] + +# Entrypoint +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/lab/Dockerfile_production b/docker/lab/Dockerfile_production deleted file mode 100644 index becfeeea2..000000000 --- a/docker/lab/Dockerfile_production +++ /dev/null @@ -1,132 +0,0 @@ -FROM python:3.7.16-slim-buster - -# copy src for lab -COPY lab /appsrc/lab -COPY ai /appsrc/ai - - -#nodejs -# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH - -RUN apt-get update --fix-missing && \ - apt-get install -y wget xz-utils git \ - openssh-client openssh-server telnet apache2 \ - net-tools iputils-ping \ - screen ngrep \ - mercurial subversion \ - build-essential cmake cpio mesa-common-dev \ - dos2unix \ - --no-install-recommends && \ - ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && wget --quiet https://nodejs.org/dist/v16.18.1/node-v16.18.1-linux-$ARCH.tar.xz -O ~/node.tar.xz && \ - tar -xvf ~/node.tar.xz -C /usr/local/ --strip-components=1 --no-same-owner && \ - rm ~/node.tar.xz && \ - ln -s /usr/local/bin/node /usr/local/bin/nodejs && \ - rm -rf /var/lib/apt/lists/* && \ - apt-get remove -y wget && \ - apt-get clean - -ARG docker_filepath=docker/lab/files - -# RUN apt-get update --fix-missing && apt-get install -y \ -# vim openssh-client openssh-server telnet apache2 \ -# net-tools iputils-ping xz-utils \ -# screen ngrep ca-cacert \ -# mercurial subversion \ -# build-essential cmake lsb-core cpio mesa-common-dev \ -# dos2unix \ -# --no-install-recommends && \ -# apt-get clean && \ -# rm -rf /var/lib/apt/lists/* - -# setup python environment -COPY ${docker_filepath}/requirements.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements.txt - -## Webserver - -RUN rm /etc/apache2/sites-enabled/* -COPY ${docker_filepath}/ports.conf /etc/apache2/ -RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled - -# setup node environment -RUN npm install -g pm2 --silent --progress=false - -# install lab/webapp/node_modules to an anon volume -WORKDIR /appsrc/lab/webapp -COPY lab/webapp/package.json /appsrc/lab/webapp/ -COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ -RUN dos2unix /appsrc/lab/webapp/*.json -RUN npm install --silent --progress=false - -# install lab/node_modules to an anon volume -WORKDIR /appsrc/lab -COPY lab/package.json /appsrc/lab/ -COPY lab/package-lock.json /appsrc/lab/ -RUN dos2unix /appsrc/lab/*.json -RUN npm install --silent --progress=false - -COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ -COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd -COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ - -WORKDIR /root/ - -# Webserver - paiwww -COPY ${docker_filepath}/start.sh /root/ - -## Utility script, used when starting ai -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - - -## Aliro Lab server -COPY ${docker_filepath}/entrypoint.sh /root/ -RUN ["chmod", "+x", "/root/entrypoint.sh"] - -RUN dos2unix /root/start.sh \ - && dos2unix /root/wait-for-it.sh \ - && dos2unix /root/entrypoint.sh - - -# copy knowledgebase -COPY 'data/knowledgebases/sklearn-benchmark-data-knowledgebase-r6.tsv.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_classification_metafeatures.csv.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_regression_results.pkl.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_regression_metafeatures.csv.gz' /appsrc/data/knowledgebases/ - - -# set version and build environment; tag.sh is sourced in entrypoint.sh -ENV BUILD_ENV='prod' -COPY .env /etc/profile.d/ -RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' -RUN dos2unix /etc/profile.d/tag.sh -RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh - - -# Start the webserver -CMD ["/bin/bash", "/root/start.sh"] - -# EXPOSE 443 -EXPOSE 5080 -WORKDIR /appsrc/lab/ -ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/lab/rpi.Dockerfile b/docker/lab/rpi.Dockerfile deleted file mode 100644 index fa04fbdad..000000000 --- a/docker/lab/rpi.Dockerfile +++ /dev/null @@ -1,242 +0,0 @@ -FROM ubuntu:bionic - -#RUN apt-get update --fix-missing && apt-get install -y wget - -#nodejs -#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH - -RUN groupadd --gid 1000 node \ - && useradd --uid 1000 --gid node --shell /bin/bash --create-home node - -ENV NODE_VERSION 12.22.9 - -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && set -ex \ - # libatomic1 for arm - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - 1C050899334244A8AF75E53792EF661D867B9DFA \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && apt-mark auto '.*' > /dev/null \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ - # smoke tests - && node --version \ - && npm --version - -ENV YARN_VERSION 1.22.5 - -RUN set -ex \ - && savedAptMark="$(apt-mark showmanual)" \ - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 6A010C5166006599AA17F08146C2130DFD2497F5 \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ - && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && mkdir -p /opt \ - && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ - && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && apt-mark auto '.*' > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - # smoke test - && yarn --version - -ARG docker_filepath=docker/lab/files -ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel - -RUN apt-get update --fix-missing && apt-get install -y \ - openssh-client openssh-server telnet apache2 \ - net-tools iputils-ping xz-utils \ - ngrep ca-cacert \ - build-essential cmake lsb-core cpio mesa-common-dev \ - dos2unix curl git libatlas-base-dev gfortran \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Ensure specific python version -# See: https://stackoverflow.com/a/58562728/1730417 -# and: https://askubuntu.com/a/1176271/260220 -RUN apt-get update && apt-get install -y software-properties-common -# RUN -E add-apt-repository 'ppa:deadsnakes/ppa' -# RUN -E add-apt-repository ppa:deadsnakes/ppa -# RUN add-apt-repository 'ppa:deadsnakes/ppa' -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 -RUN update-alternatives --set python /usr/bin/python3.7 -RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python get-pip.py --force-reinstall && \ - rm get-pip.py - -# ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -ENV TZ=America/New_York -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - libbluetooth-dev \ - tk-dev \ - uuid-dev \ - && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget python3.7-venv python3-numpy \ - && apt-get clean \ - && rm -rf /var/lib/apt/list/* - -RUN python3 --version -RUN python --version -RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH -RUN python --version -RUN pip --version - -# setup python environment -COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ -RUN mkdir /root/wheel -COPY ${python_wheel_directory}/*.whl /root/wheel/ -RUN pip3 install --no-cache-dir cython -RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt -### bill's surprise fork -# NOTE: For some reason, this install doesn't seem to recognize the installed numpy -# when run on Docker. As a workaround, the `wheel` directory includes a wheel -# for this install. -#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 -RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise - -## Webserver - -RUN rm /etc/apache2/sites-enabled/* -COPY ${docker_filepath}/ports.conf /etc/apache2/ -RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled - -# setup node environment -RUN npm install -g pm2 webpack --silent --progress=false - -# install lab/webapp/node_modules to an anon volume -WORKDIR /appsrc/lab/webapp -COPY lab/webapp/package.json /appsrc/lab/webapp/ -COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ -RUN dos2unix /appsrc/lab/webapp/*.json -RUN npm install --silent --progress=false - -# install lab/node_modules to an anon volume -WORKDIR /appsrc/lab -COPY lab/package.json /appsrc/lab/ -COPY lab/package-lock.json /appsrc/lab/ -RUN dos2unix /appsrc/lab/*.json -RUN npm install --silent --progress=false - -RUN apt-get install -y libgfortran5 liblapack-dev - -COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ -COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd -COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ - -WORKDIR /root/ - -# Webserver - paiwww -COPY ${docker_filepath}/start.sh /root/ - -## Utility script, used when starting ai -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - - -## PennAI Lab server -COPY ${docker_filepath}/entrypoint.sh /root/ -RUN ["chmod", "+x", "/root/entrypoint.sh"] - -RUN dos2unix /root/start.sh \ - && dos2unix /root/wait-for-it.sh \ - && dos2unix /root/entrypoint.sh - -# set version and build environment; tag.sh is sourced in entrypoint.sh -ENV BUILD_ENV='dev' -COPY .env /etc/profile.d/ -RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' -RUN dos2unix /etc/profile.d/tag.sh -RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh - - -# Start the webserver -CMD ["/bin/bash", "/root/start.sh"] - -# EXPOSE 443 -EXPOSE 5080 -WORKDIR /appsrc/lab/ -ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/lab/rpi_production.Dockerfile b/docker/lab/rpi_production.Dockerfile deleted file mode 100644 index 78fc0dece..000000000 --- a/docker/lab/rpi_production.Dockerfile +++ /dev/null @@ -1,249 +0,0 @@ -FROM ubuntu:bionic - -# copy src for lab -COPY lab /appsrc/lab -COPY ai /appsrc/ai - -#RUN apt-get update --fix-missing && apt-get install -y wget - -#nodejs -#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH - -RUN groupadd --gid 1000 node \ - && useradd --uid 1000 --gid node --shell /bin/bash --create-home node - -ENV NODE_VERSION 12.22.9 - -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && set -ex \ - # libatomic1 for arm - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - 1C050899334244A8AF75E53792EF661D867B9DFA \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && apt-mark auto '.*' > /dev/null \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ - # smoke tests - && node --version \ - && npm --version - -ENV YARN_VERSION 1.22.5 - -RUN set -ex \ - && savedAptMark="$(apt-mark showmanual)" \ - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 6A010C5166006599AA17F08146C2130DFD2497F5 \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ - && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ - && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && mkdir -p /opt \ - && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ - && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ - && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ - && apt-mark auto '.*' > /dev/null \ - && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - # smoke test - && yarn --version - -ARG docker_filepath=docker/lab/files -ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel - -RUN apt-get update --fix-missing && apt-get install -y \ - openssh-client openssh-server telnet apache2 \ - net-tools iputils-ping xz-utils \ - ngrep ca-cacert \ - build-essential cmake lsb-core cpio mesa-common-dev \ - dos2unix curl git libatlas-base-dev gfortran \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Ensure specific python version -# See: https://stackoverflow.com/a/58562728/1730417 -# and: https://askubuntu.com/a/1176271/260220 -RUN apt-get update && apt-get install -y software-properties-common -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 -RUN update-alternatives --set python /usr/bin/python3.7 -RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python get-pip.py --force-reinstall && \ - rm get-pip.py - -# ensure local python is preferred over distribution python -ENV PATH /usr/local/bin:$PATH - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -ENV TZ=America/New_York -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - libbluetooth-dev \ - tk-dev \ - uuid-dev \ - && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget python3.7-venv python3-numpy \ - && apt-get clean \ - && rm -rf /var/lib/apt/list/* - -RUN python3 --version -RUN python --version -RUN python -m venv /venv -ENV PATH=/venv/bin:$PATH -RUN python --version -RUN pip --version - -# setup python environment -COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ -RUN mkdir /root/wheel -COPY ${python_wheel_directory}/*.whl /root/wheel/ -RUN pip3 install --no-cache-dir cython -RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt -### bill's surprise fork -# NOTE: For some reason, this install doesn't seem to recognize the installed numpy -# when run on Docker. As a workaround, the `wheel` directory includes a wheel -# for this install. -#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 -RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise - -## Webserver - -RUN rm /etc/apache2/sites-enabled/* -COPY ${docker_filepath}/ports.conf /etc/apache2/ -RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ - && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled - -# setup node environment -RUN npm install -g pm2 webpack --silent --progress=false - -# install lab/webapp/node_modules to an anon volume -WORKDIR /appsrc/lab/webapp -COPY lab/webapp/package.json /appsrc/lab/webapp/ -COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ -RUN dos2unix /appsrc/lab/webapp/*.json -RUN npm install --silent --progress=false - -# install lab/node_modules to an anon volume -WORKDIR /appsrc/lab -COPY lab/package.json /appsrc/lab/ -COPY lab/package-lock.json /appsrc/lab/ -RUN dos2unix /appsrc/lab/*.json -RUN npm install --silent --progress=false - -RUN apt-get install -y libgfortran5 liblapack-dev - -COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ -COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd -COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ - -WORKDIR /root/ - -# Webserver - paiwww -COPY ${docker_filepath}/start.sh /root/ - -## Utility script, used when starting ai -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - - -## PennAI Lab server -COPY ${docker_filepath}/entrypoint.sh /root/ -RUN ["chmod", "+x", "/root/entrypoint.sh"] - -RUN dos2unix /root/start.sh \ - && dos2unix /root/wait-for-it.sh \ - && dos2unix /root/entrypoint.sh - -# copy knowledgebase -COPY 'data/knowledgebases/sklearn-benchmark-data-knowledgebase-r6.tsv.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_classification_metafeatures.csv.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_regression_results.pkl.gz' /appsrc/data/knowledgebases/ -COPY 'data/knowledgebases/pmlb_regression_metafeatures.csv.gz' /appsrc/data/knowledgebases/ - -# set version and build environment; tag.sh is sourced in entrypoint.sh -ENV BUILD_ENV='prod' -COPY .env /etc/profile.d/ -RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' -RUN dos2unix /etc/profile.d/tag.sh -RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh - - -# Start the webserver -CMD ["/bin/bash", "/root/start.sh"] - -# EXPOSE 443 -EXPOSE 5080 -WORKDIR /appsrc/lab/ -ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/machine/Dockerfile b/docker/machine/Dockerfile index 2c38384b7..346fda4be 100644 --- a/docker/machine/Dockerfile +++ b/docker/machine/Dockerfile @@ -1,5 +1,5 @@ -# Stage 1: Build and install Python and Node.js dependencies -FROM python:3.7.16-slim-buster AS builder +# Stage 1: Build and install Python and NodeJS dependencies +FROM python:3.7.16-slim-buster as base ARG USE_WHEELS ARG wheel_directory=docker/wheels @@ -48,13 +48,11 @@ RUN pip install --no-cache-dir -r /root/requirements-build.txt # Setup node environment RUN npm install -g pm2 --silent --progress=false -# Stage 2: Final runtime image -FROM python:3.7.16-slim-buster - +# Stage 2a: Build dev version of machine +FROM python:3.7.16-slim-buster as dev ARG docker_filepath=docker/machine/files -# Copy installed Python dependencies -COPY --from=builder /usr/local /usr/local +COPY --from=base /usr/local /usr/local # Copy node_modules to an anonymous volume WORKDIR /appsrc/machine @@ -75,3 +73,31 @@ RUN apt-get update && apt-get install -y graphviz dos2unix && \ CMD ["/bin/bash", "/root/entrypoint.sh"] +# Stage 2b: Build prod version of machine +FROM python:3.7.16-slim-buster as prod +ARG docker_filepath=docker/machine/files + +# copy src for machine +COPY machine /appsrc/machine/ +COPY config/machine_config.json /appsrc/config/machine_config.json + +COPY --from=base /usr/local /usr/local + +# Copy node_modules to an anonymous volume +WORKDIR /appsrc/machine +COPY machine/package.json /appsrc/machine/ +COPY machine/package-lock.json /appsrc/machine/ +# Copy entrypoint script and other files +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ + +RUN apt-get update && apt-get install -y graphviz dos2unix && \ + dos2unix /appsrc/machine/*.json && \ + npm install --silent --progress=false && \ + dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh && \ + chmod +x /root/wait-for-it.sh && chmod +x /root/entrypoint.sh && \ + apt-get remove -y dos2unix && \ + apt-get clean + # apt-get autoremove -y && \ + +CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production deleted file mode 100644 index a91e95aa9..000000000 --- a/docker/machine/Dockerfile_production +++ /dev/null @@ -1,64 +0,0 @@ -FROM python:3.7.16-slim-buster - -# copy src for machine -COPY machine /appsrc/machine/ -COPY config/machine_config.json /appsrc/config/machine_config.json - -#nodejs -# RUN wget --quiet https://nodejs.org/dist/v12.22.9/node-v12.22.9-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -# ENV PATH /opt/node-v12.22.9-linux-x64/bin:$PATH - -ARG docker_filepath=docker/machine/files - -RUN apt-get update --fix-missing && apt-get install -y \ - wget openssh-client openssh-server graphviz \ - net-tools iputils-ping xz-utils \ - screen ngrep \ - mercurial subversion \ - build-essential cmake cpio mesa-common-dev \ - libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ - --no-install-recommends && \ - ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && wget --quiet https://nodejs.org/dist/v16.18.1/node-v16.18.1-linux-$ARCH.tar.xz -O ~/node.tar.xz && \ - tar -xvf ~/node.tar.xz -C /usr/local/ --strip-components=1 --no-same-owner && \ - rm ~/node.tar.xz && \ - ln -s /usr/local/bin/node /usr/local/bin/nodejs && \ - rm -rf /var/lib/apt/lists/* && \ - apt-get remove -y wget && \ - apt-get clean - -# setup python environment -COPY ${docker_filepath}/requirements.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements.txt -COPY ${docker_filepath}/requirements_shap.txt /root/ -RUN pip install --no-cache-dir -r /root/requirements_shap.txt - - -# setup node environment -RUN npm install -g pm2 --silent --progress=false - -# install node_modules to an anon volume -WORKDIR /appsrc/machine - -COPY machine/package.json /appsrc/machine/ -COPY machine/package-lock.json /appsrc/machine/ -RUN dos2unix /appsrc/machine/*.json -RUN npm install --silent --progress=false - -COPY ${docker_filepath}/entrypoint.sh /root/ -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - -CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/machine/rpi.Dockerfile b/docker/machine/rpi.Dockerfile deleted file mode 100644 index e3d5dfcb8..000000000 --- a/docker/machine/rpi.Dockerfile +++ /dev/null @@ -1,95 +0,0 @@ -FROM python:3.7-buster - -ENV NODE_VERSION 12.22.9 - -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && set -ex \ - # libatomic1 for arm - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - 1C050899334244A8AF75E53792EF661D867B9DFA \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && apt-mark auto '.*' > /dev/null \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ - # smoke tests - && node --version \ - && npm --version - -ARG docker_filepath=docker/machine/files -ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel - -RUN apt-get update --fix-missing && apt-get install -y \ - openssh-client openssh-server graphviz \ - net-tools iputils-ping xz-utils \ - ngrep \ - libatlas-base-dev \ - build-essential cmake cpio mesa-common-dev \ - libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -# Removed: lsb-core ca-cacert glibc - -# setup python environment -COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ -RUN mkdir /root/wheel -COPY ${python_wheel_directory}/*.whl /root/wheel/ -RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt - -# setup node environment -RUN npm install -g pm2 --silent --progress=false - -# install node_modules to an anon volume -WORKDIR /appsrc/machine - -COPY machine/package.json /appsrc/machine/ -COPY machine/package-lock.json /appsrc/machine/ -RUN dos2unix /appsrc/machine/*.json -RUN npm install --silent --progress=false - -COPY ${docker_filepath}/entrypoint.sh /root/ -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - -CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/machine/rpi_production.Dockerfile b/docker/machine/rpi_production.Dockerfile deleted file mode 100644 index 39a61fb65..000000000 --- a/docker/machine/rpi_production.Dockerfile +++ /dev/null @@ -1,99 +0,0 @@ -FROM python:3.7-buster - -# copy src for machine -COPY machine /appsrc/machine/ -COPY config/machine_config.json /appsrc/config/machine_config.json - -ENV NODE_VERSION 12.22.9 - -RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ - && case "${dpkgArch##*-}" in \ - amd64) ARCH='x64';; \ - ppc64el) ARCH='ppc64le';; \ - s390x) ARCH='s390x';; \ - arm64) ARCH='arm64';; \ - armhf) ARCH='armv7l';; \ - i386) ARCH='x86';; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac \ - && set -ex \ - # libatomic1 for arm - && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ - && rm -rf /var/lib/apt/lists/* \ - && for key in \ - 4ED778F539E3634C779C87C6D7062848A1AB005C \ - 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ - 1C050899334244A8AF75E53792EF661D867B9DFA \ - 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ - 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ - C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ - C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ - DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ - A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ - 108F52B48DB57BB0CC439B2997B01419BD92F80A \ - B9E2F5981AA6E0CD28160D9FF13993A75599653C \ - ; do \ - gpg --batch --keyserver hkp://keyring.debian.org:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ - gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ - gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ - done \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ - && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ - && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ - && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ - && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ - && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ - && apt-mark auto '.*' > /dev/null \ - && find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ - && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ - # smoke tests - && node --version \ - && npm --version - -ARG docker_filepath=docker/machine/files -ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel - -RUN apt-get update --fix-missing && apt-get install -y \ - openssh-client openssh-server graphviz \ - net-tools iputils-ping xz-utils \ - ngrep \ - libatlas-base-dev \ - build-essential cmake cpio mesa-common-dev \ - libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ - --no-install-recommends && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -# Removed: lsb-core ca-cacert glibc - -# setup python environment -COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ -RUN mkdir /root/wheel -COPY ${python_wheel_directory}/*.whl /root/wheel/ -RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt - -# setup node environment -RUN npm install -g pm2 --silent --progress=false - -# install node_modules to an anon volume -WORKDIR /appsrc/machine - -COPY machine/package.json /appsrc/machine/ -COPY machine/package-lock.json /appsrc/machine/ -RUN dos2unix /appsrc/machine/*.json -RUN npm install --silent --progress=false - -COPY ${docker_filepath}/entrypoint.sh /root/ -COPY ${docker_filepath}/wait-for-it.sh /root/ -RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh -RUN ["chmod", "+x", "/root/wait-for-it.sh"] - -CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docs/guides/developerGuide.md b/docs/guides/developerGuide.md index 5f12d7155..bbc57a0f2 100644 --- a/docs/guides/developerGuide.md +++ b/docs/guides/developerGuide.md @@ -92,10 +92,10 @@ start the AI service: python -m ai.ai -v -n 2 ``` - - Note: If `docker exec -it "aliro_lab_1" /bin/bash` returns + - Note: If `docker exec -it "aliro-lab-1" /bin/bash` returns **'Error: no such container'**, use `docker container ps` to get the name of the lab container - - Note: `docker attach aliro_lab_1` will attach to the lab container, but if + - Note: `docker attach aliro-lab-1` will attach to the lab container, but if the last command run by the startup script was not bash it will appear to hang. @@ -334,6 +334,8 @@ puts the test results and code coverage reports in the `.\target` directory To create a production release: +The following steps will be performed by following the **Release procedure** +steps below: - the source should be added to the `production` branch with a tagged commit - the production docker images should be added to DockerHub with appropriate tags diff --git a/lab/package.json b/lab/package.json index af8e9ae6e..bbae6b4ae 100644 --- a/lab/package.json +++ b/lab/package.json @@ -42,7 +42,7 @@ "devDependencies": { "@types/node": "^10.17.21", "jest": "^26.0.1", - "jest-html-reporter": "^3.1.0", + "jest-html-reporter": "^3.1.3", "jest-junit": "^10.0.0", "supertest": "^4.0.2", "typescript": "^3.8.3" diff --git a/machine/learn/skl_utils.py b/machine/learn/skl_utils.py index 18ce8e433..8654ff96e 100644 --- a/machine/learn/skl_utils.py +++ b/machine/learn/skl_utils.py @@ -1817,6 +1817,7 @@ def balanced_accuracy(y_true, y_pred): print("Holdout score: ", end="") print(model.score(testing_features, testing_target)) + # Application 2: predict outcome by fitted model # In this application, the input dataset may not include target column # Please comment this line below if there is no target column in input dataset diff --git a/machine/test/learn_tests.py b/machine/test/learn_tests.py index 40e0bae6a..c985cc9f3 100644 --- a/machine/test/learn_tests.py +++ b/machine/test/learn_tests.py @@ -497,10 +497,10 @@ def test_main_classification_algorithms(self, mock_get): assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) assert os.path.isfile( - '{}/confusion_matrix_{}.png'.format(outdir, _id)) + '{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile( '{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file @@ -570,8 +570,8 @@ def test_main_2(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test SHAP summary files # GradientBoostingClassifier for multiclass case using Kernel Explainer @@ -633,8 +633,8 @@ def test_main_3(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test SHAP summary files for class_id in range(3): @@ -693,8 +693,8 @@ def test_main_4(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test SHAP summary files - Linear Explainer, Generates only one matrix assert os.path.isfile('{}/shap_summary_curve{}_0_.png'.format(outdir, _id)) @@ -734,8 +734,8 @@ def test_main_5(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test SHAP summary files - Linear Explainer, Generates only one matrix assert os.path.isfile('{}/shap_summary_curve{}_0_.png'.format(outdir, _id)) @@ -774,10 +774,10 @@ def test_main_6(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test SHAP summary files for class_id in range(3): @@ -849,8 +849,8 @@ def test_main_7(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) assert os.path.isfile( '{}/grid_search_results_{}.csv'.format(outdir, _id)) @@ -922,8 +922,8 @@ def test_main_8(self, mock_get): assert train_score assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) assert os.path.isfile( '{}/grid_search_results_{}.csv'.format(outdir, _id)) @@ -1058,10 +1058,10 @@ def test_generate_results_1(): assert value['_scores']['train_score'] > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1102,10 +1102,10 @@ def test_generate_results_2(): assert train_score > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert not os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert not os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert not os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1148,12 +1148,12 @@ def test_generate_results_3(): assert value['_scores']['r2_score'] > 0 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert not os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/reg_cv_pred_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/reg_cv_resi_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/reg_cv_pred.png'.format(outdir, _id)) + assert os.path.isfile('{}/reg_cv_resi.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1179,10 +1179,10 @@ def test_generate_results_4(): assert value['_scores']['r2_score'] > 0 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert not os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert not os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert not os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1253,10 +1253,10 @@ def test_generate_results_6(): assert train_score > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert not os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert not os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert not os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1312,10 +1312,10 @@ def test_generate_results_7(): assert train_score > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert not os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert not os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert not os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1359,10 +1359,10 @@ def test_generate_results_9(): assert value['_scores']['train_score'] > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) @@ -1404,10 +1404,10 @@ def test_generate_results_10(): assert value['_scores']['train_score'] > 0.9 assert os.path.isfile('{}/prediction_values.json'.format(outdir)) assert os.path.isfile('{}/feature_importances.json'.format(outdir)) - assert os.path.isfile('{}/confusion_matrix_{}.png'.format(outdir, _id)) + assert os.path.isfile('{}/confusion_matrix.png'.format(outdir, _id)) # only has roc for binary outcome - assert not os.path.isfile('{}/roc_curve_{}.png'.format(outdir, _id)) - assert os.path.isfile('{}/imp_score_{}.png'.format(outdir, _id)) + assert not os.path.isfile('{}/roc_curve.png'.format(outdir, _id)) + assert os.path.isfile('{}/imp_score.png'.format(outdir, _id)) assert os.path.isfile('{}/scripts_reproduce_{}.py'.format(outdir, _id)) # test pickle file pickle_file = '{}/model_{}.pkl'.format(outdir, _id) From e2ab85be1449d87ee2f7d500b21ca71349575b88 Mon Sep 17 00:00:00 2001 From: jay-m-dev Date: Thu, 27 Jul 2023 14:21:23 -0700 Subject: [PATCH 2/2] 1dockerfile env vars --- .env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.env b/.env index ec535b6b4..620592600 100644 --- a/.env +++ b/.env @@ -7,9 +7,5 @@ # Leave this set to 0 on the GitHub repo so the unit and # integration tests do not need to have wheels (until we # find a convenient way to use wheels on GitHub) -# DOCKER_BUILD_ENV - the environment to use for the docker build -# Set to dev for development and prod for production -# The corresponding docker image will be built using this variable TAG=1.0.a0 USE_WHEELS=1 -DOCKER_BUILD_ENV=dev