Merge branch 'master' into neural_type_system2

Signed-off-by: Oleksii Kuchaiev <okuchaiev@nvidia.com>
NVIDIA · Feb 12, 2020 · 5bf06af · 5bf06af
2 parents 51120d9 + 233a0fe
commit 5bf06af
Show file tree

Hide file tree

Showing 18 changed files with 171 additions and 113 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,17 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+env
+pip-log.txt
+pip-delete-this-directory.txt
+.tox
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+*.log
+.git
diff --git a/Dockerfile b/Dockerfile
@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:experimental
+
 # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:19.11-py3
+ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:20.01-py3
 
-FROM ${FROM_IMAGE_NAME}
+# build an image that includes only the nemo dependencies, ensures that dependencies
+# are included first for optimal caching, and useful for building a development
+# image (by specifying build target as `nemo-deps`)
+FROM ${BASE_IMAGE} as nemo-deps
 
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
@@ -25,27 +30,42 @@ RUN apt-get update && \
     python-dev && \
     rm -rf /var/lib/apt/lists/*
 
+# install onnx trt open source plugins
 ENV PATH=$PATH:/usr/src/tensorrt/bin
 WORKDIR /tmp/onnx-trt
 COPY scripts/docker/onnx-trt.patch .
-RUN git clone -n https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && git checkout 8716c9b && git submodule update --init --recursive && patch -f < ../onnx-trt.patch && \
-    mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig
-
-WORKDIR /workspace/nemo
-ARG NEMO_GIT_BRANCH="master"
+RUN git clone -n https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && \
+    git checkout 8716c9b && git submodule update --init --recursive && patch -f < ../onnx-trt.patch && \
+    mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && \
+    make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig && rm -rf /tmp/onnx-tensorrt
 
+# install nemo dependencies
+WORKDIR /tmp/nemo
 COPY requirements/requirements_docker.txt requirements.txt
-RUN pip install \
-    --disable-pip-version-check --no-cache-dir\
-    --upgrade -r requirements.txt && \
-    pip install --no-cache-dir \
-    git+git://github.com/NVIDIA/NeMo.git@${NEMO_GIT_BRANCH}#egg=project[all]
-
-RUN printf "#!/bin/bash\njupyter lab --no-browser --allow-root --ip=0.0.0.0" >> start-jupyter.sh && \
-    chmod +x start-jupyter.sh
-
+RUN pip install --disable-pip-version-check --no-cache-dir -r requirements.txt
 
+# copy nemo source into a scratch image
+FROM scratch as nemo-src
+COPY . .
 
+# start building the final container
+FROM nemo-deps as nemo
+ARG NEMO_VERSION
+ARG BASE_IMAGE
+# Check that NEMO_VERSION is set. Build will fail without this. Expose NEMO and base container
+# version information as runtime environment variable for introspection purposes
+RUN /usr/bin/test -n "$NEMO_VERSION" && \
+    /bin/echo "export NEMO_VERSION=${NEMO_VERSION}" >> /root/.bashrc && \
+    /bin/echo "export BASE_IMAGE=${BASE_IMAGE}" >> /root/.bashrc
+RUN --mount=from=nemo-src,target=/tmp/nemo cd /tmp/nemo && pip install ".[all]"
 
+# copy scripts/examples/tests into container for end user
+WORKDIR /workspace/nemo
+COPY scripts /workspace/nemo/scripts
+COPY examples /workspace/nemo/examples
+COPY tests /workspace/nemo/tests
+COPY README.rst LICENSE /workspace/nemo/
 
+RUN printf "#!/bin/bash\njupyter lab --no-browser --allow-root --ip=0.0.0.0" >> start-jupyter.sh && \
+    chmod +x start-jupyter.sh
 
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,12 +1,14 @@
 pipeline {
-  agent any
-  environment {
-      PATH="/home/mrjenkins/anaconda3/envs/py37p1.4.0/bin:$PATH"
+  agent {
+        docker {
+            image 'nvcr.io/nvidia/pytorch:20.01-py3'
+            args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home --shm-size=8g'
+        }
   }
   options {
     timeout(time: 1, unit: 'HOURS')
     disableConcurrentBuilds()
-   }
+  }
   stages {
 
     stage('PyTorch version') {
@@ -16,33 +18,17 @@ pipeline {
     }
     stage('Install test requirements') {
       steps {
-        sh 'pip install -r requirements/requirements_test.txt'
+        sh 'apt-get update && apt-get install -y bc && pip install -r requirements/requirements_test.txt'
       }
     }
     stage('Code formatting checks') {
       steps {
         sh 'python setup.py style'
       }
     }
-    stage('Unittests general') {
-      steps {
-        sh './reinstall.sh && python -m unittest tests/*.py'
-      }
-    }
-
-    stage('Unittests ASR') {
-      steps {
-        sh 'python -m unittest tests/asr/*.py'
-      }
-    }
-    stage('Unittests NLP') {
+    stage('Unittests ALL') {
       steps {
-        sh 'python -m unittest tests/nlp/*.py'
-      }
-    }
-    stage('Unittests TTS') {
-      steps {
-        sh 'python -m unittest tests/tts/*.py'
+        sh './reinstall.sh && python -m unittest'
       }
     }
 
@@ -147,8 +133,6 @@ pipeline {
       }
     }
 
-
-
     stage('NLP-ASR processing') {
       failFast true
       parallel { 

diff --git a/docs/docs_zh/sources/source/collections/nemo_asr.rst b/docs/docs_zh/sources/source/collections/nemo_asr.rst
@@ -1,17 +1,17 @@
-NeMo_ASR collection
+NeMo ASR collection
 ===================
 
 语音数据处理模块
 ------------------------------
-.. automodule:: nemo_asr.data_layer
+.. automodule:: nemo.collections.asr.data_layer
     :members:
     :undoc-members:
     :show-inheritance:
     :exclude-members: forward
 
 语音识别模块
 ------------------------------------
-.. automodule:: nemo_asr.jasper
+.. automodule:: nemo.collections.asr.jasper
     :members:
     :undoc-members:
     :show-inheritance:

diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst
@@ -1,76 +1,100 @@
-NeMo_NLP collection
+NeMo NLP collection
 ===================
 
 NLP 数据处理模块
 ---------------------------
-.. automodule:: nemo_nlp.data_layers
+.. automodule:: nemo.collections.nlp.data.datasets
     :members:
     :undoc-members:
     :show-inheritance:
-    :exclude-members: forward
 
 NLP 分词器
 --------------
 
-.. automodule:: nemo_nlp.data.tokenizers.bert_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.bert_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.char_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.char_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.gpt2_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.gpt2_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.spc_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.sentencepiece_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.tokenizer_spec
+.. automodule:: nemo.collections.nlp.data.tokenizers.tokenizer_spec
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.word_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.word_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
-.. automodule:: nemo_nlp.data.tokenizers.yttm_tokenizer
+.. automodule:: nemo.collections.nlp.data.tokenizers.youtokentome_tokenizer
    :members:
    :undoc-members:
    :show-inheritance:
 
 NLP 神经模块
 ------------------
 
-.. automodule:: nemo_nlp.modules.classifiers
+
+.. automodule:: nemo.collections.nlp.nm.data_layers
    :members:
    :undoc-members:
    :show-inheritance:
    :exclude-members: forward
 
 
-.. automodule:: nemo_nlp.modules.losses
+.. automodule:: nemo.collections.nlp.nm.losses
    :members:
    :undoc-members:
    :show-inheritance:
    :exclude-members: forward
 
 
-.. automodule:: nemo_nlp.modules.pytorch_utils
+.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_classification_nm
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :exclude-members: forward
+
+.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_regression_nm
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :exclude-members: forward
+
+.. automodule:: nemo.collections.nlp.nm.trainables.common.token_classification_nm
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :exclude-members: forward
+
+.. automodule:: nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm
+   :members:
+   :undoc-members:
+   :show-inheritance:
+   :exclude-members: forward
+
+.. automodule:: nemo.collections.nlp.nm.trainables.dialogue_state_tracking.state_tracking_trade_nm
    :members:
    :undoc-members:
    :show-inheritance:
    :exclude-members: forward
 
-.. automodule:: nemo_nlp.modules.transformer_nm
+.. automodule:: nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm
    :members:
    :undoc-members:
    :show-inheritance:
@@ -79,7 +103,7 @@ NLP 神经模块
 NLP Hugging Face 神经模块
 -------------------------------
 
-.. automodule:: nemo_nlp.huggingface.bert
+.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm
    :members:
    :undoc-members:
    :show-inheritance:

diff --git a/docs/docs_zh/sources/source/collections/nemo_tts.rst b/docs/docs_zh/sources/source/collections/nemo_tts.rst
@@ -1,25 +1,25 @@
-NeMo_TTS collection
+NeMo TTS collection
 ===================
 
 语音数据处理模块
 ------------------------------
-.. automodule:: nemo_tts.data_layers
+.. automodule:: nemo.collections.tts.data_layers
     :members:
     :undoc-members:
     :show-inheritance:
     :exclude-members: forward
 
 Tacotron 2 模块
 ------------------
-.. automodule:: nemo_tts.tacotron2_modules
+.. automodule:: nemo.collections.tts.tacotron2_modules
     :members:
     :undoc-members:
     :show-inheritance:
     :exclude-members: forward
 
 Waveglow 模块
 ------------------
-.. automodule:: nemo_tts.waveglow_modules
+.. automodule:: nemo.collections.tts.waveglow_modules
     :members:
     :undoc-members:
     :show-inheritance:

diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py
@@ -25,11 +25,6 @@
 
 sys.path.insert(0, os.path.abspath("."))
 sys.path.insert(0, os.path.abspath("../../../"))
-sys.path.insert(0, os.path.abspath("../../../nemo/nemo"))
-sys.path.insert(0, os.path.abspath("../../../collections"))
-sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr"))
-sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp"))
-# sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr"))
 
 # ---- Mocking up the classes. -----
 MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'}
@@ -63,6 +58,7 @@ def __getattr__(cls, name):
     'h5py',
     'kaldi_io',
     'transformers',
+    'transformers.tokenization_bert',
 ]
 
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

diff --git a/docs/sources/source/collections/nemo_asr.rst b/docs/sources/source/collections/nemo_asr.rst
@@ -1,17 +1,17 @@
-NeMo_ASR collection
+NeMo ASR collection
 ===================
 
 Speech data processing modules
 ------------------------------
-.. automodule:: nemo_asr.data_layer
+.. automodule:: nemo.collections.asr.data_layer
     :members:
     :undoc-members:
     :show-inheritance:
     :exclude-members: forward
 
 Automatic Speech Recognition modules
 ------------------------------------
-.. automodule:: nemo_asr.jasper
+.. automodule:: nemo.collections.asr.jasper
     :members:
     :undoc-members:
     :show-inheritance: