From 90a87326ca372262d082748e1a914fb0ae300ebf Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 11:05:42 -0800 Subject: [PATCH 01/30] test Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3bd1946fe5fb..6772c0d68792 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,10 @@ pipeline { - agent any + agent { + docker { + image 'nvcr.io/nvidia/pytorch:20.01-py3' + args '--gpus all' + } + } environment { PATH="/home/mrjenkins/anaconda3/envs/py37p1.4.0/bin:$PATH" } From f1bc768b98b6487586a1d51ff8fe5f3222fa5ea6 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 11:27:12 -0800 Subject: [PATCH 02/30] --user mode Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6772c0d68792..1ad858218798 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -r requirements/requirements_test.txt' + sh 'pip install -u -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From 6860a874d67bff5200806472949c033f2ff74a0a Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 11:28:45 -0800 Subject: [PATCH 03/30] update jenkinsfile Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1ad858218798..e47888a8dd3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -u -r requirements/requirements_test.txt' + sh 'pip install --user -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From 712a35c36204ce15ca8e36c68ae74365a4c36cbd Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:00:18 -0800 Subject: [PATCH 04/30] try dockerfile Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e47888a8dd3a..df2489c791fd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,5 @@ pipeline { - agent { - docker { - image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--gpus all' - } - } + agent { dockerfile true } environment { PATH="/home/mrjenkins/anaconda3/envs/py37p1.4.0/bin:$PATH" } From 3c7c3445977a7b07f97161cc9a3d049b74c19087 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:18:24 -0800 Subject: [PATCH 05/30] revert docker to fixed container Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index df2489c791fd..e47888a8dd3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,10 @@ pipeline { - agent { dockerfile true } + agent { + docker { + image 'nvcr.io/nvidia/pytorch:20.01-py3' + args '--gpus all' + } + } environment { PATH="/home/mrjenkins/anaconda3/envs/py37p1.4.0/bin:$PATH" } From c1a916284e9971fa998fb988b7056cfb6bfd4256 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:20:37 -0800 Subject: [PATCH 06/30] update Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e47888a8dd3a..6772c0d68792 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,7 +21,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install --user -r requirements/requirements_test.txt' + sh 'pip install -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From 8144ea4ac0a3aaf69e3d7faa59dae60c70aa635f Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:25:40 -0800 Subject: [PATCH 07/30] remove python env Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6772c0d68792..018e4b9a9b4b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,9 +5,6 @@ pipeline { args '--gpus all' } } - environment { - PATH="/home/mrjenkins/anaconda3/envs/py37p1.4.0/bin:$PATH" - } options { timeout(time: 1, unit: 'HOURS') disableConcurrentBuilds() From b1aa3ff405d55ab6153efc0c5ba76a768cc12c3b Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:26:43 -0800 Subject: [PATCH 08/30] add user back Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 018e4b9a9b4b..7c420f93a5d2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -r requirements/requirements_test.txt' + sh 'pip install --user -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From b37fd55d713238f25aeecd956057dd0f9c5f397f Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:28:31 -0800 Subject: [PATCH 09/30] try to enter container as root Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7c420f93a5d2..777a77accfd7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--gpus all' + args '--gpus all --user 0:128' } } options { @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install --user -r requirements/requirements_test.txt' + sh 'pip install -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From a886598f31a9f74717d8ce8e8d677a70a502fdd5 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 15:29:57 -0800 Subject: [PATCH 10/30] install docker requirements Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 777a77accfd7..f60ba17e97c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -r requirements/requirements_test.txt' + sh 'pip install -r requirements/requirements_docker.txt' } } stage('Code formatting checks') { From 7d14c20206a08c7226a2b1c71d6f5322bc21eb2c Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 16:18:01 -0800 Subject: [PATCH 11/30] add wget to test reqs Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- requirements/requirements_test.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index f60ba17e97c6..777a77accfd7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -r requirements/requirements_docker.txt' + sh 'pip install -r requirements/requirements_test.txt' } } stage('Code formatting checks') { diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index 493b8268cfd1..321780254517 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -4,3 +4,4 @@ pytest-runner black isort[requirements] wrapt +wget From 68eee28e7bdbea7ab83543f7ef929811c67ee791 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 16:20:46 -0800 Subject: [PATCH 12/30] add nvidia-smi Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 777a77accfd7..a72b6e014818 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,6 +10,11 @@ pipeline { disableConcurrentBuilds() } stages { + stage('NVIDIA SMI') { + steps { + sh 'nvidia-smi' + } + } stage('PyTorch version') { steps { From a769e77c7a66d35d6da00f51ce6b2bae1f1b1f15 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 16:23:57 -0800 Subject: [PATCH 13/30] add nvidia runtime flag Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a72b6e014818..1c7c627b1ede 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--gpus all --user 0:128' + args '--runtime=nvidia --gpus all --user 0:128' } } options { From 1edf4b68878a20947b1eeca7c94cfbd0aec3f2f6 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 16:36:56 -0800 Subject: [PATCH 14/30] update Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1c7c627b1ede..07b70be9752a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--runtime=nvidia --gpus all --user 0:128' + args '--gpus all --user 0:128' } } options { @@ -12,7 +12,7 @@ pipeline { stages { stage('NVIDIA SMI') { steps { - sh 'nvidia-smi' + sh 'cat /dev/null > /etc/shinit && nvidia-smi' } } From 42a0bc5c678d080d3c0f06b2ddda0d82f20f70b1 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 16:43:16 -0800 Subject: [PATCH 15/30] test Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 07b70be9752a..be805fbc362f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--gpus all --user 0:128' + args '--device=/dev/nvidia0 --gpus all --user 0:128' } } options { From da78134da04754c3f647bbbfffe9882e3686bf4a Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 17:08:42 -0800 Subject: [PATCH 16/30] mount folder inside docker Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index be805fbc362f..6023520e9962 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,20 +2,13 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--device=/dev/nvidia0 --gpus all --user 0:128' + args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home' } } options { timeout(time: 1, unit: 'HOURS') disableConcurrentBuilds() } - stages { - stage('NVIDIA SMI') { - steps { - sh 'cat /dev/null > /etc/shinit && nvidia-smi' - } - } - stage('PyTorch version') { steps { sh 'python -c "import torch; print(torch.__version__)"' @@ -33,23 +26,7 @@ pipeline { } stage('Unittests general') { steps { - sh './reinstall.sh && python -m unittest tests/*.py' - } - } - - stage('Unittests ASR') { - steps { - sh 'python -m unittest tests/asr/*.py' - } - } - stage('Unittests NLP') { - steps { - sh 'python -m unittest tests/nlp/*.py' - } - } - stage('Unittests TTS') { - steps { - sh 'python -m unittest tests/tts/*.py' + sh './reinstall.sh && python -m unittest' } } From 86f8feb6cb4b594ca646bc126bf09078a74730c2 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 17:10:15 -0800 Subject: [PATCH 17/30] unittest Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6023520e9962..fb31e238d36e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { sh 'python setup.py style' } } - stage('Unittests general') { + stage('Unittests ALL') { steps { sh './reinstall.sh && python -m unittest' } From b8e3e86f37a09cd44f4c55bfc56ec6c61163a6ff Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 10 Feb 2020 17:31:10 -0800 Subject: [PATCH 18/30] Fix of documentation after package reorganization Signed-off-by: Tomasz Kornuta --- .../sources/source/collections/nemo_asr.rst | 6 ++-- .../sources/source/collections/nemo_nlp.rst | 28 +++++++++---------- .../sources/source/collections/nemo_tts.rst | 8 +++--- docs/docs_zh/sources/source/conf.py | 9 +++--- docs/sources/source/collections/nemo_asr.rst | 4 +-- docs/sources/source/collections/nemo_nlp.rst | 28 +++++++++---------- docs/sources/source/collections/nemo_tts.rst | 8 +++--- docs/sources/source/conf.py | 8 ++---- examples/image/resnet50.py | 14 ++++------ nemo/collections/simple_gan/gan.py | 4 ++- 10 files changed, 56 insertions(+), 61 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_asr.rst b/docs/docs_zh/sources/source/collections/nemo_asr.rst index 19135143a520..90c921d68442 100644 --- a/docs/docs_zh/sources/source/collections/nemo_asr.rst +++ b/docs/docs_zh/sources/source/collections/nemo_asr.rst @@ -1,9 +1,9 @@ -NeMo_ASR collection +nemo.collections.ASR collection =================== 语音数据处理模块 ------------------------------ -.. automodule:: nemo_asr.data_layer +.. automodule:: nemo.collections.asr.data_layer :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ NeMo_ASR collection 语音识别模块 ------------------------------------ -.. automodule:: nemo_asr.jasper +.. automodule:: nemo.collections.asr.jasper :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index ccd744efd4a4..4ae22e89e2c2 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -1,9 +1,9 @@ -NeMo_NLP collection +nemo.collections.NLP collection =================== NLP 数据处理模块 --------------------------- -.. automodule:: nemo_nlp.data_layers +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: @@ -12,37 +12,37 @@ NLP 数据处理模块 NLP 分词器 -------------- -.. automodule:: nemo_nlp.data.tokenizers.bert_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.bert_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.char_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.char_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.gpt2_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.gpt2_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.tokenizer_spec +.. automodule:: nemo.collections.nlp.data.tokenizers.tokenizer_spec :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.word_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.word_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,27 +50,27 @@ NLP 分词器 NLP 神经模块 ------------------ -.. automodule:: nemo_nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.modules.classifiers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.losses +.. automodule:: nemo.collections.nlp.modules.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.modules.pytorch_utils :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.modules.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP 神经模块 NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo_nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.huggingface.bert :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_tts.rst b/docs/docs_zh/sources/source/collections/nemo_tts.rst index 8b189efd5224..c0a59399e335 100644 --- a/docs/docs_zh/sources/source/collections/nemo_tts.rst +++ b/docs/docs_zh/sources/source/collections/nemo_tts.rst @@ -1,9 +1,9 @@ -NeMo_TTS collection +nemo.collections.TTS collection =================== 语音数据处理模块 ------------------------------ -.. automodule:: nemo_tts.data_layers +.. automodule:: nemo.collections.tts.data_layers :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ NeMo_TTS collection Tacotron 2 模块 ------------------ -.. automodule:: nemo_tts.tacotron2_modules +.. automodule:: nemo.collections.tts.tacotron2_modules :members: :undoc-members: :show-inheritance: @@ -19,7 +19,7 @@ Tacotron 2 模块 Waveglow 模块 ------------------ -.. automodule:: nemo_tts.waveglow_modules +.. automodule:: nemo.collections.tts.waveglow_modules :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py index faa8502da823..5561d9825fdd 100644 --- a/docs/docs_zh/sources/source/conf.py +++ b/docs/docs_zh/sources/source/conf.py @@ -25,10 +25,10 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -sys.path.insert(0, os.path.abspath("../../../collections")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) +# sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) +# sys.path.insert(0, os.path.abspath("../../../collections")) +# sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) +# sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) # sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) # ---- Mocking up the classes. ----- @@ -63,6 +63,7 @@ def __getattr__(cls, name): 'h5py', 'kaldi_io', 'transformers', + 'transformers.tokenization_bert', ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/docs/sources/source/collections/nemo_asr.rst b/docs/sources/source/collections/nemo_asr.rst index ae1765070e20..3cf745863686 100644 --- a/docs/sources/source/collections/nemo_asr.rst +++ b/docs/sources/source/collections/nemo_asr.rst @@ -3,7 +3,7 @@ NeMo_ASR collection Speech data processing modules ------------------------------ -.. automodule:: nemo_asr.data_layer +.. automodule:: nemo.collections.asr.data_layer :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ Speech data processing modules Automatic Speech Recognition modules ------------------------------------ -.. automodule:: nemo_asr.jasper +.. automodule:: nemo.collections.asr.jasper :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index b0ddc62c4f8e..fcbd5a7346bf 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -1,9 +1,9 @@ -NeMo_NLP collection +nemo.collections.nlp collection =================== NLP data processing modules --------------------------- -.. automodule:: nemo_nlp.data_layers +.. automodule:: nemo.collections.nlp.data_layers :members: :undoc-members: :show-inheritance: @@ -12,37 +12,37 @@ NLP data processing modules NLP Tokenizers -------------- -.. automodule:: nemo_nlp.data.tokenizers.bert_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.bert_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.char_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.char_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.gpt2_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.gpt2_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.tokenizer_spec +.. automodule:: nemo.collections.nlp.data.tokenizers.tokenizer_spec :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.word_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.word_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,27 +50,27 @@ NLP Tokenizers NLP Neural Modules ------------------ -.. automodule:: nemo_nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.modules.classifiers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.losses +.. automodule:: nemo.collections.nlp.modules.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.modules.pytorch_utils :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.modules.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP Neural Modules NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo_nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.huggingface.bert :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_tts.rst b/docs/sources/source/collections/nemo_tts.rst index 958642a2f092..4145429011ee 100644 --- a/docs/sources/source/collections/nemo_tts.rst +++ b/docs/sources/source/collections/nemo_tts.rst @@ -1,9 +1,9 @@ -NeMo_TTS collection +nemo.collections.TTS collection =================== Speech data processing modules ------------------------------ -.. automodule:: nemo_tts.data_layers +.. automodule:: nemo.collections.tts.data_layers :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ Speech data processing modules Tacotron 2 modules ------------------ -.. automodule:: nemo_tts.tacotron2_modules +.. automodule:: nemo.collections.tts.tacotron2_modules :members: :undoc-members: :show-inheritance: @@ -19,7 +19,7 @@ Tacotron 2 modules Waveglow modules ------------------ -.. automodule:: nemo_tts.waveglow_modules +.. automodule:: nemo.collections.tts.waveglow_modules :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/conf.py b/docs/sources/source/conf.py index 77a048fad9f5..5a220a6cfbe1 100644 --- a/docs/sources/source/conf.py +++ b/docs/sources/source/conf.py @@ -25,12 +25,7 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -sys.path.insert(0, os.path.abspath("../../../collections")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_tts")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) +sys.path.insert(0, os.path.abspath("../../../nemo/")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} @@ -66,6 +61,7 @@ def __getattr__(cls, name): 'h5py', 'kaldi_io', 'transformers', + 'transformers.tokenization_bert', ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/examples/image/resnet50.py b/examples/image/resnet50.py index b6ca608f06b5..92c7e2d4b037 100644 --- a/examples/image/resnet50.py +++ b/examples/image/resnet50.py @@ -54,41 +54,37 @@ optimization_level=nemo.core.Optimization.mxprO0, ) -resnet = neural_factory.get_module( - name="resnet50", params={"placement": device}, collection="torchvision", pretrained=False, -) +resnet = neural_factory.get_module(name="resnet50", params={}, collection="torchvision", pretrained=False) dl_train = neural_factory.get_module( name="ImageFolderDataLayer", collection="torchvision", params={ "batch_size": batch_size, - "input_size": resnet.inputs["x"].axis2type[2].dim, + "input_size": resnet.input_ports["x"].axis2type[2].dim, "shuffle": True, "path": args.data_root + "train", # "path": "/mnt/D1/Data/ImageNet/ImageFolder/train", - "placement": device, }, ) -L_train = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={"placement": device}) +L_train = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={}) dl_eval = neural_factory.get_module( name="ImageFolderDataLayer", collection="torchvision", params={ "batch_size": batch_size, - "input_size": resnet.inputs["x"].axis2type[2].dim, + "input_size": resnet.input_ports["x"].axis2type[2].dim, "shuffle": False, "is_eval": True, "path": args.data_root + "val", # "path": "/mnt/D1/Data/ImageNet/ImageFolder/val", # "path": "/raid/okuchaiev/Data/ImageNet/ImageFolder/val", - "placement": device, }, ) -L_eval = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={"placement": device}) +L_eval = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={}) step_per_epoch = int(len(dl_train) / (batch_size * num_gpus)) diff --git a/nemo/collections/simple_gan/gan.py b/nemo/collections/simple_gan/gan.py index 16e83bbdf5c5..cc28d7dad4d9 100644 --- a/nemo/collections/simple_gan/gan.py +++ b/nemo/collections/simple_gan/gan.py @@ -123,8 +123,10 @@ def output_ports(self): ) } - def __init__(self): + def __init__(self, batch_size): super().__init__() + self._batch_size = batch_size + self.layers = torch.nn.Sequential( torch.nn.ConvTranspose2d(64, 128, 3, stride=2), torch.nn.ReLU(), From 09d4678fe57389c964b0f7146c5b74cb09f8c431 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 10 Feb 2020 17:55:16 -0800 Subject: [PATCH 19/30] cleanups and fixes of sphinx errors and warnings Signed-off-by: Tomasz Kornuta --- docs/docs_zh/sources/source/collections/nemo_asr.rst | 2 +- docs/docs_zh/sources/source/collections/nemo_nlp.rst | 8 ++++---- docs/docs_zh/sources/source/collections/nemo_tts.rst | 2 +- docs/docs_zh/sources/source/conf.py | 5 ----- docs/sources/source/collections/nemo_asr.rst | 2 +- docs/sources/source/collections/nemo_nlp.rst | 6 +++--- docs/sources/source/collections/nemo_tts.rst | 2 +- docs/sources/source/conf.py | 1 - 8 files changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_asr.rst b/docs/docs_zh/sources/source/collections/nemo_asr.rst index 90c921d68442..83f92615e563 100644 --- a/docs/docs_zh/sources/source/collections/nemo_asr.rst +++ b/docs/docs_zh/sources/source/collections/nemo_asr.rst @@ -1,4 +1,4 @@ -nemo.collections.ASR collection +NeMo ASR collection =================== 语音数据处理模块 diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index 4ae22e89e2c2..14848f756ebf 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -1,4 +1,4 @@ -nemo.collections.NLP collection +NeMo NLP collection =================== NLP 数据处理模块 @@ -57,7 +57,7 @@ NLP 神经模块 :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.losses +.. automodule:: nemo.collections.nlp.nm.losses :members: :undoc-members: :show-inheritance: @@ -70,7 +70,7 @@ NLP 神经模块 :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP 神经模块 NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo.collections.nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_tts.rst b/docs/docs_zh/sources/source/collections/nemo_tts.rst index c0a59399e335..aa852bba56eb 100644 --- a/docs/docs_zh/sources/source/collections/nemo_tts.rst +++ b/docs/docs_zh/sources/source/collections/nemo_tts.rst @@ -1,4 +1,4 @@ -nemo.collections.TTS collection +NeMo TTS collection =================== 语音数据处理模块 diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py index 5561d9825fdd..b8220000049e 100644 --- a/docs/docs_zh/sources/source/conf.py +++ b/docs/docs_zh/sources/source/conf.py @@ -25,11 +25,6 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -# sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -# sys.path.insert(0, os.path.abspath("../../../collections")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} diff --git a/docs/sources/source/collections/nemo_asr.rst b/docs/sources/source/collections/nemo_asr.rst index 3cf745863686..95d2e53779a2 100644 --- a/docs/sources/source/collections/nemo_asr.rst +++ b/docs/sources/source/collections/nemo_asr.rst @@ -1,4 +1,4 @@ -NeMo_ASR collection +NeMo ASR collection =================== Speech data processing modules diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index fcbd5a7346bf..58ab80ca989a 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -1,4 +1,4 @@ -nemo.collections.nlp collection +NeMo NLP collection =================== NLP data processing modules @@ -70,7 +70,7 @@ NLP Neural Modules :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP Neural Modules NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo.collections.nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_tts.rst b/docs/sources/source/collections/nemo_tts.rst index 4145429011ee..be7a8c7a8d07 100644 --- a/docs/sources/source/collections/nemo_tts.rst +++ b/docs/sources/source/collections/nemo_tts.rst @@ -1,4 +1,4 @@ -nemo.collections.TTS collection +NeMo TTS collection =================== Speech data processing modules diff --git a/docs/sources/source/conf.py b/docs/sources/source/conf.py index 5a220a6cfbe1..8caeaaede9b5 100644 --- a/docs/sources/source/conf.py +++ b/docs/sources/source/conf.py @@ -25,7 +25,6 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} From b7407978c42a768612a58854b7eeb176a5cd4f3c Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 21:18:17 -0800 Subject: [PATCH 20/30] fix Jenkins file Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fb31e238d36e..7d4b964bec39 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,7 +8,9 @@ pipeline { options { timeout(time: 1, unit: 'HOURS') disableConcurrentBuilds() - } + } + stages { + stage('PyTorch version') { steps { sh 'python -c "import torch; print(torch.__version__)"' @@ -131,8 +133,6 @@ pipeline { } } - - stage('NLP-ASR processing') { failFast true parallel { From fd40f4cb64cc1125d33cd3dea2df9b1a19ede587 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 21:28:57 -0800 Subject: [PATCH 21/30] install bc inside the container necessary by some test Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7d4b964bec39..b7b6eb5f6f88 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'pip install -r requirements/requirements_test.txt' + sh 'apt-get install -y bc && pip install -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From f7983a768d5f3d19c0fa595019bacc4dd0f31ec4 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Mon, 10 Feb 2020 21:34:55 -0800 Subject: [PATCH 22/30] bugfix Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b7b6eb5f6f88..b550fb359429 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { } stage('Install test requirements') { steps { - sh 'apt-get install -y bc && pip install -r requirements/requirements_test.txt' + sh 'apt-get update && apt-get install -y bc && pip install -r requirements/requirements_test.txt' } } stage('Code formatting checks') { From 1000ce69b15ea4a4f730fe034621f77e540a3060 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 09:04:22 -0800 Subject: [PATCH 23/30] disable beam search Signed-off-by: Oleksii Kuchaiev --- examples/asr/jasper_an4.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/asr/jasper_an4.py b/examples/asr/jasper_an4.py index 77d81c9c364f..dada712f156d 100644 --- a/examples/asr/jasper_an4.py +++ b/examples/asr/jasper_an4.py @@ -128,7 +128,8 @@ def main(): parser.add_argument("--eval_datasets", type=str, nargs=1, help="validation dataset path") # Create new args - parser.add_argument("--lm", default="./an4-lm.3gram.binary", type=str) + # parser.add_argument("--lm", default="./an4-lm.3gram.binary", type=str) + parser.add_argument("--lm", default=None, type=str) parser.add_argument("--test_after_training", action='store_true') parser.add_argument("--momentum", type=float) parser.add_argument("--beta1", default=0.95, type=float) @@ -198,7 +199,7 @@ def main(): if args.test_after_training: logging.info("Testing greedy and beam search with LM WER.") # Create BeamSearch NM - if nf.world_size > 1: + if nf.world_size > 1 or args.lm is None: logging.warning("Skipping beam search WER as it does not " "work if doing distributed training.") else: beam_search_with_lm = nemo_asr.BeamSearchDecoderWithLM( From b35543cee21353f104d2e3dd8b01780a116a2cdc Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 09:21:06 -0800 Subject: [PATCH 24/30] bugfix Signed-off-by: Oleksii Kuchaiev --- examples/asr/jasper_an4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/asr/jasper_an4.py b/examples/asr/jasper_an4.py index dada712f156d..1ed10e0437e5 100644 --- a/examples/asr/jasper_an4.py +++ b/examples/asr/jasper_an4.py @@ -219,7 +219,7 @@ def main(): raise ValueError(f"Final eval greedy WER {wer * 100:.2f}% > :" f"than {wer_thr * 100:.2f}%") nf.sync_all_processes() - if nf.world_size == 1: + if nf.world_size == 1 and args.lm is not None: beam_hypotheses = [] # Over mini-batch for i in evaluated_tensors[-1]: From fe2ff0450dfb39afc3e4b8768bee64be9e81ccab Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 10:52:19 -0800 Subject: [PATCH 25/30] increase mem limits for docker Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index b550fb359429..36f8d3070353 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home' + args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home --shm-size=8g --ulimit memlock=-1 --ulimit stack=67108864' } } options { From 986626a0c589f80661aedb110366b304ec9f5260 Mon Sep 17 00:00:00 2001 From: Oleksii Kuchaiev Date: Tue, 11 Feb 2020 10:55:46 -0800 Subject: [PATCH 26/30] update container args Signed-off-by: Oleksii Kuchaiev --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 36f8d3070353..b6ec2d321979 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent { docker { image 'nvcr.io/nvidia/pytorch:20.01-py3' - args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home --shm-size=8g --ulimit memlock=-1 --ulimit stack=67108864' + args '--device=/dev/nvidia0 --gpus all --user 0:128 -v /home:/home --shm-size=8g' } } options { From 8bf4389e5112bb2b936387323e2807b217498c90 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 11:07:40 -0800 Subject: [PATCH 27/30] fix sphinx warning Signed-off-by: Yang Zhang --- .../sources/source/collections/nemo_nlp.rst | 38 ++++++++++++++---- docs/sources/source/collections/nemo_nlp.rst | 39 +++++++++++++++---- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index 14848f756ebf..223cd71d3b6d 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -3,11 +3,10 @@ NeMo NLP collection NLP 数据处理模块 --------------------------- -.. automodule:: nemo.collections.nlp.nm.data_layers +.. automodule:: nemo.collections.nlp.data.datasets :members: :undoc-members: :show-inheritance: - :exclude-members: forward NLP 分词器 -------------- @@ -27,7 +26,7 @@ NLP 分词器 :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.sentencepiece_tokenizer :members: :undoc-members: :show-inheritance: @@ -42,7 +41,7 @@ NLP 分词器 :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.youtokentome_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,7 +49,8 @@ NLP 分词器 NLP 神经模块 ------------------ -.. automodule:: nemo.collections.nlp.modules.classifiers + +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: @@ -64,7 +64,19 @@ NLP 神经模块 :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_classification_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_regression_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.token_classification_nm :members: :undoc-members: :show-inheritance: @@ -76,10 +88,22 @@ NLP 神经模块 :show-inheritance: :exclude-members: forward +.. automodule:: nemo.collections.nlp.nm.trainables.dialogue_state_tracking.state_tracking_trade_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index 58ab80ca989a..8c862d84fb66 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -3,11 +3,10 @@ NeMo NLP collection NLP data processing modules --------------------------- -.. automodule:: nemo.collections.nlp.data_layers +.. automodule:: nemo.collections.nlp.data.datasets :members: :undoc-members: :show-inheritance: - :exclude-members: forward NLP Tokenizers -------------- @@ -27,7 +26,7 @@ NLP Tokenizers :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.sentencepiece_tokenizer :members: :undoc-members: :show-inheritance: @@ -42,7 +41,7 @@ NLP Tokenizers :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.youtokentome_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,21 +49,33 @@ NLP Tokenizers NLP Neural Modules ------------------ -.. automodule:: nemo.collections.nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.losses +.. automodule:: nemo.collections.nlp.nm.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_classification_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_regression_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.token_classification_nm :members: :undoc-members: :show-inheritance: @@ -76,10 +87,22 @@ NLP Neural Modules :show-inheritance: :exclude-members: forward +.. automodule:: nemo.collections.nlp.nm.trainables.dialogue_state_tracking.state_tracking_trade_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm :members: :undoc-members: :show-inheritance: From 9b5acd4d4783ac90d022348e899ee61bb5591f15 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 11:26:13 -0800 Subject: [PATCH 28/30] debug Signed-off-by: Yang Zhang --- examples/nlp/language_modeling/bert_pretraining.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 1b6945801813..c861ab287428 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -84,6 +84,8 @@ from nemo.collections.nlp.data.datasets.lm_bert_dataset import BERTPretrainingDataDesc from nemo.utils.lr_policies import get_lr_policy +print("nemo_nlp path", nemo_nlp.__file__) + parser = argparse.ArgumentParser(description='BERT pretraining') parser.add_argument("--local_rank", default=None, type=int) parser.add_argument("--num_gpus", default=1, type=int) From a1ebf56b29b6cf90c4d43c89cd9e458f320230dc Mon Sep 17 00:00:00 2001 From: Ryan Leary Date: Thu, 6 Feb 2020 21:48:59 -0500 Subject: [PATCH 29/30] Fix Dockerfile to include checked out version Signed-off-by: Ryan Leary --- .dockerignore | 17 +++++++++ Dockerfile | 52 +++++++++++++++++++--------- requirements/requirements_docker.txt | 3 +- 3 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..fdc976d48579 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env +pip-log.txt +pip-delete-this-directory.txt +.tox +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +*.log +.git diff --git a/Dockerfile b/Dockerfile index 5d843f8c51cd..d77a8fcd26a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:experimental + # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:19.11-py3 +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:20.01-py3 -FROM ${FROM_IMAGE_NAME} +# build an image that includes only the nemo dependencies, ensures that dependencies +# are included first for optimal caching, and useful for building a development +# image (by specifying build target as `nemo-deps`) +FROM ${BASE_IMAGE} as nemo-deps # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive @@ -25,27 +30,42 @@ RUN apt-get update && \ python-dev && \ rm -rf /var/lib/apt/lists/* +# install onnx trt open source plugins ENV PATH=$PATH:/usr/src/tensorrt/bin WORKDIR /tmp/onnx-trt COPY scripts/docker/onnx-trt.patch . -RUN git clone -n https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && git checkout 8716c9b && git submodule update --init --recursive && patch -f < ../onnx-trt.patch && \ - mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig - -WORKDIR /workspace/nemo -ARG NEMO_GIT_BRANCH="master" +RUN git clone -n https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && \ + git checkout 8716c9b && git submodule update --init --recursive && patch -f < ../onnx-trt.patch && \ + mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && \ + make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig && rm -rf /tmp/onnx-tensorrt +# install nemo dependencies +WORKDIR /tmp/nemo COPY requirements/requirements_docker.txt requirements.txt -RUN pip install \ - --disable-pip-version-check --no-cache-dir\ - --upgrade -r requirements.txt && \ - pip install --no-cache-dir \ - git+git://github.com/NVIDIA/NeMo.git@${NEMO_GIT_BRANCH}#egg=project[all] - -RUN printf "#!/bin/bash\njupyter lab --no-browser --allow-root --ip=0.0.0.0" >> start-jupyter.sh && \ - chmod +x start-jupyter.sh - +RUN pip install --disable-pip-version-check --no-cache-dir -r requirements.txt +# copy nemo source into a scratch image +FROM scratch as nemo-src +COPY . . +# start building the final container +FROM nemo-deps as nemo +ARG NEMO_VERSION +ARG BASE_IMAGE +# Check that NEMO_VERSION is set. Build will fail without this. Expose NEMO and base container +# version information as runtime environment variable for introspection purposes +RUN /usr/bin/test -n "$NEMO_VERSION" && \ + /bin/echo "export NEMO_VERSION=${NEMO_VERSION}" >> /root/.bashrc && \ + /bin/echo "export BASE_IMAGE=${BASE_IMAGE}" >> /root/.bashrc +RUN --mount=from=nemo-src,target=/tmp/nemo cd /tmp/nemo && pip install ".[all]" +# copy scripts/examples/tests into container for end user +WORKDIR /workspace/nemo +COPY scripts /workspace/nemo/scripts +COPY examples /workspace/nemo/examples +COPY tests /workspace/nemo/tests +COPY README.rst LICENSE /workspace/nemo/ +RUN printf "#!/bin/bash\njupyter lab --no-browser --allow-root --ip=0.0.0.0" >> start-jupyter.sh && \ + chmod +x start-jupyter.sh diff --git a/requirements/requirements_docker.txt b/requirements/requirements_docker.txt index a7ac755041c9..82b84f9f6ca3 100644 --- a/requirements/requirements_docker.txt +++ b/requirements/requirements_docker.txt @@ -21,8 +21,7 @@ ruamel.yaml sentencepiece six sox -torch tqdm unidecode wget -youtokentome \ No newline at end of file +youtokentome From 2316a34885130724e80405e85547f5b048128e3b Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 11 Feb 2020 14:55:15 -0800 Subject: [PATCH 30/30] gen and bert_pretraining fix/revert Signed-off-by: Tomasz Kornuta --- examples/image/gan.py | 2 +- examples/nlp/language_modeling/bert_pretraining.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/image/gan.py b/examples/image/gan.py index 61aee8c252f7..08c43899ef21 100644 --- a/examples/image/gan.py +++ b/examples/image/gan.py @@ -44,7 +44,7 @@ batch_size=batch_size, shuffle=True, train=True, root=args.train_dataset ) -generator = nemo_simple_gan.SimpleGenerator() +generator = nemo_simple_gan.SimpleGenerator(batch_size) discriminator = nemo_simple_gan.SimpleDiscriminator() neg_disc_loss = nemo_simple_gan.DiscriminatorLoss(neg=True) disc_loss = nemo_simple_gan.DiscriminatorLoss() diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index c861ab287428..1b6945801813 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -84,8 +84,6 @@ from nemo.collections.nlp.data.datasets.lm_bert_dataset import BERTPretrainingDataDesc from nemo.utils.lr_policies import get_lr_policy -print("nemo_nlp path", nemo_nlp.__file__) - parser = argparse.ArgumentParser(description='BERT pretraining') parser.add_argument("--local_rank", default=None, type=int) parser.add_argument("--num_gpus", default=1, type=int)