From b8e3e86f37a09cd44f4c55bfc56ec6c61163a6ff Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 10 Feb 2020 17:31:10 -0800 Subject: [PATCH 1/5] Fix of documentation after package reorganization Signed-off-by: Tomasz Kornuta --- .../sources/source/collections/nemo_asr.rst | 6 ++-- .../sources/source/collections/nemo_nlp.rst | 28 +++++++++---------- .../sources/source/collections/nemo_tts.rst | 8 +++--- docs/docs_zh/sources/source/conf.py | 9 +++--- docs/sources/source/collections/nemo_asr.rst | 4 +-- docs/sources/source/collections/nemo_nlp.rst | 28 +++++++++---------- docs/sources/source/collections/nemo_tts.rst | 8 +++--- docs/sources/source/conf.py | 8 ++---- examples/image/resnet50.py | 14 ++++------ nemo/collections/simple_gan/gan.py | 4 ++- 10 files changed, 56 insertions(+), 61 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_asr.rst b/docs/docs_zh/sources/source/collections/nemo_asr.rst index 19135143a520..90c921d68442 100644 --- a/docs/docs_zh/sources/source/collections/nemo_asr.rst +++ b/docs/docs_zh/sources/source/collections/nemo_asr.rst @@ -1,9 +1,9 @@ -NeMo_ASR collection +nemo.collections.ASR collection =================== 语音数据处理模块 ------------------------------ -.. automodule:: nemo_asr.data_layer +.. automodule:: nemo.collections.asr.data_layer :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ NeMo_ASR collection 语音识别模块 ------------------------------------ -.. automodule:: nemo_asr.jasper +.. automodule:: nemo.collections.asr.jasper :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index ccd744efd4a4..4ae22e89e2c2 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -1,9 +1,9 @@ -NeMo_NLP collection +nemo.collections.NLP collection =================== NLP 数据处理模块 --------------------------- -.. automodule:: nemo_nlp.data_layers +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: @@ -12,37 +12,37 @@ NLP 数据处理模块 NLP 分词器 -------------- -.. automodule:: nemo_nlp.data.tokenizers.bert_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.bert_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.char_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.char_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.gpt2_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.gpt2_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.tokenizer_spec +.. automodule:: nemo.collections.nlp.data.tokenizers.tokenizer_spec :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.word_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.word_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,27 +50,27 @@ NLP 分词器 NLP 神经模块 ------------------ -.. automodule:: nemo_nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.modules.classifiers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.losses +.. automodule:: nemo.collections.nlp.modules.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.modules.pytorch_utils :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.modules.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP 神经模块 NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo_nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.huggingface.bert :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_tts.rst b/docs/docs_zh/sources/source/collections/nemo_tts.rst index 8b189efd5224..c0a59399e335 100644 --- a/docs/docs_zh/sources/source/collections/nemo_tts.rst +++ b/docs/docs_zh/sources/source/collections/nemo_tts.rst @@ -1,9 +1,9 @@ -NeMo_TTS collection +nemo.collections.TTS collection =================== 语音数据处理模块 ------------------------------ -.. automodule:: nemo_tts.data_layers +.. automodule:: nemo.collections.tts.data_layers :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ NeMo_TTS collection Tacotron 2 模块 ------------------ -.. automodule:: nemo_tts.tacotron2_modules +.. automodule:: nemo.collections.tts.tacotron2_modules :members: :undoc-members: :show-inheritance: @@ -19,7 +19,7 @@ Tacotron 2 模块 Waveglow 模块 ------------------ -.. automodule:: nemo_tts.waveglow_modules +.. automodule:: nemo.collections.tts.waveglow_modules :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py index faa8502da823..5561d9825fdd 100644 --- a/docs/docs_zh/sources/source/conf.py +++ b/docs/docs_zh/sources/source/conf.py @@ -25,10 +25,10 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -sys.path.insert(0, os.path.abspath("../../../collections")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) +# sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) +# sys.path.insert(0, os.path.abspath("../../../collections")) +# sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) +# sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) # sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) # ---- Mocking up the classes. ----- @@ -63,6 +63,7 @@ def __getattr__(cls, name): 'h5py', 'kaldi_io', 'transformers', + 'transformers.tokenization_bert', ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/docs/sources/source/collections/nemo_asr.rst b/docs/sources/source/collections/nemo_asr.rst index ae1765070e20..3cf745863686 100644 --- a/docs/sources/source/collections/nemo_asr.rst +++ b/docs/sources/source/collections/nemo_asr.rst @@ -3,7 +3,7 @@ NeMo_ASR collection Speech data processing modules ------------------------------ -.. automodule:: nemo_asr.data_layer +.. automodule:: nemo.collections.asr.data_layer :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ Speech data processing modules Automatic Speech Recognition modules ------------------------------------ -.. automodule:: nemo_asr.jasper +.. automodule:: nemo.collections.asr.jasper :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index b0ddc62c4f8e..fcbd5a7346bf 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -1,9 +1,9 @@ -NeMo_NLP collection +nemo.collections.nlp collection =================== NLP data processing modules --------------------------- -.. automodule:: nemo_nlp.data_layers +.. automodule:: nemo.collections.nlp.data_layers :members: :undoc-members: :show-inheritance: @@ -12,37 +12,37 @@ NLP data processing modules NLP Tokenizers -------------- -.. automodule:: nemo_nlp.data.tokenizers.bert_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.bert_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.char_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.char_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.gpt2_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.gpt2_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.tokenizer_spec +.. automodule:: nemo.collections.nlp.data.tokenizers.tokenizer_spec :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.word_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.word_tokenizer :members: :undoc-members: :show-inheritance: -.. automodule:: nemo_nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,27 +50,27 @@ NLP Tokenizers NLP Neural Modules ------------------ -.. automodule:: nemo_nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.modules.classifiers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.losses +.. automodule:: nemo.collections.nlp.modules.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.modules.pytorch_utils :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo_nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.modules.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP Neural Modules NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo_nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.huggingface.bert :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_tts.rst b/docs/sources/source/collections/nemo_tts.rst index 958642a2f092..4145429011ee 100644 --- a/docs/sources/source/collections/nemo_tts.rst +++ b/docs/sources/source/collections/nemo_tts.rst @@ -1,9 +1,9 @@ -NeMo_TTS collection +nemo.collections.TTS collection =================== Speech data processing modules ------------------------------ -.. automodule:: nemo_tts.data_layers +.. automodule:: nemo.collections.tts.data_layers :members: :undoc-members: :show-inheritance: @@ -11,7 +11,7 @@ Speech data processing modules Tacotron 2 modules ------------------ -.. automodule:: nemo_tts.tacotron2_modules +.. automodule:: nemo.collections.tts.tacotron2_modules :members: :undoc-members: :show-inheritance: @@ -19,7 +19,7 @@ Tacotron 2 modules Waveglow modules ------------------ -.. automodule:: nemo_tts.waveglow_modules +.. automodule:: nemo.collections.tts.waveglow_modules :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/conf.py b/docs/sources/source/conf.py index 77a048fad9f5..5a220a6cfbe1 100644 --- a/docs/sources/source/conf.py +++ b/docs/sources/source/conf.py @@ -25,12 +25,7 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -sys.path.insert(0, os.path.abspath("../../../collections")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) -sys.path.insert(0, os.path.abspath("../../../collections/nemo_tts")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) +sys.path.insert(0, os.path.abspath("../../../nemo/")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} @@ -66,6 +61,7 @@ def __getattr__(cls, name): 'h5py', 'kaldi_io', 'transformers', + 'transformers.tokenization_bert', ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/examples/image/resnet50.py b/examples/image/resnet50.py index b6ca608f06b5..92c7e2d4b037 100644 --- a/examples/image/resnet50.py +++ b/examples/image/resnet50.py @@ -54,41 +54,37 @@ optimization_level=nemo.core.Optimization.mxprO0, ) -resnet = neural_factory.get_module( - name="resnet50", params={"placement": device}, collection="torchvision", pretrained=False, -) +resnet = neural_factory.get_module(name="resnet50", params={}, collection="torchvision", pretrained=False) dl_train = neural_factory.get_module( name="ImageFolderDataLayer", collection="torchvision", params={ "batch_size": batch_size, - "input_size": resnet.inputs["x"].axis2type[2].dim, + "input_size": resnet.input_ports["x"].axis2type[2].dim, "shuffle": True, "path": args.data_root + "train", # "path": "/mnt/D1/Data/ImageNet/ImageFolder/train", - "placement": device, }, ) -L_train = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={"placement": device}) +L_train = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={}) dl_eval = neural_factory.get_module( name="ImageFolderDataLayer", collection="torchvision", params={ "batch_size": batch_size, - "input_size": resnet.inputs["x"].axis2type[2].dim, + "input_size": resnet.input_ports["x"].axis2type[2].dim, "shuffle": False, "is_eval": True, "path": args.data_root + "val", # "path": "/mnt/D1/Data/ImageNet/ImageFolder/val", # "path": "/raid/okuchaiev/Data/ImageNet/ImageFolder/val", - "placement": device, }, ) -L_eval = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={"placement": device}) +L_eval = neural_factory.get_module(name="CrossEntropyLoss", collection="toys", params={}) step_per_epoch = int(len(dl_train) / (batch_size * num_gpus)) diff --git a/nemo/collections/simple_gan/gan.py b/nemo/collections/simple_gan/gan.py index 16e83bbdf5c5..cc28d7dad4d9 100644 --- a/nemo/collections/simple_gan/gan.py +++ b/nemo/collections/simple_gan/gan.py @@ -123,8 +123,10 @@ def output_ports(self): ) } - def __init__(self): + def __init__(self, batch_size): super().__init__() + self._batch_size = batch_size + self.layers = torch.nn.Sequential( torch.nn.ConvTranspose2d(64, 128, 3, stride=2), torch.nn.ReLU(), From 09d4678fe57389c964b0f7146c5b74cb09f8c431 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 10 Feb 2020 17:55:16 -0800 Subject: [PATCH 2/5] cleanups and fixes of sphinx errors and warnings Signed-off-by: Tomasz Kornuta --- docs/docs_zh/sources/source/collections/nemo_asr.rst | 2 +- docs/docs_zh/sources/source/collections/nemo_nlp.rst | 8 ++++---- docs/docs_zh/sources/source/collections/nemo_tts.rst | 2 +- docs/docs_zh/sources/source/conf.py | 5 ----- docs/sources/source/collections/nemo_asr.rst | 2 +- docs/sources/source/collections/nemo_nlp.rst | 6 +++--- docs/sources/source/collections/nemo_tts.rst | 2 +- docs/sources/source/conf.py | 1 - 8 files changed, 11 insertions(+), 17 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_asr.rst b/docs/docs_zh/sources/source/collections/nemo_asr.rst index 90c921d68442..83f92615e563 100644 --- a/docs/docs_zh/sources/source/collections/nemo_asr.rst +++ b/docs/docs_zh/sources/source/collections/nemo_asr.rst @@ -1,4 +1,4 @@ -nemo.collections.ASR collection +NeMo ASR collection =================== 语音数据处理模块 diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index 4ae22e89e2c2..14848f756ebf 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -1,4 +1,4 @@ -nemo.collections.NLP collection +NeMo NLP collection =================== NLP 数据处理模块 @@ -57,7 +57,7 @@ NLP 神经模块 :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.losses +.. automodule:: nemo.collections.nlp.nm.losses :members: :undoc-members: :show-inheritance: @@ -70,7 +70,7 @@ NLP 神经模块 :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP 神经模块 NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo.collections.nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface :members: :undoc-members: :show-inheritance: diff --git a/docs/docs_zh/sources/source/collections/nemo_tts.rst b/docs/docs_zh/sources/source/collections/nemo_tts.rst index c0a59399e335..aa852bba56eb 100644 --- a/docs/docs_zh/sources/source/collections/nemo_tts.rst +++ b/docs/docs_zh/sources/source/collections/nemo_tts.rst @@ -1,4 +1,4 @@ -nemo.collections.TTS collection +NeMo TTS collection =================== 语音数据处理模块 diff --git a/docs/docs_zh/sources/source/conf.py b/docs/docs_zh/sources/source/conf.py index 5561d9825fdd..b8220000049e 100644 --- a/docs/docs_zh/sources/source/conf.py +++ b/docs/docs_zh/sources/source/conf.py @@ -25,11 +25,6 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -# sys.path.insert(0, os.path.abspath("../../../nemo/nemo")) -# sys.path.insert(0, os.path.abspath("../../../collections")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_asr")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_nlp")) -# sys.path.insert(0, os.path.abspath("../../../collections/nemo_lpr")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} diff --git a/docs/sources/source/collections/nemo_asr.rst b/docs/sources/source/collections/nemo_asr.rst index 3cf745863686..95d2e53779a2 100644 --- a/docs/sources/source/collections/nemo_asr.rst +++ b/docs/sources/source/collections/nemo_asr.rst @@ -1,4 +1,4 @@ -NeMo_ASR collection +NeMo ASR collection =================== Speech data processing modules diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index fcbd5a7346bf..58ab80ca989a 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -1,4 +1,4 @@ -nemo.collections.nlp collection +NeMo NLP collection =================== NLP data processing modules @@ -70,7 +70,7 @@ NLP Neural Modules :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.transformer_nm +.. automodule:: nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm :members: :undoc-members: :show-inheritance: @@ -79,7 +79,7 @@ NLP Neural Modules NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo.collections.nlp.huggingface.bert +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_tts.rst b/docs/sources/source/collections/nemo_tts.rst index 4145429011ee..be7a8c7a8d07 100644 --- a/docs/sources/source/collections/nemo_tts.rst +++ b/docs/sources/source/collections/nemo_tts.rst @@ -1,4 +1,4 @@ -nemo.collections.TTS collection +NeMo TTS collection =================== Speech data processing modules diff --git a/docs/sources/source/conf.py b/docs/sources/source/conf.py index 5a220a6cfbe1..8caeaaede9b5 100644 --- a/docs/sources/source/conf.py +++ b/docs/sources/source/conf.py @@ -25,7 +25,6 @@ sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../../")) -sys.path.insert(0, os.path.abspath("../../../nemo/")) # ---- Mocking up the classes. ----- MOCK_CLASSES = {'Dataset': 'torch.utils.data', 'Module': 'torch.nn'} From 8bf4389e5112bb2b936387323e2807b217498c90 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 11:07:40 -0800 Subject: [PATCH 3/5] fix sphinx warning Signed-off-by: Yang Zhang --- .../sources/source/collections/nemo_nlp.rst | 38 ++++++++++++++---- docs/sources/source/collections/nemo_nlp.rst | 39 +++++++++++++++---- 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/docs/docs_zh/sources/source/collections/nemo_nlp.rst b/docs/docs_zh/sources/source/collections/nemo_nlp.rst index 14848f756ebf..223cd71d3b6d 100644 --- a/docs/docs_zh/sources/source/collections/nemo_nlp.rst +++ b/docs/docs_zh/sources/source/collections/nemo_nlp.rst @@ -3,11 +3,10 @@ NeMo NLP collection NLP 数据处理模块 --------------------------- -.. automodule:: nemo.collections.nlp.nm.data_layers +.. automodule:: nemo.collections.nlp.data.datasets :members: :undoc-members: :show-inheritance: - :exclude-members: forward NLP 分词器 -------------- @@ -27,7 +26,7 @@ NLP 分词器 :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.sentencepiece_tokenizer :members: :undoc-members: :show-inheritance: @@ -42,7 +41,7 @@ NLP 分词器 :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.youtokentome_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,7 +49,8 @@ NLP 分词器 NLP 神经模块 ------------------ -.. automodule:: nemo.collections.nlp.modules.classifiers + +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: @@ -64,7 +64,19 @@ NLP 神经模块 :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_classification_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_regression_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.token_classification_nm :members: :undoc-members: :show-inheritance: @@ -76,10 +88,22 @@ NLP 神经模块 :show-inheritance: :exclude-members: forward +.. automodule:: nemo.collections.nlp.nm.trainables.dialogue_state_tracking.state_tracking_trade_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + NLP Hugging Face 神经模块 ------------------------------- -.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm :members: :undoc-members: :show-inheritance: diff --git a/docs/sources/source/collections/nemo_nlp.rst b/docs/sources/source/collections/nemo_nlp.rst index 58ab80ca989a..8c862d84fb66 100644 --- a/docs/sources/source/collections/nemo_nlp.rst +++ b/docs/sources/source/collections/nemo_nlp.rst @@ -3,11 +3,10 @@ NeMo NLP collection NLP data processing modules --------------------------- -.. automodule:: nemo.collections.nlp.data_layers +.. automodule:: nemo.collections.nlp.data.datasets :members: :undoc-members: :show-inheritance: - :exclude-members: forward NLP Tokenizers -------------- @@ -27,7 +26,7 @@ NLP Tokenizers :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.spc_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.sentencepiece_tokenizer :members: :undoc-members: :show-inheritance: @@ -42,7 +41,7 @@ NLP Tokenizers :undoc-members: :show-inheritance: -.. automodule:: nemo.collections.nlp.data.tokenizers.yttm_tokenizer +.. automodule:: nemo.collections.nlp.data.tokenizers.youtokentome_tokenizer :members: :undoc-members: :show-inheritance: @@ -50,21 +49,33 @@ NLP Tokenizers NLP Neural Modules ------------------ -.. automodule:: nemo.collections.nlp.modules.classifiers +.. automodule:: nemo.collections.nlp.nm.data_layers :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.losses +.. automodule:: nemo.collections.nlp.nm.losses :members: :undoc-members: :show-inheritance: :exclude-members: forward -.. automodule:: nemo.collections.nlp.modules.pytorch_utils +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_classification_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.sequence_regression_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.common.token_classification_nm :members: :undoc-members: :show-inheritance: @@ -76,10 +87,22 @@ NLP Neural Modules :show-inheritance: :exclude-members: forward +.. automodule:: nemo.collections.nlp.nm.trainables.dialogue_state_tracking.state_tracking_trade_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + +.. automodule:: nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_nm + :members: + :undoc-members: + :show-inheritance: + :exclude-members: forward + NLP Hugging Face Neural Modules ------------------------------- -.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface +.. automodule:: nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm :members: :undoc-members: :show-inheritance: From 9b5acd4d4783ac90d022348e899ee61bb5591f15 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 11 Feb 2020 11:26:13 -0800 Subject: [PATCH 4/5] debug Signed-off-by: Yang Zhang --- examples/nlp/language_modeling/bert_pretraining.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index 1b6945801813..c861ab287428 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -84,6 +84,8 @@ from nemo.collections.nlp.data.datasets.lm_bert_dataset import BERTPretrainingDataDesc from nemo.utils.lr_policies import get_lr_policy +print("nemo_nlp path", nemo_nlp.__file__) + parser = argparse.ArgumentParser(description='BERT pretraining') parser.add_argument("--local_rank", default=None, type=int) parser.add_argument("--num_gpus", default=1, type=int) From 2316a34885130724e80405e85547f5b048128e3b Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Tue, 11 Feb 2020 14:55:15 -0800 Subject: [PATCH 5/5] gen and bert_pretraining fix/revert Signed-off-by: Tomasz Kornuta --- examples/image/gan.py | 2 +- examples/nlp/language_modeling/bert_pretraining.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/image/gan.py b/examples/image/gan.py index 61aee8c252f7..08c43899ef21 100644 --- a/examples/image/gan.py +++ b/examples/image/gan.py @@ -44,7 +44,7 @@ batch_size=batch_size, shuffle=True, train=True, root=args.train_dataset ) -generator = nemo_simple_gan.SimpleGenerator() +generator = nemo_simple_gan.SimpleGenerator(batch_size) discriminator = nemo_simple_gan.SimpleDiscriminator() neg_disc_loss = nemo_simple_gan.DiscriminatorLoss(neg=True) disc_loss = nemo_simple_gan.DiscriminatorLoss() diff --git a/examples/nlp/language_modeling/bert_pretraining.py b/examples/nlp/language_modeling/bert_pretraining.py index c861ab287428..1b6945801813 100644 --- a/examples/nlp/language_modeling/bert_pretraining.py +++ b/examples/nlp/language_modeling/bert_pretraining.py @@ -84,8 +84,6 @@ from nemo.collections.nlp.data.datasets.lm_bert_dataset import BERTPretrainingDataDesc from nemo.utils.lr_policies import get_lr_policy -print("nemo_nlp path", nemo_nlp.__file__) - parser = argparse.ArgumentParser(description='BERT pretraining') parser.add_argument("--local_rank", default=None, type=int) parser.add_argument("--num_gpus", default=1, type=int)