-
Notifications
You must be signed in to change notification settings - Fork 50
/
tox.ini
141 lines (131 loc) · 6.3 KB
/
tox.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
; For all tests that run in Github Actions, see
; .github/workflows/cpu-ci.yml for the workflow definition.
[tox]
envlist = gpu,multi-gpu,horovod-cpu,nvtabular-cpu,systems-cpu,transformers4rec-cpu,docs,docs-multi
[testenv]
commands =
pip install --upgrade pip
pip install -e .[all]
[testenv:gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
allowlist_externals =
bash
deps =
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH}
passenv =
OPAL_PREFIX
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
sitepackages=true
commands =
bash -c 'python -m pytest --cov-report term --cov merlin -m "{env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
[testenv:horovod-gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
allowlist_externals =
horovodrun
sh
bash
passenv =
OPAL_PREFIX
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
CPATH={env:CPATH}{:}{envdir}/hugectr/include
LD_LIBRARY_PATH=${envdir}/hugectr/include/lib{:}/usr/local/lib/python3.10/dist-packages/tensorflow{:}{env:LD_LIBRARY_PATH}
LIBRARY_PATH=${envdir}/hugectr/lib{:}{env:LIBRARY_PATH}
sitepackages=true
deps =
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
commands =
sh examples/usecases/multi-gpu/install_sparse_operation_kit.sh {envdir}
bash -c 'horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m "unit and horovod {env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
[testenv:horovod-cpu]
setenv =
HOROVOD_WITH_MPI=1
HOROVOD_WITH_TENSORFLOW=1
PATH={env:PATH}{:}{envdir}/env/bin
LD_LIBRARY_PATH={env:LD_LIBRARY_PATH}{:}{envdir}/env/lib
commands =
conda update --yes --name base --channel defaults conda
conda env create --prefix {envdir}/env --file requirements/horovod-cpu-environment.yml --force
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH:main}
{envdir}/env/bin/python -m pip install 'horovod==0.28.1' --no-cache-dir
{envdir}/env/bin/horovodrun --check-build
{envdir}/env/bin/horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m "unit and horovod {env:PYTEST_MARKERS}" -rxs {posargs:tests}
[testenv:nvtabular-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/NVTabular.git nvtabular-{env:GIT_COMMIT}
python -m pip install --upgrade "./nvtabular-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./nvtabular-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest nvtabular-{env:GIT_COMMIT}/tests/unit
[testenv:systems-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/systems.git systems-{env:GIT_COMMIT}
python -m pip install --upgrade "./systems-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./systems-{env:GIT_COMMIT}/requirements/test-cpu.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest -m "not notebook" systems-{env:GIT_COMMIT}/tests/unit
[testenv:transformers4rec-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/Transformers4Rec.git Transformers4Rec-{env:GIT_COMMIT}
python -m pip install --upgrade -r "./Transformers4Rec-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade "./Transformers4Rec-{env:GIT_COMMIT}"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest Transformers4Rec-{env:GIT_COMMIT}/tests/unit
[testenv:docs]
; Runs in: Github Actions
; Generates documentation with sphinx. There are other steps in the Github Actions workflow
; to publish the documentation on release.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/dev.txt
git+https://github.com/NVIDIA-Merlin/core.git
git+https://github.com/NVIDIA-Merlin/dataloader.git
git+https://github.com/NVIDIA-Merlin/NVTabular.git
commands =
python -m sphinx.cmd.build -E -P -b html docs/source docs/build/html
[testenv:docs-multi]
; Run the multi-version build that is shown on GitHub Pages.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git
git+https://github.com/NVIDIA-Merlin/dataloader.git
git+https://github.com/NVIDIA-Merlin/NVTabular.git
commands =
sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
sphinx-multiversion docs/source docs/build/html