Skip to content

Commit

Permalink
fixing examples (#6600)
Browse files Browse the repository at this point in the history
* try Azure

* -e

* path

(cherry picked from commit cb59039)
  • Loading branch information
Borda authored and lexierule committed Mar 24, 2021
1 parent 9d31aff commit 4e133ab
Show file tree
Hide file tree
Showing 9 changed files with 29 additions and 26 deletions.
10 changes: 5 additions & 5 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@ jobs:
python -m pytest benchmarks -v --maxfail=2 --durations=0
displayName: 'Testing: benchmarks'
- bash: |
- script: |
set -e
python -m pytest pl_examples -v --maxfail=2 --durations=0
python setup.py install --user --quiet
bash pl_examples/run_ddp-example.sh
cd pl_examples/basic_examples
bash submit_ddp_job.sh
bash submit_ddp2_job.sh
pip uninstall -y pytorch-lightning
# cd pl_examples/basic_examples
# bash submit_ddp_job.sh
# bash submit_ddp2_job.sh
displayName: 'Examples'
2 changes: 1 addition & 1 deletion pl_examples/basic_examples/submit_ddp2_job.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,4 @@ source activate $1
# -------------------------

# run script from above
srun python3 image_classifier.py --accelerator 'ddp2' --gpus 2 --num_nodes 2
srun python3 simple_image_classifier.py --accelerator 'ddp2' --gpus 2 --num_nodes 2 --max_epochs 5
2 changes: 1 addition & 1 deletion pl_examples/basic_examples/submit_ddp_job.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,4 @@ source activate $1
# -------------------------

# run script from above
srun python3 image_classifier.py --accelerator 'ddp' --gpus 2 --num_nodes 2
srun python3 simple_image_classifier.py --accelerator 'ddp' --gpus 2 --num_nodes 2 --max_epochs 5
4 changes: 2 additions & 2 deletions tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
_TEST_ROOT = os.path.dirname(__file__)
_PROJECT_ROOT = os.path.dirname(_TEST_ROOT)
_TEMP_PATH = os.path.join(_PROJECT_ROOT, 'test_temp')
DATASETS_PATH = os.path.join(_PROJECT_ROOT, 'Datasets')
LEGACY_PATH = os.path.join(_PROJECT_ROOT, 'legacy')
PATH_DATASETS = os.path.join(_PROJECT_ROOT, 'Datasets')
PATH_LEGACY = os.path.join(_PROJECT_ROOT, 'legacy')

# todo: this setting `PYTHONPATH` may not be used by other evns like Conda for import packages
if _PROJECT_ROOT not in os.getenv('PYTHONPATH', ""):
Expand Down
3 changes: 2 additions & 1 deletion tests/base/model_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import torch.nn.functional as F

from pytorch_lightning.core.lightning import LightningModule
from tests import PATH_DATASETS
from tests.base.model_optimizers import ConfigureOptimizersPool
from tests.base.model_test_dataloaders import TestDataloaderVariations
from tests.base.model_test_epoch_ends import TestEpochEndVariations
Expand All @@ -28,7 +29,7 @@
from tests.base.model_valid_dataloaders import ValDataloaderVariations
from tests.base.model_valid_epoch_ends import ValidationEpochEndVariations
from tests.base.model_valid_steps import ValidationStepVariations
from tests.helpers.datasets import PATH_DATASETS, TrialMNIST
from tests.helpers.datasets import TrialMNIST


class EvalModelTemplate(
Expand Down
4 changes: 2 additions & 2 deletions tests/checkpointing/test_legacy_checkpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import pytest

from pytorch_lightning import Trainer
from tests import LEGACY_PATH
from tests import PATH_LEGACY

LEGACY_CHECKPOINTS_PATH = os.path.join(LEGACY_PATH, 'checkpoints')
LEGACY_CHECKPOINTS_PATH = os.path.join(PATH_LEGACY, 'checkpoints')
CHECKPOINT_EXTENSION = ".ckpt"


Expand Down
4 changes: 3 additions & 1 deletion tests/helpers/advanced_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from torch.utils.data import DataLoader

from pytorch_lightning.core.lightning import LightningModule
from tests import PATH_DATASETS
from tests.helpers.datasets import AverageDataset, MNIST, TrialMNIST


Expand Down Expand Up @@ -165,7 +166,7 @@ def configure_optimizers(self):
return [opt_g, opt_d], []

def train_dataloader(self):
return DataLoader(TrialMNIST(train=True, download=True), batch_size=16)
return DataLoader(TrialMNIST(root=PATH_DATASETS, train=True, download=True), batch_size=16)


class ParityModuleRNN(LightningModule):
Expand Down Expand Up @@ -223,6 +224,7 @@ def configure_optimizers(self):

def train_dataloader(self):
return DataLoader(MNIST(
root=PATH_DATASETS,
train=True,
download=True,
), batch_size=128, num_workers=1)
15 changes: 5 additions & 10 deletions tests/helpers/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,6 @@
from torch import Tensor
from torch.utils.data import Dataset

from tests import _PROJECT_ROOT

#: local path to test datasets
PATH_DATASETS = os.path.join(_PROJECT_ROOT, 'Datasets')


class MNIST(Dataset):
"""
Expand All @@ -47,7 +42,7 @@ class MNIST(Dataset):
downloaded again.
Examples:
>>> dataset = MNIST(download=True)
>>> dataset = MNIST(".", download=True)
>>> len(dataset)
60000
>>> torch.bincount(dataset.targets)
Expand All @@ -65,7 +60,7 @@ class MNIST(Dataset):

def __init__(
self,
root: str = PATH_DATASETS,
root: str,
train: bool = True,
normalize: tuple = (0.1307, 0.3081),
download: bool = True,
Expand Down Expand Up @@ -152,7 +147,7 @@ class TrialMNIST(MNIST):
kwargs: Same as MNIST
Examples:
>>> dataset = TrialMNIST(download=True)
>>> dataset = TrialMNIST(".", download=True)
>>> len(dataset)
300
>>> sorted(set([d.item() for d in dataset.targets]))
Expand All @@ -161,15 +156,15 @@ class TrialMNIST(MNIST):
tensor([100, 100, 100])
"""

def __init__(self, num_samples: int = 100, digits: Optional[Sequence] = (0, 1, 2), **kwargs):
def __init__(self, root: str, num_samples: int = 100, digits: Optional[Sequence] = (0, 1, 2), **kwargs):
# number of examples per class
self.num_samples = num_samples
# take just a subset of MNIST dataset
self.digits = sorted(digits) if digits else list(range(10))

self.cache_folder_name = f"digits-{'-'.join(str(d) for d in self.digits)}_nb-{self.num_samples}"

super().__init__(normalize=(0.5, 1.0), **kwargs)
super().__init__(root, normalize=(0.5, 1.0), **kwargs)

@staticmethod
def _prepare_subset(full_data: torch.Tensor, full_targets: torch.Tensor, num_samples: int, digits: Sequence):
Expand Down
11 changes: 8 additions & 3 deletions tests/helpers/test_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,17 @@
import cloudpickle
import pytest

from tests import PATH_DATASETS
from tests.helpers.datasets import AverageDataset, MNIST, TrialMNIST


@pytest.mark.parametrize('dataset_cls', [MNIST, TrialMNIST, AverageDataset])
def test_pickling_dataset_mnist(tmpdir, dataset_cls):
mnist = dataset_cls()
@pytest.mark.parametrize('dataset_cls,args', [
(MNIST, dict(root=PATH_DATASETS)),
(TrialMNIST, dict(root=PATH_DATASETS)),
(AverageDataset, dict()),
])
def test_pickling_dataset_mnist(tmpdir, dataset_cls, args):
mnist = dataset_cls(**args)

mnist_pickled = pickle.dumps(mnist)
pickle.loads(mnist_pickled)
Expand Down

0 comments on commit 4e133ab

Please sign in to comment.