From cf2375bed85606e218f9b51dfc6837f41ca47d6d Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Wed, 1 Apr 2020 16:29:59 +0200 Subject: [PATCH 01/47] Create metrics package --- pytorch_lightning/metrics/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 pytorch_lightning/metrics/__init__.py diff --git a/pytorch_lightning/metrics/__init__.py b/pytorch_lightning/metrics/__init__.py new file mode 100644 index 0000000000000..18522e0dda94b --- /dev/null +++ b/pytorch_lightning/metrics/__init__.py @@ -0,0 +1,5 @@ +""" +Metrics +======= +TODO +""" From a55525956009966a1645117a80959afc6e13694f Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Wed, 1 Apr 2020 16:30:32 +0200 Subject: [PATCH 02/47] Create metric.py --- pytorch_lightning/metrics/metric.py | 248 ++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 pytorch_lightning/metrics/metric.py diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py new file mode 100644 index 0000000000000..fb39d7b3b4a1a --- /dev/null +++ b/pytorch_lightning/metrics/metric.py @@ -0,0 +1,248 @@ +from abc import ABC, abstractmethod +from typing import Any, Optional + +import torch +import torch.distributed + +from pytorch_lightning.metrics.utils import tensor_metric, numpy_metric +from pytorch_lightning.utilities.apply_to_collection import apply_to_collection + +__all__ = ['AbstractMetric', 'TensorMetric', 'NumpyMetric'] + + +class AbstractMetric(torch.nn.Module, ABC): + def __init__(self, name: str): + """ + Abstract Base Class for metric implementation. + Should be used to implement metrics that + 1.) Return multiple Outputs + 2.) Handle their own DDP sync + + Args: + name: the metric's name + + """ + super().__init__() + self.name = name + self._dtype = torch.get_default_dtype() + self._device = torch.device('cpu') + + @property + def dtype(self): + return self._dtype + + @dtype.setter + def dtype(self, new_dtype: Any): + # Necessary to avoid infinite recursion + raise RuntimeError('Cannot set the dtype explicitly. Please use metric.to(new_dtype).') + + @property + def device(self): + return self._device + + @device.setter + def device(self, new_device): + # Necessary to avoid infinite recursion + raise RuntimeError('Cannot set the device explicitly. Please use metric.to(new_device).') + + @abstractmethod + def forward(self, *args, **kwargs) -> torch.Tensor: + """ + Implements the actual metric computation. + + Returns: + metric value + + """ + raise NotImplementedError + + def to(self, *args, **kwargs): + """Moves and/or casts the parameters and buffers. + + This can be called as + + .. function:: to(device=None, dtype=None, non_blocking=False) + + .. function:: to(dtype, non_blocking=False) + + .. function:: to(tensor, non_blocking=False) + + Its signature is similar to :meth:`torch.Tensor.to`, but only accepts + floating point desired :attr:`dtype` s. In addition, this method will + only cast the floating point parameters and buffers to :attr:`dtype` + (if given). The integral parameters and buffers will be moved + :attr:`device`, if that is given, but with dtypes unchanged. When + :attr:`non_blocking` is set, it tries to convert/move asynchronously + with respect to the host if possible, e.g., moving CPU Tensors with + pinned memory to CUDA devices. + + See below for examples. + + .. note:: + This method modifies the module in-place. + + Args: + device: the desired device of the parameters + and buffers in this module + dtype: the desired floating point type of + the floating point parameters and buffers in this module + tensor: Tensor whose dtype and device are the desired + dtype and device for all parameters and buffers in this module + + Returns: + Module: self + + Example:: + + >>> linear = nn.Linear(2, 2) + >>> linear.weight + Parameter containing: + tensor([[ 0.1913, -0.3420], + [-0.5113, -0.2325]]) + >>> linear.to(torch.double) + Linear(in_features=2, out_features=2, bias=True) + >>> linear.weight + Parameter containing: + tensor([[ 0.1913, -0.3420], + [-0.5113, -0.2325]], dtype=torch.float64) + >>> gpu1 = torch.device("cuda:1") + >>> linear.to(gpu1, dtype=torch.half, non_blocking=True) + Linear(in_features=2, out_features=2, bias=True) + >>> linear.weight + Parameter containing: + tensor([[ 0.1914, -0.3420], + [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1') + >>> cpu = torch.device("cpu") + >>> linear.to(cpu) + Linear(in_features=2, out_features=2, bias=True) + >>> linear.weight + Parameter containing: + tensor([[ 0.1914, -0.3420], + [-0.5112, -0.2324]], dtype=torch.float16) + + """ + device, dtype, non_blocking = torch._C._nn._parse_to(*args, **kwargs) + if device is not None: + self._device = device + + if dtype is not None: + self._dtype = dtype + + return super().to(*args, **kwargs) + + def cuda(self, device=None): + """Moves all model parameters and buffers to the GPU. + + This also makes associated parameters and buffers different objects. So + it should be called before constructing optimizer if the module will + live on GPU while being optimized. + + Arguments: + device (int, optional): if specified, all parameters will be + copied to that device + + Returns: + Module: + """ + + self._device = torch.device('cuda', index=device) + return super().cuda(device=device) + + def cpu(self): + """Moves all model parameters and buffers to the CPU. + + Returns: + Module: self + """ + self._device = torch.device('cpu') + return super().cpu() + + def type(self, dst_type): + """Casts all parameters and buffers to :attr:`dst_type`. + + Arguments: + dst_type (type or string): the desired type + + Returns: + Module: self + """ + self._dtype = dst_type + return super().type(dst_type=dst_type) + + def float(self): + """Casts all floating point parameters and buffers to float datatype. + + Returns: + Module: self + """ + self._dtype = torch.float + return super().float() + + def double(self): + """Casts all floating point parameters and buffers to ``double`` datatype. + + Returns: + Module: self + """ + self._dtype = torch.double + return super().double() + + def half(self): + """Casts all floating point parameters and buffers to ``half`` datatype. + + Returns: + Module: self + """ + self._dtype = torch.half + return super().half() + + +class TensorMetric(AbstractMetric): + def __init__(self, name: str, + reduce_group: Optional[Any] = torch.distributed.group.WORLD, + reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): + """ + Base class for metric implementation operating directly on tensors. + All inputs and outputs will be casted to tensors if necessary. + Already handles DDP sync and input/output conversions + + Args: + name: the metric's name + reduce_group: the process group for DDP reduces (only needed for DDP training). + Defaults to all processes (world) + reduce_op: the operation to perform during reduction within DDP (only needed for DDP training). + Defaults to sum. + """ + super().__init__(name) + self._orig_call = tensor_metric(group=reduce_group, + reduce_op=reduce_op)(super().__call__) + + def __call__(self, *args, **kwargs) -> torch.Tensor: + return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, + lambda x: x.to(device=self.device, dtype=self.dtype)) + + +class NumpyMetric(AbstractMetric): + def __init__(self, name: str, + reduce_group: Optional[Any] = torch.distributed.group.WORLD, + reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): + """ + Base class for metric implementation operating on numpy arrays. + All inputs will be casted to numpy if necessary and all outputs will + be casted to tensors if necessary. + Already handles DDP sync and input/output conversions + + Args: + name: the metric's name + reduce_group: the process group for DDP reduces (only needed for DDP training). + Defaults to all processes (world) + reduce_op: the operation to perform during reduction within DDP (only needed for DDP training). + Defaults to sum. + """ + super().__init__(name) + self._orig_call = numpy_metric(group=reduce_group, + reduce_op=reduce_op)(super().__call__) + + def __call__(self, *args, **kwargs) -> torch.Tensor: + return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, + lambda x: x.to(device=self.device, dtype=self.dtype)) From 88b28953ece914f9bc285e2fbd713ef3b6593a53 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Wed, 1 Apr 2020 16:30:56 +0200 Subject: [PATCH 03/47] Create utils.py --- pytorch_lightning/metrics/utils.py | 131 +++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 pytorch_lightning/metrics/utils.py diff --git a/pytorch_lightning/metrics/utils.py b/pytorch_lightning/metrics/utils.py new file mode 100644 index 0000000000000..9942545273546 --- /dev/null +++ b/pytorch_lightning/metrics/utils.py @@ -0,0 +1,131 @@ +import numbers +from typing import Union, Any, Optional + +import numpy as np +import torch +from torch.utils.data._utils.collate import default_convert + +from pytorch_lightning.utilities.apply_to_collection import apply_to_collection + + +def _apply_to_inputs(func_to_apply, *dec_args, **dec_kwargs): + def decorator_fn(func_to_decorate): + def new_func(*args, **kwargs): + args = func_to_apply(args, *dec_args, **dec_kwargs) + kwargs = func_to_apply(kwargs, *dec_args, **dec_kwargs) + return func_to_decorate(*args, **kwargs) + + return new_func + + return decorator_fn + + +def _apply_to_outputs(func_to_apply, *dec_args, **dec_kwargs): + def decorator_fn(function_to_decorate): + def new_func(*args, **kwargs): + result = function_to_decorate(*args, **kwargs) + return func_to_apply(result, *dec_args, **dec_kwargs) + + return new_func + + return decorator_fn + + +def _convert_to_tensor(data: Any) -> Any: + """ + Maps all kind of collections and numbers to tensors + + Args: + data: the data to convert to tensor + + Returns: + the converted data + + """ + if isinstance(data, numbers.Number): + return torch.tensor([data]) + + else: + return default_convert(data) + + +def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> np.ndarray: + """ + converts all tensors and numpy arrays to numpy arrays + Args: + data: the tensor or array to convert to numpy + + Returns: + the resulting numpy array + + """ + if isinstance(data, torch.Tensor): + return data.cpu().detach().numpy() + elif isinstance(data, numbers.Number): + return np.array([data]) + return data + + +def _numpy_metric_conversion(func_to_decorate): + # Applies collection conversion from tensor to numpy to all inputs + # we need to include numpy arrays here, since otherwise they will also be treated as sequences + func_convert_inputs = _apply_to_inputs( + apply_to_collection, (torch.Tensor, np.ndarray, numbers.Number), _convert_to_numpy)(func_to_decorate) + # converts all inputs back to tensors (device doesn't matter here, since this is handled by BaseMetric) + func_convert_in_out = _apply_to_outputs(_convert_to_tensor)(func_convert_inputs) + return func_convert_in_out + + +def _tensor_metric_conversion(func_to_decorate): + # Converts all inputs to tensor if possible + func_convert_inputs = _apply_to_inputs(_convert_to_tensor)(func_to_decorate) + # convert all outputs to tensor if possible + return _apply_to_outputs(_convert_to_tensor)(func_convert_inputs) + + +def _sync_ddp(result: Union[torch.Tensor], + group: Any = torch.distributed.group.WORLD, + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM, + ) -> torch.Tensor: + """ + Function to reduce the tensors from several ddp processes to one master process + + Args: + result: the value to sync and reduce (typically tensor or number) + device: the device to put the synced and reduced value to + dtype: the datatype to convert the synced and reduced value to + group: the process group to gather results from. Defaults to all processes (world) + reduce_op: the reduction operation. Defaults to sum + + Returns: + reduced value + + """ + + if torch.distributed.is_available() and torch.distributed.is_initialized(): + # sync all processes before reduction + torch.distributed.barrier(group=group) + torch.distributed.all_reduce(result, op=reduce_op, group=group, + async_op=False) + + return result + + +def numpy_metric(group: Any = torch.distributed.group.WORLD, + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM): + def decorator_fn(func_to_decorate): + return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, + group=group, + reduce_op=reduce_op)(_numpy_metric_conversion(func_to_decorate)) + + return decorator_fn + + +def tensor_metric(group: Any = torch.distributed.group.WORLD, + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM): + def decorator_fn(func_to_decorate): + return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, + group=group, + reduce_op=reduce_op)(_tensor_metric_conversion(func_to_decorate)) + + return decorator_fn From 118ac0cc5e934798d56d3cc40170d6a60fe933e2 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Wed, 1 Apr 2020 16:32:01 +0200 Subject: [PATCH 04/47] Create __init__.py --- tests/metrics/__init__.py | 205 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 tests/metrics/__init__.py diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py new file mode 100644 index 0000000000000..a6dfcf8be94a4 --- /dev/null +++ b/tests/metrics/__init__.py @@ -0,0 +1,205 @@ +import numpy as np +import pytest +import torch +import torch.distributed as dist + +import tests.base.utils as tutils +from pytorch_lightning.metrics.utils import _apply_to_inputs, _apply_to_outputs, \ + _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ + _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric + + +def test_apply_to_inputs(): + def apply_fn(inputs, factor): + if isinstance(inputs, (float, int)): + return inputs * factor + elif isinstance(inputs, dict): + return {k: apply_fn(v, factor) for k, v in inputs.items()} + elif isinstance(inputs, (tuple, list)): + return [apply_fn(x, factor) for x in inputs] + + @_apply_to_inputs(apply_fn, factor=2.) + def test_fn(*args, **kwargs): + return args, kwargs + + for args in [[], [1., 2.]]: + for kwargs in [{}, {1., 2.}]: + result_args, result_kwargs = test_fn(*args, **kwargs) + assert isinstance(result_args, list) + assert isinstance(result_kwargs, dict) + assert len(result_args) == len(args) + assert len(result_kwargs) == len(kwargs) + assert all([k in result_kwargs for k in kwargs.keys()]) + for arg, result_arg in zip(args, result_args): + assert arg * 2. == result_arg + + for key in kwargs.keys(): + arg = kwargs[key], + result_arg = result_kwargs[key] + assert arg * 2. == result_arg + + +def test_apply_to_outputs(): + def apply_fn(inputs, additional_str): + return str(inputs) + additional_str + + @_apply_to_outputs(apply_fn, additional_str='_str') + def test_fn(*args, **kwargs): + return 'dummy' + + assert test_fn() == 'dummy_str' + + +def test_convert_to_tensor(): + for test_item in [1., np.array([1.])]: + assert isinstance(_convert_to_tensor(test_item), torch.Tensor) + assert test_item.item() == 1. + + +def test_convert_to_numpy(): + for test_item in [1., torch.tensor([1.])]: + result = _convert_to_numpy(test_item) + assert isinstance(result, np.ndarray) + assert result.item() == 1. + + +def test_numpy_metric_conversion(): + @_numpy_metric_conversion + def numpy_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, np.ndarray) + + for v in kwargs.values(): + assert isinstance(v, np.ndarray) + + return 5. + + result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. + + +def test_tensor_metric_conversion(): + @_tensor_metric_conversion + def tensor_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, torch.Tensor) + + for v in kwargs.values(): + assert isinstance(v, torch.Tensor) + + return 5. + + result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_sync_reduce_ddp(): + """Make sure sync-reduce works with DDP""" + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + + tensor = torch.tensor([1.], device='cuda:0') + + reduced_tensor = _sync_ddp(tensor) + + assert reduced_tensor.item() == dist.get_world_size(), \ + 'Sync-Reduce does not work properly with DDP and Tensors' + + number = 1. + reduced_number = _sync_ddp(number) + assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' + assert reduced_number.item() == dist.get_world_size(), \ + 'Sync-Reduce does not work properly with DDP and Numbers' + + dist.destroy_process_group() + + +def test_sync_reduce_simple(): + """Make sure sync-reduce works without DDP""" + tensor = torch.tensor([1.], device='cpu') + + reduced_tensor = _sync_ddp(tensor) + + assert torch.allclose(tensor, + reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' + + number = 1. + + reduced_number = _sync_ddp(number) + assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' + assert reduced_number.item() == number, 'Sync-Reduce does not work properly without DDP and Numbers' + + +def _test_tensor_metric(is_ddp: bool): + @tensor_metric() + def tensor_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, torch.Tensor) + + for v in kwargs.values(): + assert isinstance(v, torch.Tensor) + + return 5. + + if is_ddp: + factor = dist.get_world_size() + else: + factor = 1. + + result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. * factor + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_tensor_metric_ddp(): + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + _test_tensor_metric(True) + dist.destroy_process_group() + + +def test_tensor_metric_simple(): + _test_tensor_metric(False) + + +def _test_numpy_metric(is_ddp: bool): + @numpy_metric() + def numpy_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, np.ndarray) + + for v in kwargs.values(): + assert isinstance(v, np.ndarray) + + return 5. + + if is_ddp: + factor = dist.get_world_size() + else: + factor = 1. + + result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. * factor + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_numpy_metric_ddp(): + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + _test_tensor_metric(True) + dist.destroy_process_group() + + +def test_numpy_metric_simple(): + _test_tensor_metric(False) From fba118c0881f6a52d1ecb7295bd123382ea9ce2b Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Wed, 1 Apr 2020 16:42:29 +0200 Subject: [PATCH 05/47] add tests for metric utils --- tests/metrics/test_utils.py | 205 ++++++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 tests/metrics/test_utils.py diff --git a/tests/metrics/test_utils.py b/tests/metrics/test_utils.py new file mode 100644 index 0000000000000..a6dfcf8be94a4 --- /dev/null +++ b/tests/metrics/test_utils.py @@ -0,0 +1,205 @@ +import numpy as np +import pytest +import torch +import torch.distributed as dist + +import tests.base.utils as tutils +from pytorch_lightning.metrics.utils import _apply_to_inputs, _apply_to_outputs, \ + _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ + _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric + + +def test_apply_to_inputs(): + def apply_fn(inputs, factor): + if isinstance(inputs, (float, int)): + return inputs * factor + elif isinstance(inputs, dict): + return {k: apply_fn(v, factor) for k, v in inputs.items()} + elif isinstance(inputs, (tuple, list)): + return [apply_fn(x, factor) for x in inputs] + + @_apply_to_inputs(apply_fn, factor=2.) + def test_fn(*args, **kwargs): + return args, kwargs + + for args in [[], [1., 2.]]: + for kwargs in [{}, {1., 2.}]: + result_args, result_kwargs = test_fn(*args, **kwargs) + assert isinstance(result_args, list) + assert isinstance(result_kwargs, dict) + assert len(result_args) == len(args) + assert len(result_kwargs) == len(kwargs) + assert all([k in result_kwargs for k in kwargs.keys()]) + for arg, result_arg in zip(args, result_args): + assert arg * 2. == result_arg + + for key in kwargs.keys(): + arg = kwargs[key], + result_arg = result_kwargs[key] + assert arg * 2. == result_arg + + +def test_apply_to_outputs(): + def apply_fn(inputs, additional_str): + return str(inputs) + additional_str + + @_apply_to_outputs(apply_fn, additional_str='_str') + def test_fn(*args, **kwargs): + return 'dummy' + + assert test_fn() == 'dummy_str' + + +def test_convert_to_tensor(): + for test_item in [1., np.array([1.])]: + assert isinstance(_convert_to_tensor(test_item), torch.Tensor) + assert test_item.item() == 1. + + +def test_convert_to_numpy(): + for test_item in [1., torch.tensor([1.])]: + result = _convert_to_numpy(test_item) + assert isinstance(result, np.ndarray) + assert result.item() == 1. + + +def test_numpy_metric_conversion(): + @_numpy_metric_conversion + def numpy_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, np.ndarray) + + for v in kwargs.values(): + assert isinstance(v, np.ndarray) + + return 5. + + result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. + + +def test_tensor_metric_conversion(): + @_tensor_metric_conversion + def tensor_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, torch.Tensor) + + for v in kwargs.values(): + assert isinstance(v, torch.Tensor) + + return 5. + + result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_sync_reduce_ddp(): + """Make sure sync-reduce works with DDP""" + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + + tensor = torch.tensor([1.], device='cuda:0') + + reduced_tensor = _sync_ddp(tensor) + + assert reduced_tensor.item() == dist.get_world_size(), \ + 'Sync-Reduce does not work properly with DDP and Tensors' + + number = 1. + reduced_number = _sync_ddp(number) + assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' + assert reduced_number.item() == dist.get_world_size(), \ + 'Sync-Reduce does not work properly with DDP and Numbers' + + dist.destroy_process_group() + + +def test_sync_reduce_simple(): + """Make sure sync-reduce works without DDP""" + tensor = torch.tensor([1.], device='cpu') + + reduced_tensor = _sync_ddp(tensor) + + assert torch.allclose(tensor, + reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' + + number = 1. + + reduced_number = _sync_ddp(number) + assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' + assert reduced_number.item() == number, 'Sync-Reduce does not work properly without DDP and Numbers' + + +def _test_tensor_metric(is_ddp: bool): + @tensor_metric() + def tensor_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, torch.Tensor) + + for v in kwargs.values(): + assert isinstance(v, torch.Tensor) + + return 5. + + if is_ddp: + factor = dist.get_world_size() + else: + factor = 1. + + result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. * factor + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_tensor_metric_ddp(): + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + _test_tensor_metric(True) + dist.destroy_process_group() + + +def test_tensor_metric_simple(): + _test_tensor_metric(False) + + +def _test_numpy_metric(is_ddp: bool): + @numpy_metric() + def numpy_test_metric(*args, **kwargs): + for arg in args: + assert isinstance(arg, np.ndarray) + + for v in kwargs.values(): + assert isinstance(v, np.ndarray) + + return 5. + + if is_ddp: + factor = dist.get_world_size() + else: + factor = 1. + + result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) + assert isinstance(result, torch.Tensor) + assert result.item() == 5. * factor + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +def test_numpy_metric_ddp(): + tutils.reset_seed() + tutils.set_random_master_port() + + dist.init_process_group('gloo') + _test_tensor_metric(True) + dist.destroy_process_group() + + +def test_numpy_metric_simple(): + _test_tensor_metric(False) From 67ee2416b96d37a87d6b2360a81a14fed07ec87c Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Wed, 1 Apr 2020 16:52:49 +0200 Subject: [PATCH 06/47] add docstrings for metrics utils --- pytorch_lightning/metrics/utils.py | 92 +++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/metrics/utils.py b/pytorch_lightning/metrics/utils.py index 9942545273546..1deced33e8f4f 100644 --- a/pytorch_lightning/metrics/utils.py +++ b/pytorch_lightning/metrics/utils.py @@ -1,5 +1,5 @@ import numbers -from typing import Union, Any, Optional +from typing import Union, Any, Callable import numpy as np import torch @@ -8,8 +8,19 @@ from pytorch_lightning.utilities.apply_to_collection import apply_to_collection -def _apply_to_inputs(func_to_apply, *dec_args, **dec_kwargs): +def _apply_to_inputs(func_to_apply: Callable, *dec_args, **dec_kwargs) -> Callable: + """ + Decorator function to apply a function to all inputs of a function. + Args: + func_to_apply: the function to apply to the inputs + *dec_args: positional arguments for the function to be applied + **dec_kwargs: keyword arguments for the function to be applied + + Returns: + the decorated function + """ def decorator_fn(func_to_decorate): + # actual function applying the give function to inputs def new_func(*args, **kwargs): args = func_to_apply(args, *dec_args, **dec_kwargs) kwargs = func_to_apply(kwargs, *dec_args, **dec_kwargs) @@ -20,8 +31,19 @@ def new_func(*args, **kwargs): return decorator_fn -def _apply_to_outputs(func_to_apply, *dec_args, **dec_kwargs): +def _apply_to_outputs(func_to_apply: Callable, *dec_args, **dec_kwargs) -> Callable: + """ + Decorator function to apply a function to all outputs of a function. + Args: + func_to_apply: the function to apply to the outputs + *dec_args: positional arguments for the function to be applied + **dec_kwargs: keyword arguments for the function to be applied + + Returns: + the decorated function + """ def decorator_fn(function_to_decorate): + # actual function applying the give function to outputs def new_func(*args, **kwargs): result = function_to_decorate(*args, **kwargs) return func_to_apply(result, *dec_args, **dec_kwargs) @@ -66,7 +88,19 @@ def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> return data -def _numpy_metric_conversion(func_to_decorate): +def _numpy_metric_conversion(func_to_decorate: Callable) -> Callable: + """ + Decorator Handling the argument conversion for metrics working on numpy. + All inputs of the decorated function will be converted to numpy and all + outputs will be converted to Tensors + + Args: + func_to_decorate: the function whose inputs and outputs shall be converted + + Returns: + the decorated function + + """ # Applies collection conversion from tensor to numpy to all inputs # we need to include numpy arrays here, since otherwise they will also be treated as sequences func_convert_inputs = _apply_to_inputs( @@ -76,7 +110,18 @@ def _numpy_metric_conversion(func_to_decorate): return func_convert_in_out -def _tensor_metric_conversion(func_to_decorate): +def _tensor_metric_conversion(func_to_decorate: Callable) -> Callable: + """ + Decorator Handling the argument conversion for metrics working on tensors. + All inputs and outputs of the decorated function will be converted to tensors + + Args: + func_to_decorate: the function whose inputs and outputs shall be converted + + Returns: + the decorated function + + """ # Converts all inputs to tensor if possible func_convert_inputs = _apply_to_inputs(_convert_to_tensor)(func_to_decorate) # convert all outputs to tensor if possible @@ -92,8 +137,6 @@ def _sync_ddp(result: Union[torch.Tensor], Args: result: the value to sync and reduce (typically tensor or number) - device: the device to put the synced and reduced value to - dtype: the datatype to convert the synced and reduced value to group: the process group to gather results from. Defaults to all processes (world) reduce_op: the reduction operation. Defaults to sum @@ -112,7 +155,23 @@ def _sync_ddp(result: Union[torch.Tensor], def numpy_metric(group: Any = torch.distributed.group.WORLD, - reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM): + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM) -> Callable: + """ + This decorator shall be used on all function metrics working on numpy arrays. + + It handles the argument conversion and DDP reduction for metrics working on numpy. + All inputs of the decorated function will be converted to numpy and all + outputs will be converted to Tensors. + In DDP Training all output tensors will be reduced according to the given rules. + + Args: + group: the process group to gather results from. Defaults to all processes (world) + reduce_op: the reduction operation. Defaults to sum + + Returns: + the decorated function + + """ def decorator_fn(func_to_decorate): return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, group=group, @@ -122,7 +181,22 @@ def decorator_fn(func_to_decorate): def tensor_metric(group: Any = torch.distributed.group.WORLD, - reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM): + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM) -> Callable: + """ + This decorator shall be used on all function metrics working on tensors. + + It handles the argument conversion and DDP reduction for metrics working on tensors. + All inputs and outputs of the decorated function will be converted to tensors . + In DDP Training all output tensors will be reduced according to the given rules. + + Args: + group: the process group to gather results from. Defaults to all processes (world) + reduce_op: the reduction operation. Defaults to sum + + Returns: + the decorated function + + """ def decorator_fn(func_to_decorate): return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, group=group, From 915913853827e4fd8f4ae6c9b4e325a78c0ba559 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Wed, 1 Apr 2020 16:53:58 +0200 Subject: [PATCH 07/47] add function to recursively apply other function to collection --- .../utilities/apply_to_collection.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 pytorch_lightning/utilities/apply_to_collection.py diff --git a/pytorch_lightning/utilities/apply_to_collection.py b/pytorch_lightning/utilities/apply_to_collection.py new file mode 100644 index 0000000000000..5fdb58510b21d --- /dev/null +++ b/pytorch_lightning/utilities/apply_to_collection.py @@ -0,0 +1,37 @@ +from collections import Mapping, Sequence +from typing import Any, Callable, Union + + +def apply_to_collection(data: Any, dtype: Union[type, tuple], function: Callable, *args, **kwargs) -> Any: + """ + Recursively applies a function to all elements of a certain dtype. + + + Args: + data: the collection to apply the function to + dtype: the given function will be applied to all elements of this dtype + function: the function to apply + *args: positional arguments (will be forwarded to calls of ``function``) + **kwargs: keyword arguments (will be forwarded to calls of ``function``) + + Returns: + the resulting collection + + """ + elem_type = type(data) + + # Breaking condition + if isinstance(data, dtype): + return function(data, *args, **kwargs) + + # Recursively apply to collection items + elif isinstance(data, Mapping): + return elem_type({k: apply_to_collection(v, dtype, function, *args, **kwargs) + for k, v in data.items()}) + elif isinstance(data, tuple) and hasattr(data, '_fields'): # named tuple + return elem_type(*(apply_to_collection(data, dtype, function, *args, **kwargs))) + elif isinstance(data, Sequence) and not isinstance(data, str): + return elem_type([apply_to_collection(d, dtype, function, *args, **kwargs) for d in data]) + + # data is neither of dtype, nor a collection + return data From f8172aab11d98e7767dc3a145d25b2dd27bee89a Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Wed, 1 Apr 2020 16:55:12 +0200 Subject: [PATCH 08/47] add tests for this function --- tests/utilities/__init__.py | 0 tests/utilities/test_apply_to_collection.py | 68 +++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/utilities/__init__.py create mode 100644 tests/utilities/test_apply_to_collection.py diff --git a/tests/utilities/__init__.py b/tests/utilities/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_to_collection.py new file mode 100644 index 0000000000000..1eccf72b74d49 --- /dev/null +++ b/tests/utilities/test_apply_to_collection.py @@ -0,0 +1,68 @@ +import numbers +from collections import namedtuple + +import torch +import numpy as np + +from pytorch_lightning.metrics.metric import _sync_ddp_to_device_type +from pytorch_lightning.utilities.apply_to_collection import apply_to_collection + + +def test_recursive_application_to_collection(): + ntc = namedtuple('Foo', ['bar']) + to_reduce = { + 'a': torch.tensor([1.]), # Tensor + 'b': [torch.tensor([2.])], # list + 'c': (torch.tensor([100.]),), # tuple + 'd': ntc(bar=5.), # named tuple + 'e': np.array([10.]), # numpy array + 'f': 'this_is_a_dummy_str', # string + 'g': 12. # number + } + + expected_result = { + 'a': torch.tensor([1.]), + 'b': [torch.tensor([2.])], + 'c': (torch.tensor([100.]),), + 'd': ntc(bar=torch.tensor([5.])), + 'e': torch.tensor([10.]), + 'f': 'this_is_a_dummy_str', + 'g': torch.tensor([12.]) + + } + + reduced = apply_to_collection((torch.Tensor, numbers.Number), + _sync_ddp_to_device_type, device='cpu', dtype=torch.float) + + assert isinstance(reduced, dict), ' Type Consistency of dict not preserved' + assert all([x in reduced for x in to_reduce.keys()]), 'Not all entries of the dict were preserved' + assert all([isinstance(reduced[k], type(expected_result[k])) for k in to_reduce.keys()]), \ + 'At least one type was not correctly preserved' + + assert isinstance(reduced['a'], torch.Tensor), 'Reduction Result of a Tensor should be a Tensor' + assert torch.allclose(expected_result['a'], + reduced['a']), 'Reduction of a tensor does not yield the expected value' + + assert isinstance(reduced['b'], list), 'Reduction Result of a list should be a list' + assert all([torch.allclose(x, y) for x, y in zip(reduced['b'], expected_result['b'])]), \ + 'At least one value of list reduction did not come out as expected' + + assert isinstance(reduced['c'], tuple), 'Reduction Result of a tuple should be a tuple' + assert all([torch.allclose(x, y) for x, y in zip(reduced['c'], expected_result['c'])]), \ + 'At least one value of tuple reduction did not come out as expected' + + assert isinstance(reduced['d'], ntc), 'Type Consistency for named tuple not given' + assert isinstance(reduced['d'].bar, + torch.Tensor), 'Failure in type promotion while reducing fields of named tuples' + assert torch.allclose(reduced['d'].bar, expected_result['d'].bar) + + assert isinstance(reduced['e'], torch.Tensor), 'Type Promotion in reduction of numpy arrays failed' + assert torch.allclose(reduced['e'], expected_result['e']), \ + 'Reduction of numpy array did not yield the expected result' + + assert isinstance(reduced['f'], str), 'A string should not be reduced' + assert reduced['f'] == expected_result['f'], 'String not preserved during reduction' + + assert isinstance(reduced['g'], torch.Tensor), 'Reduction of a number should result in a tensor' + assert torch.allclose(reduced['g'], + expected_result['g']), 'Reduction of a number did not yield the desired result' From f44723b31e8882620d4e33e57789757a1ffc6058 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Wed, 1 Apr 2020 17:02:15 +0200 Subject: [PATCH 09/47] update test --- tests/utilities/test_apply_to_collection.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_to_collection.py index 1eccf72b74d49..d2629a6d64b29 100644 --- a/tests/utilities/test_apply_to_collection.py +++ b/tests/utilities/test_apply_to_collection.py @@ -1,10 +1,9 @@ import numbers from collections import namedtuple -import torch import numpy as np +import torch -from pytorch_lightning.metrics.metric import _sync_ddp_to_device_type from pytorch_lightning.utilities.apply_to_collection import apply_to_collection @@ -21,18 +20,18 @@ def test_recursive_application_to_collection(): } expected_result = { - 'a': torch.tensor([1.]), - 'b': [torch.tensor([2.])], - 'c': (torch.tensor([100.]),), - 'd': ntc(bar=torch.tensor([5.])), - 'e': torch.tensor([10.]), + 'a': torch.tensor([2.]), + 'b': [torch.tensor([4.])], + 'c': (torch.tensor([200.]),), + 'd': ntc(bar=torch.tensor([10.])), + 'e': torch.tensor([20.]), 'f': 'this_is_a_dummy_str', - 'g': torch.tensor([12.]) + 'g': torch.tensor([24.]) } - reduced = apply_to_collection((torch.Tensor, numbers.Number), - _sync_ddp_to_device_type, device='cpu', dtype=torch.float) + reduced = apply_to_collection(to_reduce, (torch.Tensor, numbers.Number, np.ndarray), + lambda x: x * 2) assert isinstance(reduced, dict), ' Type Consistency of dict not preserved' assert all([x in reduced for x in to_reduce.keys()]), 'Not all entries of the dict were preserved' From 5345ff9da338d29866ab36669e8ee08d588ee185 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Thu, 2 Apr 2020 14:26:29 +0200 Subject: [PATCH 10/47] Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index fb39d7b3b4a1a..c1b03e6b3a805 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -10,7 +10,7 @@ __all__ = ['AbstractMetric', 'TensorMetric', 'NumpyMetric'] -class AbstractMetric(torch.nn.Module, ABC): +class Metric(torch.nn.Module, ABC): def __init__(self, name: str): """ Abstract Base Class for metric implementation. From b6bd31c83dcff6f0d13617ecbbfd9493296ab8a9 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 14:27:46 +0200 Subject: [PATCH 11/47] update metric name --- pytorch_lightning/metrics/metric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index c1b03e6b3a805..eac8e679e0f78 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -7,7 +7,7 @@ from pytorch_lightning.metrics.utils import tensor_metric, numpy_metric from pytorch_lightning.utilities.apply_to_collection import apply_to_collection -__all__ = ['AbstractMetric', 'TensorMetric', 'NumpyMetric'] +__all__ = ['Metric', 'TensorMetric', 'NumpyMetric'] class Metric(torch.nn.Module, ABC): @@ -197,7 +197,7 @@ def half(self): return super().half() -class TensorMetric(AbstractMetric): +class TensorMetric(Metric): def __init__(self, name: str, reduce_group: Optional[Any] = torch.distributed.group.WORLD, reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): @@ -222,7 +222,7 @@ def __call__(self, *args, **kwargs) -> torch.Tensor: lambda x: x.to(device=self.device, dtype=self.dtype)) -class NumpyMetric(AbstractMetric): +class NumpyMetric(Metric): def __init__(self, name: str, reduce_group: Optional[Any] = torch.distributed.group.WORLD, reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): From 0473a26052348cf481b10d7d8bf63deae19eae14 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 14:32:53 +0200 Subject: [PATCH 12/47] remove example docs --- pytorch_lightning/metrics/metric.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index eac8e679e0f78..2714b2fb41982 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -92,34 +92,6 @@ def to(self, *args, **kwargs): Returns: Module: self - Example:: - - >>> linear = nn.Linear(2, 2) - >>> linear.weight - Parameter containing: - tensor([[ 0.1913, -0.3420], - [-0.5113, -0.2325]]) - >>> linear.to(torch.double) - Linear(in_features=2, out_features=2, bias=True) - >>> linear.weight - Parameter containing: - tensor([[ 0.1913, -0.3420], - [-0.5113, -0.2325]], dtype=torch.float64) - >>> gpu1 = torch.device("cuda:1") - >>> linear.to(gpu1, dtype=torch.half, non_blocking=True) - Linear(in_features=2, out_features=2, bias=True) - >>> linear.weight - Parameter containing: - tensor([[ 0.1914, -0.3420], - [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1') - >>> cpu = torch.device("cpu") - >>> linear.to(cpu) - Linear(in_features=2, out_features=2, bias=True) - >>> linear.weight - Parameter containing: - tensor([[ 0.1914, -0.3420], - [-0.5112, -0.2324]], dtype=torch.float16) - """ device, dtype, non_blocking = torch._C._nn._parse_to(*args, **kwargs) if device is not None: From f2b2e817f0ed8acdc038cc7812174f1fadaa5844 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 15:14:56 +0200 Subject: [PATCH 13/47] fix tests --- tests/metrics/test_utils.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/metrics/test_utils.py b/tests/metrics/test_utils.py index a6dfcf8be94a4..5a738baf60cd1 100644 --- a/tests/metrics/test_utils.py +++ b/tests/metrics/test_utils.py @@ -23,9 +23,9 @@ def test_fn(*args, **kwargs): return args, kwargs for args in [[], [1., 2.]]: - for kwargs in [{}, {1., 2.}]: + for kwargs in [{}, {'a': 1., 'b': 2.}]: result_args, result_kwargs = test_fn(*args, **kwargs) - assert isinstance(result_args, list) + assert isinstance(result_args, (list, tuple)) assert isinstance(result_kwargs, dict) assert len(result_args) == len(args) assert len(result_kwargs) == len(kwargs) @@ -34,7 +34,7 @@ def test_fn(*args, **kwargs): assert arg * 2. == result_arg for key in kwargs.keys(): - arg = kwargs[key], + arg = kwargs[key] result_arg = result_kwargs[key] assert arg * 2. == result_arg @@ -52,8 +52,9 @@ def test_fn(*args, **kwargs): def test_convert_to_tensor(): for test_item in [1., np.array([1.])]: - assert isinstance(_convert_to_tensor(test_item), torch.Tensor) - assert test_item.item() == 1. + result_tensor = _convert_to_tensor(test_item) + assert isinstance(result_tensor, torch.Tensor) + assert result_tensor.item() == 1. def test_convert_to_numpy(): @@ -95,7 +96,7 @@ def tensor_test_metric(*args, **kwargs): assert result.item() == 5. -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_sync_reduce_ddp(): """Make sure sync-reduce works with DDP""" tutils.reset_seed() @@ -110,12 +111,6 @@ def test_sync_reduce_ddp(): assert reduced_tensor.item() == dist.get_world_size(), \ 'Sync-Reduce does not work properly with DDP and Tensors' - number = 1. - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == dist.get_world_size(), \ - 'Sync-Reduce does not work properly with DDP and Numbers' - dist.destroy_process_group() @@ -128,12 +123,6 @@ def test_sync_reduce_simple(): assert torch.allclose(tensor, reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' - number = 1. - - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == number, 'Sync-Reduce does not work properly without DDP and Numbers' - def _test_tensor_metric(is_ddp: bool): @tensor_metric() @@ -156,7 +145,7 @@ def tensor_test_metric(*args, **kwargs): assert result.item() == 5. * factor -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_tensor_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() @@ -191,7 +180,7 @@ def numpy_test_metric(*args, **kwargs): assert result.item() == 5. * factor -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_numpy_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() From 3146c45e4154ff32aaed5eca846a4544933c064d Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 15:38:10 +0200 Subject: [PATCH 14/47] add metric tests --- tests/metrics/test_metrics.py | 85 +++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/metrics/test_metrics.py diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py new file mode 100644 index 0000000000000..e83a9d97b7a6c --- /dev/null +++ b/tests/metrics/test_metrics.py @@ -0,0 +1,85 @@ +import numpy as np +import torch + +from pytorch_lightning.metrics.metric import Metric, TensorMetric, NumpyMetric + + +class DummyTensorMetric(TensorMetric): + def __init__(self): + super().__init__('dummy') + + def forward(self, input1, input2): + assert isinstance(input1, torch.Tensor) + assert isinstance(input2, torch.Tensor) + return 1. + + +class DummyNumpyMetric(NumpyMetric): + def __init__(self): + super().__init__('dummy') + + def forward(self, input1, input2): + assert isinstance(input1, np.ndarray) + assert isinstance(input2, np.ndarray) + return 1. + + +def _test_metric(metric: Metric): + input1, input2 = torch.tensor([1.]), torch.tensor([2.]) + + def change_and_check_device_dtype(device, dtype): + metric.to(device=device, dtype=dtype) + + metric_val = metric(input1, input2) + assert isinstance(metric_val, torch.Tensor) + + if device is not None: + assert metric.device in [device, torch.device(device)] + assert metric_val.device in [device, torch.device(device)] + + if dtype is not None: + assert metric.dtype == dtype + assert metric_val.dtype == dtype + + devices = [None, 'cpu'] + if torch.cuda.is_available(): + devices += ['cuda:0'] + + for device in devices: + for dtype in [None, torch.float32, torch.float64]: + change_and_check_device_dtype(device=device, dtype=dtype) + + if torch.cuda.is_available(): + metric.cuda(0) + assert metric.device == torch.device('cuda', index=0) + assert metric(input1, input2).device == torch.device('cuda', index=0) + + metric.cpu() + assert metric.device == torch.device('cpu') + assert metric(input1, input2).device == torch.device('cpu') + + metric.type(torch.int8) + assert metric.dtype == torch.int8 + assert metric(input1, input2).dtype == torch.int8 + + metric.float() + assert metric.dtype == torch.float32 + assert metric(input1, input2).dtype == torch.float32 + + metric.double() + assert metric.dtype == torch.float64 + assert metric(input1, input2).dtype == torch.float64 + + if torch.cuda.is_available(): + metric.cuda() + metric.half() + assert metric.dtype == torch.float16 + assert metric(input1, input2).dtype == torch.float16 + + +def test_tensor_metric(): + _test_metric(DummyTensorMetric()) + + +def test_numpy_metric(): + _test_metric(DummyNumpyMetric()) From b9fcfc57efc881f8739443ff11a5a1d20c27cb4a Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 15:38:33 +0200 Subject: [PATCH 15/47] fix to tensor conversion --- pytorch_lightning/metrics/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/metrics/utils.py b/pytorch_lightning/metrics/utils.py index 1deced33e8f4f..84c6c3084c124 100644 --- a/pytorch_lightning/metrics/utils.py +++ b/pytorch_lightning/metrics/utils.py @@ -3,7 +3,7 @@ import numpy as np import torch -from torch.utils.data._utils.collate import default_convert +from torch.utils.data._utils.collate import np_str_obj_array_pattern from pytorch_lightning.utilities.apply_to_collection import apply_to_collection @@ -66,9 +66,11 @@ def _convert_to_tensor(data: Any) -> Any: """ if isinstance(data, numbers.Number): return torch.tensor([data]) - + # is not array of object + elif isinstance(data, np.ndarray) and np_str_obj_array_pattern.search(data.dtype.str) is None: + return torch.from_numpy(data) else: - return default_convert(data) + return data def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> np.ndarray: @@ -123,7 +125,9 @@ def _tensor_metric_conversion(func_to_decorate: Callable) -> Callable: """ # Converts all inputs to tensor if possible - func_convert_inputs = _apply_to_inputs(_convert_to_tensor)(func_to_decorate) + # we need to include tensors here, since otherwise they will also be treated as sequences + func_convert_inputs = _apply_to_inputs( + apply_to_collection, (torch.Tensor, np.ndarray, numbers.Number), _convert_to_tensor)(func_to_decorate) # convert all outputs to tensor if possible return _apply_to_outputs(_convert_to_tensor)(func_convert_inputs) From 1e52d7be4a43b42718b052deca072d45cd389441 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 16:08:49 +0200 Subject: [PATCH 16/47] fix apply to collection --- .../utilities/apply_to_collection.py | 2 +- tests/utilities/test_apply_to_collection.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/utilities/apply_to_collection.py b/pytorch_lightning/utilities/apply_to_collection.py index 5fdb58510b21d..869c29bc5fca4 100644 --- a/pytorch_lightning/utilities/apply_to_collection.py +++ b/pytorch_lightning/utilities/apply_to_collection.py @@ -29,7 +29,7 @@ def apply_to_collection(data: Any, dtype: Union[type, tuple], function: Callable return elem_type({k: apply_to_collection(v, dtype, function, *args, **kwargs) for k, v in data.items()}) elif isinstance(data, tuple) and hasattr(data, '_fields'): # named tuple - return elem_type(*(apply_to_collection(data, dtype, function, *args, **kwargs))) + return elem_type(*(apply_to_collection(d, dtype, function, *args, **kwargs) for d in data)) elif isinstance(data, Sequence) and not isinstance(data, str): return elem_type([apply_to_collection(d, dtype, function, *args, **kwargs) for d in data]) diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_to_collection.py index d2629a6d64b29..6e39a6c7878a6 100644 --- a/tests/utilities/test_apply_to_collection.py +++ b/tests/utilities/test_apply_to_collection.py @@ -9,6 +9,7 @@ def test_recursive_application_to_collection(): ntc = namedtuple('Foo', ['bar']) + to_reduce = { 'a': torch.tensor([1.]), # Tensor 'b': [torch.tensor([2.])], # list @@ -24,10 +25,9 @@ def test_recursive_application_to_collection(): 'b': [torch.tensor([4.])], 'c': (torch.tensor([200.]),), 'd': ntc(bar=torch.tensor([10.])), - 'e': torch.tensor([20.]), + 'e': np.array([20.]), 'f': 'this_is_a_dummy_str', - 'g': torch.tensor([24.]) - + 'g': 24. } reduced = apply_to_collection(to_reduce, (torch.Tensor, numbers.Number, np.ndarray), @@ -52,16 +52,15 @@ def test_recursive_application_to_collection(): assert isinstance(reduced['d'], ntc), 'Type Consistency for named tuple not given' assert isinstance(reduced['d'].bar, - torch.Tensor), 'Failure in type promotion while reducing fields of named tuples' - assert torch.allclose(reduced['d'].bar, expected_result['d'].bar) + numbers.Number), 'Failure in type promotion while reducing fields of named tuples' + assert reduced['d'].bar == expected_result['d'].bar - assert isinstance(reduced['e'], torch.Tensor), 'Type Promotion in reduction of numpy arrays failed' - assert torch.allclose(reduced['e'], expected_result['e']), \ + assert isinstance(reduced['e'], np.ndarray), 'Type Promotion in reduction of numpy arrays failed' + assert reduced['e'] == expected_result['e'], \ 'Reduction of numpy array did not yield the expected result' assert isinstance(reduced['f'], str), 'A string should not be reduced' assert reduced['f'] == expected_result['f'], 'String not preserved during reduction' - assert isinstance(reduced['g'], torch.Tensor), 'Reduction of a number should result in a tensor' - assert torch.allclose(reduced['g'], - expected_result['g']), 'Reduction of a number did not yield the desired result' + assert isinstance(reduced['g'], numbers.Number), 'Reduction of a number should result in a tensor' + assert reduced['g'] == expected_result['g'], 'Reduction of a number did not yield the desired result' From b2330b81b6ca10eaa19b9895834e90ed0fe21781 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Thu, 2 Apr 2020 16:33:45 +0200 Subject: [PATCH 17/47] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d7af014edecb..9a1802fa67809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). +## Metrics (will be added to unreleased once the metric branch was finished) +- Add Metric Base Classes ([#1326](https://github.com/PyTorchLightning/pytorch-lightning/pull/1326)) + ## [unreleased] - YYYY-MM-DD ### Added From f6cd0430de73e4aa9bc2cfb20f18ab734fc30338 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Thu, 2 Apr 2020 17:13:18 +0200 Subject: [PATCH 18/47] Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 2714b2fb41982..8d738c9cf04d7 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -78,7 +78,7 @@ def to(self, *args, **kwargs): See below for examples. - .. note:: + Note: This method modifies the module in-place. Args: From b6cd51b7364be6c95a028fe38b518263ffbe7d2f Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 17:15:44 +0200 Subject: [PATCH 19/47] remove tests from init --- tests/metrics/__init__.py | 205 -------------------------------------- 1 file changed, 205 deletions(-) diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py index a6dfcf8be94a4..e69de29bb2d1d 100644 --- a/tests/metrics/__init__.py +++ b/tests/metrics/__init__.py @@ -1,205 +0,0 @@ -import numpy as np -import pytest -import torch -import torch.distributed as dist - -import tests.base.utils as tutils -from pytorch_lightning.metrics.utils import _apply_to_inputs, _apply_to_outputs, \ - _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ - _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric - - -def test_apply_to_inputs(): - def apply_fn(inputs, factor): - if isinstance(inputs, (float, int)): - return inputs * factor - elif isinstance(inputs, dict): - return {k: apply_fn(v, factor) for k, v in inputs.items()} - elif isinstance(inputs, (tuple, list)): - return [apply_fn(x, factor) for x in inputs] - - @_apply_to_inputs(apply_fn, factor=2.) - def test_fn(*args, **kwargs): - return args, kwargs - - for args in [[], [1., 2.]]: - for kwargs in [{}, {1., 2.}]: - result_args, result_kwargs = test_fn(*args, **kwargs) - assert isinstance(result_args, list) - assert isinstance(result_kwargs, dict) - assert len(result_args) == len(args) - assert len(result_kwargs) == len(kwargs) - assert all([k in result_kwargs for k in kwargs.keys()]) - for arg, result_arg in zip(args, result_args): - assert arg * 2. == result_arg - - for key in kwargs.keys(): - arg = kwargs[key], - result_arg = result_kwargs[key] - assert arg * 2. == result_arg - - -def test_apply_to_outputs(): - def apply_fn(inputs, additional_str): - return str(inputs) + additional_str - - @_apply_to_outputs(apply_fn, additional_str='_str') - def test_fn(*args, **kwargs): - return 'dummy' - - assert test_fn() == 'dummy_str' - - -def test_convert_to_tensor(): - for test_item in [1., np.array([1.])]: - assert isinstance(_convert_to_tensor(test_item), torch.Tensor) - assert test_item.item() == 1. - - -def test_convert_to_numpy(): - for test_item in [1., torch.tensor([1.])]: - result = _convert_to_numpy(test_item) - assert isinstance(result, np.ndarray) - assert result.item() == 1. - - -def test_numpy_metric_conversion(): - @_numpy_metric_conversion - def numpy_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, np.ndarray) - - for v in kwargs.values(): - assert isinstance(v, np.ndarray) - - return 5. - - result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. - - -def test_tensor_metric_conversion(): - @_tensor_metric_conversion - def tensor_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, torch.Tensor) - - for v in kwargs.values(): - assert isinstance(v, torch.Tensor) - - return 5. - - result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_sync_reduce_ddp(): - """Make sure sync-reduce works with DDP""" - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - - tensor = torch.tensor([1.], device='cuda:0') - - reduced_tensor = _sync_ddp(tensor) - - assert reduced_tensor.item() == dist.get_world_size(), \ - 'Sync-Reduce does not work properly with DDP and Tensors' - - number = 1. - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == dist.get_world_size(), \ - 'Sync-Reduce does not work properly with DDP and Numbers' - - dist.destroy_process_group() - - -def test_sync_reduce_simple(): - """Make sure sync-reduce works without DDP""" - tensor = torch.tensor([1.], device='cpu') - - reduced_tensor = _sync_ddp(tensor) - - assert torch.allclose(tensor, - reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' - - number = 1. - - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == number, 'Sync-Reduce does not work properly without DDP and Numbers' - - -def _test_tensor_metric(is_ddp: bool): - @tensor_metric() - def tensor_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, torch.Tensor) - - for v in kwargs.values(): - assert isinstance(v, torch.Tensor) - - return 5. - - if is_ddp: - factor = dist.get_world_size() - else: - factor = 1. - - result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. * factor - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_tensor_metric_ddp(): - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - _test_tensor_metric(True) - dist.destroy_process_group() - - -def test_tensor_metric_simple(): - _test_tensor_metric(False) - - -def _test_numpy_metric(is_ddp: bool): - @numpy_metric() - def numpy_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, np.ndarray) - - for v in kwargs.values(): - assert isinstance(v, np.ndarray) - - return 5. - - if is_ddp: - factor = dist.get_world_size() - else: - factor = 1. - - result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. * factor - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_numpy_metric_ddp(): - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - _test_tensor_metric(True) - dist.destroy_process_group() - - -def test_numpy_metric_simple(): - _test_tensor_metric(False) From b57b933226351bdb0f28356e53f2c678a5f7a5f7 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 17:19:08 +0200 Subject: [PATCH 20/47] add missing type annotations --- pytorch_lightning/metrics/metric.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 8d738c9cf04d7..5979f322d6348 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Optional +from typing import Any, Optional, Union import torch import torch.distributed @@ -28,20 +28,20 @@ def __init__(self, name: str): self._device = torch.device('cpu') @property - def dtype(self): + def dtype(self) -> Union[str, torch.dtype]: return self._dtype @dtype.setter - def dtype(self, new_dtype: Any): + def dtype(self, new_dtype: Union[str, torch.dtype]): # Necessary to avoid infinite recursion raise RuntimeError('Cannot set the dtype explicitly. Please use metric.to(new_dtype).') @property - def device(self): + def device(self) -> Union[str, torch.device]: return self._device @device.setter - def device(self, new_device): + def device(self, new_device: Union[str, torch.device]): # Necessary to avoid infinite recursion raise RuntimeError('Cannot set the device explicitly. Please use metric.to(new_device).') @@ -56,7 +56,7 @@ def forward(self, *args, **kwargs) -> torch.Tensor: """ raise NotImplementedError - def to(self, *args, **kwargs): + def to(self, *args, **kwargs) -> torch.nn.Module: """Moves and/or casts the parameters and buffers. This can be called as @@ -102,7 +102,7 @@ def to(self, *args, **kwargs): return super().to(*args, **kwargs) - def cuda(self, device=None): + def cuda(self, device: Optional[int] = None) -> torch.nn.Module: """Moves all model parameters and buffers to the GPU. This also makes associated parameters and buffers different objects. So @@ -120,7 +120,7 @@ def cuda(self, device=None): self._device = torch.device('cuda', index=device) return super().cuda(device=device) - def cpu(self): + def cpu(self) -> torch.nn.Module: """Moves all model parameters and buffers to the CPU. Returns: @@ -129,7 +129,7 @@ def cpu(self): self._device = torch.device('cpu') return super().cpu() - def type(self, dst_type): + def type(self, dst_type: Union[str, torch.dtype]) -> torch.nn.Module: """Casts all parameters and buffers to :attr:`dst_type`. Arguments: @@ -141,7 +141,7 @@ def type(self, dst_type): self._dtype = dst_type return super().type(dst_type=dst_type) - def float(self): + def float(self) -> torch.nn.Module: """Casts all floating point parameters and buffers to float datatype. Returns: @@ -150,7 +150,7 @@ def float(self): self._dtype = torch.float return super().float() - def double(self): + def double(self) -> torch.nn.Module: """Casts all floating point parameters and buffers to ``double`` datatype. Returns: @@ -159,7 +159,7 @@ def double(self): self._dtype = torch.double return super().double() - def half(self): + def half(self) -> torch.nn.Module: """Casts all floating point parameters and buffers to ``half`` datatype. Returns: From ee58051f0cd51d7b700dd8ca12ba220d3bd07ec3 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Thu, 2 Apr 2020 17:21:30 +0200 Subject: [PATCH 21/47] rename utils to convertors --- pytorch_lightning/metrics/{utils.py => convertors.py} | 0 pytorch_lightning/metrics/metric.py | 2 +- tests/metrics/{test_utils.py => convertors.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename pytorch_lightning/metrics/{utils.py => convertors.py} (100%) rename tests/metrics/{test_utils.py => convertors.py} (98%) diff --git a/pytorch_lightning/metrics/utils.py b/pytorch_lightning/metrics/convertors.py similarity index 100% rename from pytorch_lightning/metrics/utils.py rename to pytorch_lightning/metrics/convertors.py diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 5979f322d6348..d75c47449c881 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -4,7 +4,7 @@ import torch import torch.distributed -from pytorch_lightning.metrics.utils import tensor_metric, numpy_metric +from pytorch_lightning.metrics.convertors import tensor_metric, numpy_metric from pytorch_lightning.utilities.apply_to_collection import apply_to_collection __all__ = ['Metric', 'TensorMetric', 'NumpyMetric'] diff --git a/tests/metrics/test_utils.py b/tests/metrics/convertors.py similarity index 98% rename from tests/metrics/test_utils.py rename to tests/metrics/convertors.py index 5a738baf60cd1..713ea14ba64a1 100644 --- a/tests/metrics/test_utils.py +++ b/tests/metrics/convertors.py @@ -4,7 +4,7 @@ import torch.distributed as dist import tests.base.utils as tutils -from pytorch_lightning.metrics.utils import _apply_to_inputs, _apply_to_outputs, \ +from pytorch_lightning.metrics.convertors import _apply_to_inputs, _apply_to_outputs, \ _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric From 4154090e0636a2515f268d6d920d0e3e669f1482 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 07:32:36 +0200 Subject: [PATCH 22/47] Create metrics.rst --- docs/source/metrics.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/source/metrics.rst diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst new file mode 100644 index 0000000000000..6f70a3c73f2d0 --- /dev/null +++ b/docs/source/metrics.rst @@ -0,0 +1,4 @@ +.. automodule:: pytorch_lightning.metrics + :members: + :noindex: + :exclude-members: From 2b0504379dce04fb09c018e8d22f4f672941ab50 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 07:33:03 +0200 Subject: [PATCH 23/47] Update index.rst --- docs/source/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1e11f7a0e9487..b6a9510bb3435 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,6 +23,7 @@ PyTorch Lightning Documentation hooks lightning-module loggers + metrics trainer .. toctree:: @@ -108,4 +109,4 @@ Indices and tables pytorch_lightning.overrides pytorch_lightning.profiler pytorch_lightning.trainer - pytorch_lightning.utilities \ No newline at end of file + pytorch_lightning.utilities From f980c62af159120589c5e699471bb13d13b6cea7 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 08:49:03 +0200 Subject: [PATCH 24/47] Update index.rst --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index b6a9510bb3435..68b9b2abcb263 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -106,6 +106,7 @@ Indices and tables pytorch_lightning.core pytorch_lightning.callbacks pytorch_lightning.loggers + pytorch_lightning.metrics pytorch_lightning.overrides pytorch_lightning.profiler pytorch_lightning.trainer From 2540555015c8127fbec6645e72ad49b1781a9a4d Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:44:54 +0200 Subject: [PATCH 25/47] Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/convertors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/metrics/convertors.py b/pytorch_lightning/metrics/convertors.py index 84c6c3084c124..90be579d8b5fd 100644 --- a/pytorch_lightning/metrics/convertors.py +++ b/pytorch_lightning/metrics/convertors.py @@ -69,8 +69,7 @@ def _convert_to_tensor(data: Any) -> Any: # is not array of object elif isinstance(data, np.ndarray) and np_str_obj_array_pattern.search(data.dtype.str) is None: return torch.from_numpy(data) - else: - return data + return data def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> np.ndarray: From 4df6f98e50ed60cd59ee5638dee9883bc4f52f77 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:45:03 +0200 Subject: [PATCH 26/47] Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/convertors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/metrics/convertors.py b/pytorch_lightning/metrics/convertors.py index 90be579d8b5fd..a281b9fd792c5 100644 --- a/pytorch_lightning/metrics/convertors.py +++ b/pytorch_lightning/metrics/convertors.py @@ -73,8 +73,7 @@ def _convert_to_tensor(data: Any) -> Any: def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> np.ndarray: - """ - converts all tensors and numpy arrays to numpy arrays + """Convert all tensors and numpy arrays to numpy arrays. Args: data: the tensor or array to convert to numpy From 20ec3755bc3dac37e7707e5051ed5b29d022464a Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:45:14 +0200 Subject: [PATCH 27/47] Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/convertors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/metrics/convertors.py b/pytorch_lightning/metrics/convertors.py index a281b9fd792c5..815db726aea9c 100644 --- a/pytorch_lightning/metrics/convertors.py +++ b/pytorch_lightning/metrics/convertors.py @@ -55,7 +55,7 @@ def new_func(*args, **kwargs): def _convert_to_tensor(data: Any) -> Any: """ - Maps all kind of collections and numbers to tensors + Maps all kind of collections and numbers to tensors. Args: data: the data to convert to tensor From 4a2ce4b72423b731e002e9f7e934d445fe78f8c7 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:45:29 +0200 Subject: [PATCH 28/47] Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index d75c47449c881..f5270461e51de 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -15,8 +15,8 @@ def __init__(self, name: str): """ Abstract Base Class for metric implementation. Should be used to implement metrics that - 1.) Return multiple Outputs - 2.) Handle their own DDP sync + 1. Return multiple Outputs + 2. Handle their own DDP sync Args: name: the metric's name From cdc3250e65667578c4febe667c7330e14d75fbeb Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:45:46 +0200 Subject: [PATCH 29/47] Update tests/utilities/test_apply_to_collection.py Co-Authored-By: Jirka Borovec --- tests/utilities/test_apply_to_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_to_collection.py index 6e39a6c7878a6..a3391bdc43356 100644 --- a/tests/utilities/test_apply_to_collection.py +++ b/tests/utilities/test_apply_to_collection.py @@ -39,8 +39,8 @@ def test_recursive_application_to_collection(): 'At least one type was not correctly preserved' assert isinstance(reduced['a'], torch.Tensor), 'Reduction Result of a Tensor should be a Tensor' - assert torch.allclose(expected_result['a'], - reduced['a']), 'Reduction of a tensor does not yield the expected value' + assert torch.allclose(expected_result['a'], reduced['a']), \ + 'Reduction of a tensor does not yield the expected value' assert isinstance(reduced['b'], list), 'Reduction Result of a list should be a list' assert all([torch.allclose(x, y) for x, y in zip(reduced['b'], expected_result['b'])]), \ From 25bf8ebdd807e7ac5b9ebf9e07230631760fe470 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:45:59 +0200 Subject: [PATCH 30/47] Update tests/utilities/test_apply_to_collection.py Co-Authored-By: Jirka Borovec --- tests/utilities/test_apply_to_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_to_collection.py index a3391bdc43356..c51d454308aea 100644 --- a/tests/utilities/test_apply_to_collection.py +++ b/tests/utilities/test_apply_to_collection.py @@ -51,8 +51,8 @@ def test_recursive_application_to_collection(): 'At least one value of tuple reduction did not come out as expected' assert isinstance(reduced['d'], ntc), 'Type Consistency for named tuple not given' - assert isinstance(reduced['d'].bar, - numbers.Number), 'Failure in type promotion while reducing fields of named tuples' + assert isinstance(reduced['d'].bar, numbers.Number), \ + 'Failure in type promotion while reducing fields of named tuples' assert reduced['d'].bar == expected_result['d'].bar assert isinstance(reduced['e'], np.ndarray), 'Type Promotion in reduction of numpy arrays failed' From 1577858a1c4bc91124ec903611a07ae8390f5ed5 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:46:14 +0200 Subject: [PATCH 31/47] Update tests/metrics/convertors.py Co-Authored-By: Jirka Borovec --- tests/metrics/convertors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metrics/convertors.py b/tests/metrics/convertors.py index 713ea14ba64a1..9d17032af6e96 100644 --- a/tests/metrics/convertors.py +++ b/tests/metrics/convertors.py @@ -120,8 +120,8 @@ def test_sync_reduce_simple(): reduced_tensor = _sync_ddp(tensor) - assert torch.allclose(tensor, - reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' + assert torch.allclose(tensor, reduced_tensor), \ + 'Sync-Reduce does not work properly without DDP and Tensors' def _test_tensor_metric(is_ddp: bool): From ad69ba376f03caa760c4db2f469de5febdad4901 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 09:50:38 +0200 Subject: [PATCH 32/47] Apply suggestions from code review Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/metric.py | 4 ++-- pytorch_lightning/utilities/apply_to_collection.py | 1 - tests/metrics/convertors.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index f5270461e51de..77182847b31b6 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -216,5 +216,5 @@ def __init__(self, name: str, reduce_op=reduce_op)(super().__call__) def __call__(self, *args, **kwargs) -> torch.Tensor: - return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, - lambda x: x.to(device=self.device, dtype=self.dtype)) + func_ = lambda x: x.to(device=self.device, dtype=self.dtype) + return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, func_) diff --git a/pytorch_lightning/utilities/apply_to_collection.py b/pytorch_lightning/utilities/apply_to_collection.py index 869c29bc5fca4..724715c3d8607 100644 --- a/pytorch_lightning/utilities/apply_to_collection.py +++ b/pytorch_lightning/utilities/apply_to_collection.py @@ -6,7 +6,6 @@ def apply_to_collection(data: Any, dtype: Union[type, tuple], function: Callable """ Recursively applies a function to all elements of a certain dtype. - Args: data: the collection to apply the function to dtype: the given function will be applied to all elements of this dtype diff --git a/tests/metrics/convertors.py b/tests/metrics/convertors.py index 9d17032af6e96..976261a12590a 100644 --- a/tests/metrics/convertors.py +++ b/tests/metrics/convertors.py @@ -4,9 +4,9 @@ import torch.distributed as dist import tests.base.utils as tutils -from pytorch_lightning.metrics.convertors import _apply_to_inputs, _apply_to_outputs, \ - _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ - _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric +from pytorch_lightning.metrics.convertors import ( + _apply_to_inputs, _apply_to_outputs, _convert_to_tensor, _convert_to_numpy, + _numpy_metric_conversion, tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric) def test_apply_to_inputs(): From 3c32445fb69242e2248fd1a997bbc76e65ddb9cc Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:19:06 +0200 Subject: [PATCH 33/47] add doctest example --- pytorch_lightning/metrics/metric.py | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 77182847b31b6..cd8f9b1effa56 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -92,6 +92,40 @@ def to(self, *args, **kwargs) -> torch.nn.Module: Returns: Module: self + Example:: + >>> class ExampleMetric(Metric): + ... def __init__(self, weight: torch.Tensor): + ... super().__init__('example') + ... self.register_buffer('weight', weight) + ... def forward(self, pred, target) -> torch.Tensor: + ... return (pred - target) * self.weight + >>> metric = ExampleMetric(torch.rand(3, 4)) + >>> metric.weight + tensor([[0.4242, 0.1002, 0.0840, 0.5361], + [0.9062, 0.2759, 0.0258, 0.2173], + [0.6806, 0.4048, 0.0426, 0.4487]]) + >>> metric.to(torch.double) + ExampleMetric() + >>> metric.weight + tensor([[0.4242, 0.1002, 0.0840, 0.5361], + [0.9062, 0.2759, 0.0258, 0.2173], + [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float64) + >>> gpu1 = torch.device("cuda:1") + >>> metric.to(gpu1, dtype=torch.half, non_blocking=True) + ExampleMetric() + >>> metric.weight + tensor([[0.4242, 0.1002, 0.0840, 0.5361], + [0.9062, 0.2759, 0.0258, 0.2173], + [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float16, device='cuda:1') + >>> cpu = torch.device("cpu") + >>> metric.to(cpu) + ExampleMetric() + >>> metric.weight + tensor([[0.4242, 0.1002, 0.0840, 0.5361], + [0.9062, 0.2759, 0.0258, 0.2173], + [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float16) + + """ device, dtype, non_blocking = torch._C._nn._parse_to(*args, **kwargs) if device is not None: From 80a8e7a4683a1df5ba7711b4bd4eb0e3e94a7c9f Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:20:38 +0200 Subject: [PATCH 34/47] rename file and fix imports --- tests/metrics/{convertors.py => test_convertors.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/metrics/{convertors.py => test_convertors.py} (98%) diff --git a/tests/metrics/convertors.py b/tests/metrics/test_convertors.py similarity index 98% rename from tests/metrics/convertors.py rename to tests/metrics/test_convertors.py index 976261a12590a..25bc203e050d4 100644 --- a/tests/metrics/convertors.py +++ b/tests/metrics/test_convertors.py @@ -6,7 +6,7 @@ import tests.base.utils as tutils from pytorch_lightning.metrics.convertors import ( _apply_to_inputs, _apply_to_outputs, _convert_to_tensor, _convert_to_numpy, - _numpy_metric_conversion, tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric) + _numpy_metric_conversion, _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric) def test_apply_to_inputs(): From 04cd3fefb38985fa3ae2c6c8e11580869518648a Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:27:39 +0200 Subject: [PATCH 35/47] added parametrized test --- tests/metrics/test_convertors.py | 41 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/metrics/test_convertors.py b/tests/metrics/test_convertors.py index 25bc203e050d4..dc114edd7e414 100644 --- a/tests/metrics/test_convertors.py +++ b/tests/metrics/test_convertors.py @@ -9,7 +9,12 @@ _numpy_metric_conversion, _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric) -def test_apply_to_inputs(): +@pytest.mark.parametrize(['args', 'kwargs'], + [pytest.param([], {}), + pytest.param([1., 2.], {}), + pytest.param([], {'a': 1., 'b': 2.}), + pytest.param([1., 2.], {'a': 1., 'b': 2.})]) +def test_apply_to_inputs(args, kwargs): def apply_fn(inputs, factor): if isinstance(inputs, (float, int)): return inputs * factor @@ -19,24 +24,22 @@ def apply_fn(inputs, factor): return [apply_fn(x, factor) for x in inputs] @_apply_to_inputs(apply_fn, factor=2.) - def test_fn(*args, **kwargs): - return args, kwargs - - for args in [[], [1., 2.]]: - for kwargs in [{}, {'a': 1., 'b': 2.}]: - result_args, result_kwargs = test_fn(*args, **kwargs) - assert isinstance(result_args, (list, tuple)) - assert isinstance(result_kwargs, dict) - assert len(result_args) == len(args) - assert len(result_kwargs) == len(kwargs) - assert all([k in result_kwargs for k in kwargs.keys()]) - for arg, result_arg in zip(args, result_args): - assert arg * 2. == result_arg - - for key in kwargs.keys(): - arg = kwargs[key] - result_arg = result_kwargs[key] - assert arg * 2. == result_arg + def test_fn(*func_args, **func_kwargs): + return func_args, func_kwargs + + result_args, result_kwargs = test_fn(*args, **kwargs) + assert isinstance(result_args, (list, tuple)) + assert isinstance(result_kwargs, dict) + assert len(result_args) == len(args) + assert len(result_kwargs) == len(kwargs) + assert all([k in result_kwargs for k in kwargs.keys()]) + for arg, result_arg in zip(args, result_args): + assert arg * 2. == result_arg + + for key in kwargs.keys(): + arg = kwargs[key] + result_arg = result_kwargs[key] + assert arg * 2. == result_arg def test_apply_to_outputs(): From 8c411ea4f77600838d4ceaa970979ad784df8562 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:29:32 +0200 Subject: [PATCH 36/47] replace lambda with inlined function --- pytorch_lightning/metrics/metric.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index cd8f9b1effa56..bf6071e007dff 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -224,8 +224,11 @@ def __init__(self, name: str, reduce_op=reduce_op)(super().__call__) def __call__(self, *args, **kwargs) -> torch.Tensor: + def _to_device_dtype(x: torch.Tensor) -> torch.Tensor: + return x.to(device=self.device, dtype=self.dtype) + return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, - lambda x: x.to(device=self.device, dtype=self.dtype)) + _to_device_dtype) class NumpyMetric(Metric): @@ -250,5 +253,8 @@ def __init__(self, name: str, reduce_op=reduce_op)(super().__call__) def __call__(self, *args, **kwargs) -> torch.Tensor: - func_ = lambda x: x.to(device=self.device, dtype=self.dtype) - return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, func_) + def _to_device_dtype(x: torch.Tensor) -> torch.Tensor: + return x.to(device=self.device, dtype=self.dtype) + + return apply_to_collection(self._orig_call(*args, **kwargs), torch.Tensor, + _to_device_dtype) From 1ec23d4a395e993383a47611327c6fdbce086882 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:32:45 +0200 Subject: [PATCH 37/47] rename apply_to_collection to apply_func --- pytorch_lightning/metrics/convertors.py | 2 +- pytorch_lightning/metrics/metric.py | 2 +- .../utilities/{apply_to_collection.py => apply_func.py} | 0 .../{test_apply_to_collection.py => test_apply_func.py} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename pytorch_lightning/utilities/{apply_to_collection.py => apply_func.py} (100%) rename tests/utilities/{test_apply_to_collection.py => test_apply_func.py} (97%) diff --git a/pytorch_lightning/metrics/convertors.py b/pytorch_lightning/metrics/convertors.py index 815db726aea9c..dbba7869ab8d3 100644 --- a/pytorch_lightning/metrics/convertors.py +++ b/pytorch_lightning/metrics/convertors.py @@ -5,7 +5,7 @@ import torch from torch.utils.data._utils.collate import np_str_obj_array_pattern -from pytorch_lightning.utilities.apply_to_collection import apply_to_collection +from pytorch_lightning.utilities.apply_func import apply_to_collection def _apply_to_inputs(func_to_apply: Callable, *dec_args, **dec_kwargs) -> Callable: diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index bf6071e007dff..b57702ed7eab9 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -5,7 +5,7 @@ import torch.distributed from pytorch_lightning.metrics.convertors import tensor_metric, numpy_metric -from pytorch_lightning.utilities.apply_to_collection import apply_to_collection +from pytorch_lightning.utilities.apply_func import apply_to_collection __all__ = ['Metric', 'TensorMetric', 'NumpyMetric'] diff --git a/pytorch_lightning/utilities/apply_to_collection.py b/pytorch_lightning/utilities/apply_func.py similarity index 100% rename from pytorch_lightning/utilities/apply_to_collection.py rename to pytorch_lightning/utilities/apply_func.py diff --git a/tests/utilities/test_apply_to_collection.py b/tests/utilities/test_apply_func.py similarity index 97% rename from tests/utilities/test_apply_to_collection.py rename to tests/utilities/test_apply_func.py index c51d454308aea..dce1e56e2b332 100644 --- a/tests/utilities/test_apply_to_collection.py +++ b/tests/utilities/test_apply_func.py @@ -4,7 +4,7 @@ import numpy as np import torch -from pytorch_lightning.utilities.apply_to_collection import apply_to_collection +from pytorch_lightning.utilities.apply_func import apply_to_collection def test_recursive_application_to_collection(): From 89a905008ae000c4a5e00a497fa9ecbd1e279ef8 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:36:36 +0200 Subject: [PATCH 38/47] Separated class description from init args --- pytorch_lightning/metrics/metric.py | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index b57702ed7eab9..c933ac6ec1404 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -11,13 +11,15 @@ class Metric(torch.nn.Module, ABC): + """ + Abstract Base Class for metric implementation. + + Should be used to implement metrics that + 1. Return multiple Outputs + 2. Handle their own DDP sync + """ def __init__(self, name: str): """ - Abstract Base Class for metric implementation. - Should be used to implement metrics that - 1. Return multiple Outputs - 2. Handle their own DDP sync - Args: name: the metric's name @@ -204,13 +206,15 @@ def half(self) -> torch.nn.Module: class TensorMetric(Metric): + """ + Base class for metric implementation operating directly on tensors. + All inputs and outputs will be casted to tensors if necessary. + Already handles DDP sync and input/output conversions. + """ def __init__(self, name: str, reduce_group: Optional[Any] = torch.distributed.group.WORLD, reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): """ - Base class for metric implementation operating directly on tensors. - All inputs and outputs will be casted to tensors if necessary. - Already handles DDP sync and input/output conversions Args: name: the metric's name @@ -232,14 +236,16 @@ def _to_device_dtype(x: torch.Tensor) -> torch.Tensor: class NumpyMetric(Metric): + """ + Base class for metric implementation operating on numpy arrays. + All inputs will be casted to numpy if necessary and all outputs will + be casted to tensors if necessary. + Already handles DDP sync and input/output conversions. + """ def __init__(self, name: str, reduce_group: Optional[Any] = torch.distributed.group.WORLD, reduce_op: Optional[Any] = torch.distributed.ReduceOp.SUM): """ - Base class for metric implementation operating on numpy arrays. - All inputs will be casted to numpy if necessary and all outputs will - be casted to tensors if necessary. - Already handles DDP sync and input/output conversions Args: name: the metric's name From c091c831669d593f24c374ab597f9f1240e91b2f Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 10:55:02 +0200 Subject: [PATCH 39/47] Apply suggestions from code review Co-Authored-By: Jirka Borovec --- pytorch_lightning/metrics/metric.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index c933ac6ec1404..9bf638e2266ed 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -101,6 +101,7 @@ def to(self, *args, **kwargs) -> torch.nn.Module: ... self.register_buffer('weight', weight) ... def forward(self, pred, target) -> torch.Tensor: ... return (pred - target) * self.weight + >>> torch.random.seed(0) >>> metric = ExampleMetric(torch.rand(3, 4)) >>> metric.weight tensor([[0.4242, 0.1002, 0.0840, 0.5361], From 91ce47b1918aa165c9ed15adc1c6e4ead95793eb Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 10:57:07 +0200 Subject: [PATCH 40/47] adjust random values --- pytorch_lightning/metrics/metric.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 9bf638e2266ed..85cc68c487fb6 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -101,32 +101,32 @@ def to(self, *args, **kwargs) -> torch.nn.Module: ... self.register_buffer('weight', weight) ... def forward(self, pred, target) -> torch.Tensor: ... return (pred - target) * self.weight - >>> torch.random.seed(0) + >>> torch.manual_seed(0) >>> metric = ExampleMetric(torch.rand(3, 4)) >>> metric.weight - tensor([[0.4242, 0.1002, 0.0840, 0.5361], - [0.9062, 0.2759, 0.0258, 0.2173], - [0.6806, 0.4048, 0.0426, 0.4487]]) + tensor([[0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017]]) >>> metric.to(torch.double) ExampleMetric() >>> metric.weight - tensor([[0.4242, 0.1002, 0.0840, 0.5361], - [0.9062, 0.2759, 0.0258, 0.2173], - [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float64) + tensor([[0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float64) >>> gpu1 = torch.device("cuda:1") >>> metric.to(gpu1, dtype=torch.half, non_blocking=True) ExampleMetric() >>> metric.weight - tensor([[0.4242, 0.1002, 0.0840, 0.5361], - [0.9062, 0.2759, 0.0258, 0.2173], - [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float16, device='cuda:1') + tensor([[0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16, device='cuda:1') >>> cpu = torch.device("cpu") >>> metric.to(cpu) ExampleMetric() >>> metric.weight - tensor([[0.4242, 0.1002, 0.0840, 0.5361], - [0.9062, 0.2759, 0.0258, 0.2173], - [0.6806, 0.4048, 0.0426, 0.4487]], dtype=torch.float16) + tensor([[0.4963, 0.7682, 0.0885, 0.1320], + [0.3074, 0.6341, 0.4901, 0.8964], + [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16) """ From f62dec69991b1eaf0287f1a20b83baa16a4d5043 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 11:08:57 +0200 Subject: [PATCH 41/47] suppress output when seeding --- pytorch_lightning/metrics/metric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 85cc68c487fb6..1b2a43cb5e5cb 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -101,7 +101,7 @@ def to(self, *args, **kwargs) -> torch.nn.Module: ... self.register_buffer('weight', weight) ... def forward(self, pred, target) -> torch.Tensor: ... return (pred - target) * self.weight - >>> torch.manual_seed(0) + >>> _ = torch.manual_seed(0) >>> metric = ExampleMetric(torch.rand(3, 4)) >>> metric.weight tensor([[0.4963, 0.7682, 0.0885, 0.1320], From b6e8ce4441b33212fa7fd0b4486b6aecdd4beac6 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 11:27:19 +0200 Subject: [PATCH 42/47] remove gpu from doctest --- pytorch_lightning/metrics/metric.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index 1b2a43cb5e5cb..de489627554b1 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -113,14 +113,13 @@ def to(self, *args, **kwargs) -> torch.nn.Module: tensor([[0.4963, 0.7682, 0.0885, 0.1320], [0.3074, 0.6341, 0.4901, 0.8964], [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float64) - >>> gpu1 = torch.device("cuda:1") - >>> metric.to(gpu1, dtype=torch.half, non_blocking=True) + >>> cpu = torch.device('cpu') + >>> metric.to(cpu, dtype=torch.half, non_blocking=True) ExampleMetric() >>> metric.weight tensor([[0.4963, 0.7682, 0.0885, 0.1320], [0.3074, 0.6341, 0.4901, 0.8964], - [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16, device='cuda:1') - >>> cpu = torch.device("cpu") + [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16) >>> metric.to(cpu) ExampleMetric() >>> metric.weight From 6d212e421b0fd54625b4a062e11473a61926d208 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 14:18:30 +0200 Subject: [PATCH 43/47] Add requested changes and add ellipsis for doctest --- .../metrics/{convertors.py => converters.py} | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) rename pytorch_lightning/metrics/{convertors.py => converters.py} (88%) diff --git a/pytorch_lightning/metrics/convertors.py b/pytorch_lightning/metrics/converters.py similarity index 88% rename from pytorch_lightning/metrics/convertors.py rename to pytorch_lightning/metrics/converters.py index dbba7869ab8d3..629da04c7db21 100644 --- a/pytorch_lightning/metrics/convertors.py +++ b/pytorch_lightning/metrics/converters.py @@ -1,3 +1,9 @@ +""" +This file provides functions and decorators for automated input and output +conversion to/from numpy.ndarray and torch.Tensor as well as utilities to +sync tensors between different processes in a DDP scenario, when needed. +""" + import numbers from typing import Union, Any, Callable @@ -19,6 +25,7 @@ def _apply_to_inputs(func_to_apply: Callable, *dec_args, **dec_kwargs) -> Callab Returns: the decorated function """ + def decorator_fn(func_to_decorate): # actual function applying the give function to inputs def new_func(*args, **kwargs): @@ -42,6 +49,7 @@ def _apply_to_outputs(func_to_apply: Callable, *dec_args, **dec_kwargs) -> Calla Returns: the decorated function """ + def decorator_fn(function_to_decorate): # actual function applying the give function to outputs def new_func(*args, **kwargs): @@ -69,7 +77,8 @@ def _convert_to_tensor(data: Any) -> Any: # is not array of object elif isinstance(data, np.ndarray) and np_str_obj_array_pattern.search(data.dtype.str) is None: return torch.from_numpy(data) - return data + + raise TypeError("The given type ('%s') cannot be converted to a tensor!" % type(data).__name__) def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> np.ndarray: @@ -85,7 +94,8 @@ def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> return data.cpu().detach().numpy() elif isinstance(data, numbers.Number): return np.array([data]) - return data + + raise TypeError("The given type ('%s') cannot be converted to a numpy array!" % type(data).__name__) def _numpy_metric_conversion(func_to_decorate: Callable) -> Callable: @@ -101,7 +111,7 @@ def _numpy_metric_conversion(func_to_decorate: Callable) -> Callable: the decorated function """ - # Applies collection conversion from tensor to numpy to all inputs + # applies collection conversion from tensor to numpy to all inputs # we need to include numpy arrays here, since otherwise they will also be treated as sequences func_convert_inputs = _apply_to_inputs( apply_to_collection, (torch.Tensor, np.ndarray, numbers.Number), _convert_to_numpy)(func_to_decorate) @@ -122,7 +132,7 @@ def _tensor_metric_conversion(func_to_decorate: Callable) -> Callable: the decorated function """ - # Converts all inputs to tensor if possible + # converts all inputs to tensor if possible # we need to include tensors here, since otherwise they will also be treated as sequences func_convert_inputs = _apply_to_inputs( apply_to_collection, (torch.Tensor, np.ndarray, numbers.Number), _convert_to_tensor)(func_to_decorate) @@ -130,10 +140,10 @@ def _tensor_metric_conversion(func_to_decorate: Callable) -> Callable: return _apply_to_outputs(_convert_to_tensor)(func_convert_inputs) -def _sync_ddp(result: Union[torch.Tensor], - group: Any = torch.distributed.group.WORLD, - reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM, - ) -> torch.Tensor: +def _sync_ddp_if_available(result: Union[torch.Tensor], + group: Any = torch.distributed.group.WORLD, + reduce_op: torch.distributed.ReduceOp = torch.distributed.ReduceOp.SUM, + ) -> torch.Tensor: """ Function to reduce the tensors from several ddp processes to one master process @@ -174,8 +184,9 @@ def numpy_metric(group: Any = torch.distributed.group.WORLD, the decorated function """ + def decorator_fn(func_to_decorate): - return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, + return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp_if_available, group=group, reduce_op=reduce_op)(_numpy_metric_conversion(func_to_decorate)) @@ -199,8 +210,9 @@ def tensor_metric(group: Any = torch.distributed.group.WORLD, the decorated function """ + def decorator_fn(func_to_decorate): - return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp, + return _apply_to_outputs(apply_to_collection, torch.Tensor, _sync_ddp_if_available, group=group, reduce_op=reduce_op)(_tensor_metric_conversion(func_to_decorate)) From b0739e19a4b1c59c4144dd6e4c3f8811ef82a202 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 14:22:20 +0200 Subject: [PATCH 44/47] forgot to push these files... --- pytorch_lightning/metrics/metric.py | 22 +++++++------------ ...{test_convertors.py => test_converters.py} | 8 +++---- 2 files changed, 12 insertions(+), 18 deletions(-) rename tests/metrics/{test_convertors.py => test_converters.py} (96%) diff --git a/pytorch_lightning/metrics/metric.py b/pytorch_lightning/metrics/metric.py index de489627554b1..50853105f94f9 100644 --- a/pytorch_lightning/metrics/metric.py +++ b/pytorch_lightning/metrics/metric.py @@ -4,7 +4,7 @@ import torch import torch.distributed -from pytorch_lightning.metrics.convertors import tensor_metric, numpy_metric +from pytorch_lightning.metrics.converters import tensor_metric, numpy_metric from pytorch_lightning.utilities.apply_func import apply_to_collection __all__ = ['Metric', 'TensorMetric', 'NumpyMetric'] @@ -35,7 +35,7 @@ def dtype(self) -> Union[str, torch.dtype]: @dtype.setter def dtype(self, new_dtype: Union[str, torch.dtype]): - # Necessary to avoid infinite recursion + # necessary to avoid infinite recursion raise RuntimeError('Cannot set the dtype explicitly. Please use metric.to(new_dtype).') @property @@ -107,25 +107,19 @@ def to(self, *args, **kwargs) -> torch.nn.Module: tensor([[0.4963, 0.7682, 0.0885, 0.1320], [0.3074, 0.6341, 0.4901, 0.8964], [0.4556, 0.6323, 0.3489, 0.4017]]) - >>> metric.to(torch.double) + >>> metric.to(torch.double) #doctest: +ELLIPSIS ExampleMetric() >>> metric.weight - tensor([[0.4963, 0.7682, 0.0885, 0.1320], - [0.3074, 0.6341, 0.4901, 0.8964], - [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float64) + tensor([[...]], dtype=torch.float64) >>> cpu = torch.device('cpu') >>> metric.to(cpu, dtype=torch.half, non_blocking=True) ExampleMetric() - >>> metric.weight - tensor([[0.4963, 0.7682, 0.0885, 0.1320], - [0.3074, 0.6341, 0.4901, 0.8964], - [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16) + >>> metric.weight #doctest: +ELLIPSIS + tensor([[...]], dtype=torch.float16) >>> metric.to(cpu) ExampleMetric() - >>> metric.weight - tensor([[0.4963, 0.7682, 0.0885, 0.1320], - [0.3074, 0.6341, 0.4901, 0.8964], - [0.4556, 0.6323, 0.3489, 0.4017]], dtype=torch.float16) + >>> metric.weight #doctest: +ELLIPSIS + tensor([[...]], dtype=torch.float16) """ diff --git a/tests/metrics/test_convertors.py b/tests/metrics/test_converters.py similarity index 96% rename from tests/metrics/test_convertors.py rename to tests/metrics/test_converters.py index dc114edd7e414..0ad9e476ed092 100644 --- a/tests/metrics/test_convertors.py +++ b/tests/metrics/test_converters.py @@ -4,9 +4,9 @@ import torch.distributed as dist import tests.base.utils as tutils -from pytorch_lightning.metrics.convertors import ( +from pytorch_lightning.metrics.converters import ( _apply_to_inputs, _apply_to_outputs, _convert_to_tensor, _convert_to_numpy, - _numpy_metric_conversion, _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric) + _numpy_metric_conversion, _tensor_metric_conversion, _sync_ddp_if_available, tensor_metric, numpy_metric) @pytest.mark.parametrize(['args', 'kwargs'], @@ -109,7 +109,7 @@ def test_sync_reduce_ddp(): tensor = torch.tensor([1.], device='cuda:0') - reduced_tensor = _sync_ddp(tensor) + reduced_tensor = _sync_ddp_if_available(tensor) assert reduced_tensor.item() == dist.get_world_size(), \ 'Sync-Reduce does not work properly with DDP and Tensors' @@ -121,7 +121,7 @@ def test_sync_reduce_simple(): """Make sure sync-reduce works without DDP""" tensor = torch.tensor([1.], device='cpu') - reduced_tensor = _sync_ddp(tensor) + reduced_tensor = _sync_ddp_if_available(tensor) assert torch.allclose(tensor, reduced_tensor), \ 'Sync-Reduce does not work properly without DDP and Tensors' From a70513b9fb9651854b59769f8117633360c84964 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 14:29:20 +0200 Subject: [PATCH 45/47] add explicit check for dtype to convert to --- pytorch_lightning/metrics/converters.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/metrics/converters.py b/pytorch_lightning/metrics/converters.py index 629da04c7db21..8162876fc3b00 100644 --- a/pytorch_lightning/metrics/converters.py +++ b/pytorch_lightning/metrics/converters.py @@ -77,6 +77,8 @@ def _convert_to_tensor(data: Any) -> Any: # is not array of object elif isinstance(data, np.ndarray) and np_str_obj_array_pattern.search(data.dtype.str) is None: return torch.from_numpy(data) + elif isinstance(data, torch.Tensor): + return data raise TypeError("The given type ('%s') cannot be converted to a tensor!" % type(data).__name__) @@ -94,6 +96,8 @@ def _convert_to_numpy(data: Union[torch.Tensor, np.ndarray, numbers.Number]) -> return data.cpu().detach().numpy() elif isinstance(data, numbers.Number): return np.array([data]) + elif isinstance(data, np.ndarray): + return data raise TypeError("The given type ('%s') cannot be converted to a numpy array!" % type(data).__name__) From 507b6f941e19153d401a72033a254841dba00052 Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 16:37:56 +0200 Subject: [PATCH 46/47] fix ddp tests --- tests/metrics/test_converters.py | 46 ++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/metrics/test_converters.py b/tests/metrics/test_converters.py index 0ad9e476ed092..babfafa79e97e 100644 --- a/tests/metrics/test_converters.py +++ b/tests/metrics/test_converters.py @@ -2,6 +2,7 @@ import pytest import torch import torch.distributed as dist +import torch.multiprocessing as mp import tests.base.utils as tutils from pytorch_lightning.metrics.converters import ( @@ -99,14 +100,17 @@ def tensor_test_metric(*args, **kwargs): assert result.item() == 5. -@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") -def test_sync_reduce_ddp(): - """Make sure sync-reduce works with DDP""" - tutils.reset_seed() - tutils.set_random_master_port() +def setup_ddp(rank, worldsize, ): + import os + + os.environ['MASTER_ADDR'] = 'localhost' - dist.init_process_group('gloo') + # initialize the process group + dist.init_process_group("gloo", rank=rank, world_size=worldsize) + +def ddp_test_fn(rank, worldsize): + setup_ddp(rank, worldsize) tensor = torch.tensor([1.], device='cuda:0') reduced_tensor = _sync_ddp_if_available(tensor) @@ -114,6 +118,16 @@ def test_sync_reduce_ddp(): assert reduced_tensor.item() == dist.get_world_size(), \ 'Sync-Reduce does not work properly with DDP and Tensors' + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") +def test_sync_reduce_ddp(): + """Make sure sync-reduce works with DDP""" + tutils.reset_seed() + tutils.set_random_master_port() + + worldsize = 2 + mp.spawn(ddp_test_fn, args=(worldsize,), nprocs=worldsize) + dist.destroy_process_group() @@ -148,13 +162,19 @@ def tensor_test_metric(*args, **kwargs): assert result.item() == 5. * factor +def _ddp_test_tensor_metric(rank, worldsize): + setup_ddp(rank, worldsize) + _test_tensor_metric(True) + + @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_tensor_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() - dist.init_process_group('gloo') - _test_tensor_metric(True) + world_size = 2 + mp.spawn(_ddp_test_tensor_metric, args=(world_size,), nprocs=world_size) + dist.destroy_process_group() @@ -183,13 +203,17 @@ def numpy_test_metric(*args, **kwargs): assert result.item() == 5. * factor +def _ddp_test_numpy_metric(rank, worldsize): + setup_ddp(rank, worldsize) + _test_numpy_metric(True) + + @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_numpy_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() - - dist.init_process_group('gloo') - _test_tensor_metric(True) + world_size = 2 + mp.spawn(_ddp_test_numpy_metric, args=(world_size,), nprocs=world_size) dist.destroy_process_group() From 79f0731bcf32338e7d604cf5a930dcbbd88ad4db Mon Sep 17 00:00:00 2001 From: Justus Schock Date: Fri, 3 Apr 2020 16:50:02 +0200 Subject: [PATCH 47/47] remove explicit ddp destruction --- tests/metrics/test_converters.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/metrics/test_converters.py b/tests/metrics/test_converters.py index babfafa79e97e..9abc11d4b07ad 100644 --- a/tests/metrics/test_converters.py +++ b/tests/metrics/test_converters.py @@ -128,8 +128,6 @@ def test_sync_reduce_ddp(): worldsize = 2 mp.spawn(ddp_test_fn, args=(worldsize,), nprocs=worldsize) - dist.destroy_process_group() - def test_sync_reduce_simple(): """Make sure sync-reduce works without DDP""" @@ -167,7 +165,6 @@ def _ddp_test_tensor_metric(rank, worldsize): _test_tensor_metric(True) -@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_tensor_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() @@ -175,8 +172,6 @@ def test_tensor_metric_ddp(): world_size = 2 mp.spawn(_ddp_test_tensor_metric, args=(world_size,), nprocs=world_size) - dist.destroy_process_group() - def test_tensor_metric_simple(): _test_tensor_metric(False) @@ -208,13 +203,11 @@ def _ddp_test_numpy_metric(rank, worldsize): _test_numpy_metric(True) -@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_numpy_metric_ddp(): tutils.reset_seed() tutils.set_random_master_port() world_size = 2 mp.spawn(_ddp_test_numpy_metric, args=(world_size,), nprocs=world_size) - dist.destroy_process_group() def test_numpy_metric_simple():