From 2722c08e9ffb0e4ac46519c8f916d3635a64b701 Mon Sep 17 00:00:00 2001 From: Justus Schock <12886177+justusschock@users.noreply.github.com> Date: Fri, 3 Apr 2020 21:10:40 +0200 Subject: [PATCH] New metric classes (#1326) * Create metrics package * Create metric.py * Create utils.py * Create __init__.py * add tests for metric utils * add docstrings for metrics utils * add function to recursively apply other function to collection * add tests for this function * update test * Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec * update metric name * remove example docs * fix tests * add metric tests * fix to tensor conversion * fix apply to collection * Update CHANGELOG.md * Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec * remove tests from init * add missing type annotations * rename utils to convertors * Create metrics.rst * Update index.rst * Update index.rst * Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec * Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec * Update pytorch_lightning/metrics/convertors.py Co-Authored-By: Jirka Borovec * Update pytorch_lightning/metrics/metric.py Co-Authored-By: Jirka Borovec * Update tests/utilities/test_apply_to_collection.py Co-Authored-By: Jirka Borovec * Update tests/utilities/test_apply_to_collection.py Co-Authored-By: Jirka Borovec * Update tests/metrics/convertors.py Co-Authored-By: Jirka Borovec * Apply suggestions from code review Co-Authored-By: Jirka Borovec * add doctest example * rename file and fix imports * added parametrized test * replace lambda with inlined function * rename apply_to_collection to apply_func * Separated class description from init args * Apply suggestions from code review Co-Authored-By: Jirka Borovec * adjust random values * suppress output when seeding * remove gpu from doctest * Add requested changes and add ellipsis for doctest * forgot to push these files... * add explicit check for dtype to convert to * fix ddp tests * remove explicit ddp destruction Co-authored-by: Jirka Borovec --- tests/metrics/__init__.py | 205 -------------------------------------- 1 file changed, 205 deletions(-) diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py index a6dfcf8be94a4..e69de29bb2d1d 100644 --- a/tests/metrics/__init__.py +++ b/tests/metrics/__init__.py @@ -1,205 +0,0 @@ -import numpy as np -import pytest -import torch -import torch.distributed as dist - -import tests.base.utils as tutils -from pytorch_lightning.metrics.utils import _apply_to_inputs, _apply_to_outputs, \ - _convert_to_tensor, _convert_to_numpy, _numpy_metric_conversion, \ - _tensor_metric_conversion, _sync_ddp, tensor_metric, numpy_metric - - -def test_apply_to_inputs(): - def apply_fn(inputs, factor): - if isinstance(inputs, (float, int)): - return inputs * factor - elif isinstance(inputs, dict): - return {k: apply_fn(v, factor) for k, v in inputs.items()} - elif isinstance(inputs, (tuple, list)): - return [apply_fn(x, factor) for x in inputs] - - @_apply_to_inputs(apply_fn, factor=2.) - def test_fn(*args, **kwargs): - return args, kwargs - - for args in [[], [1., 2.]]: - for kwargs in [{}, {1., 2.}]: - result_args, result_kwargs = test_fn(*args, **kwargs) - assert isinstance(result_args, list) - assert isinstance(result_kwargs, dict) - assert len(result_args) == len(args) - assert len(result_kwargs) == len(kwargs) - assert all([k in result_kwargs for k in kwargs.keys()]) - for arg, result_arg in zip(args, result_args): - assert arg * 2. == result_arg - - for key in kwargs.keys(): - arg = kwargs[key], - result_arg = result_kwargs[key] - assert arg * 2. == result_arg - - -def test_apply_to_outputs(): - def apply_fn(inputs, additional_str): - return str(inputs) + additional_str - - @_apply_to_outputs(apply_fn, additional_str='_str') - def test_fn(*args, **kwargs): - return 'dummy' - - assert test_fn() == 'dummy_str' - - -def test_convert_to_tensor(): - for test_item in [1., np.array([1.])]: - assert isinstance(_convert_to_tensor(test_item), torch.Tensor) - assert test_item.item() == 1. - - -def test_convert_to_numpy(): - for test_item in [1., torch.tensor([1.])]: - result = _convert_to_numpy(test_item) - assert isinstance(result, np.ndarray) - assert result.item() == 1. - - -def test_numpy_metric_conversion(): - @_numpy_metric_conversion - def numpy_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, np.ndarray) - - for v in kwargs.values(): - assert isinstance(v, np.ndarray) - - return 5. - - result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. - - -def test_tensor_metric_conversion(): - @_tensor_metric_conversion - def tensor_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, torch.Tensor) - - for v in kwargs.values(): - assert isinstance(v, torch.Tensor) - - return 5. - - result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_sync_reduce_ddp(): - """Make sure sync-reduce works with DDP""" - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - - tensor = torch.tensor([1.], device='cuda:0') - - reduced_tensor = _sync_ddp(tensor) - - assert reduced_tensor.item() == dist.get_world_size(), \ - 'Sync-Reduce does not work properly with DDP and Tensors' - - number = 1. - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == dist.get_world_size(), \ - 'Sync-Reduce does not work properly with DDP and Numbers' - - dist.destroy_process_group() - - -def test_sync_reduce_simple(): - """Make sure sync-reduce works without DDP""" - tensor = torch.tensor([1.], device='cpu') - - reduced_tensor = _sync_ddp(tensor) - - assert torch.allclose(tensor, - reduced_tensor), 'Sync-Reduce does not work properly without DDP and Tensors' - - number = 1. - - reduced_number = _sync_ddp(number) - assert isinstance(reduced_number, torch.Tensor), 'When reducing a number we should get a tensor out' - assert reduced_number.item() == number, 'Sync-Reduce does not work properly without DDP and Numbers' - - -def _test_tensor_metric(is_ddp: bool): - @tensor_metric() - def tensor_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, torch.Tensor) - - for v in kwargs.values(): - assert isinstance(v, torch.Tensor) - - return 5. - - if is_ddp: - factor = dist.get_world_size() - else: - factor = 1. - - result = tensor_test_metric(np.array([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. * factor - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_tensor_metric_ddp(): - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - _test_tensor_metric(True) - dist.destroy_process_group() - - -def test_tensor_metric_simple(): - _test_tensor_metric(False) - - -def _test_numpy_metric(is_ddp: bool): - @numpy_metric() - def numpy_test_metric(*args, **kwargs): - for arg in args: - assert isinstance(arg, np.ndarray) - - for v in kwargs.values(): - assert isinstance(v, np.ndarray) - - return 5. - - if is_ddp: - factor = dist.get_world_size() - else: - factor = 1. - - result = numpy_test_metric(torch.tensor([1.]), dummy_kwarg=2.) - assert isinstance(result, torch.Tensor) - assert result.item() == 5. * factor - - -@pytest.mark.skipif(torch.cuda.device_count() < 2, "test requires multi-GPU machine") -def test_numpy_metric_ddp(): - tutils.reset_seed() - tutils.set_random_master_port() - - dist.init_process_group('gloo') - _test_tensor_metric(True) - dist.destroy_process_group() - - -def test_numpy_metric_simple(): - _test_tensor_metric(False)