diff --git a/frontend/archive/src/main/java/org/pytorch/serve/archive/utils/ArchiveUtils.java b/frontend/archive/src/main/java/org/pytorch/serve/archive/utils/ArchiveUtils.java index ff24e483e3..370763a6ae 100644 --- a/frontend/archive/src/main/java/org/pytorch/serve/archive/utils/ArchiveUtils.java +++ b/frontend/archive/src/main/java/org/pytorch/serve/archive/utils/ArchiveUtils.java @@ -21,6 +21,7 @@ import org.pytorch.serve.archive.DownloadArchiveException; import org.pytorch.serve.archive.model.InvalidModelException; import org.pytorch.serve.archive.s3.HttpUtils; +import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.error.YAMLException; @@ -47,7 +48,7 @@ public static T readFile(File file, Class type) public static T readYamlFile(File file, Class type) throws InvalidModelException, IOException { - Yaml yaml = new Yaml(new Constructor(type)); + Yaml yaml = new Yaml(new Constructor(type, new LoaderOptions())); try (Reader r = new InputStreamReader( Files.newInputStream(file.toPath()), StandardCharsets.UTF_8)) { diff --git a/frontend/gradle.properties b/frontend/gradle.properties index 2590230cc1..c9f7d91936 100644 --- a/frontend/gradle.properties +++ b/frontend/gradle.properties @@ -8,7 +8,7 @@ slf4j_api_version=1.7.32 slf4j_log4j_version=2.17.1 testng_version=7.1.0 torchserve_sdk_version=0.0.5 -snakeyaml_version=1.31 +snakeyaml_version=2.1 grpc_version=1.50.0 protoc_version=3.18.0 lmax_disruptor_version=3.4.4 diff --git a/frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricConfiguration.java b/frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricConfiguration.java index cb41a0d907..12f4c8bd72 100644 --- a/frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricConfiguration.java +++ b/frontend/server/src/main/java/org/pytorch/serve/metrics/configuration/MetricConfiguration.java @@ -6,6 +6,7 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.composer.ComposerException; import org.yaml.snakeyaml.constructor.Constructor; @@ -64,7 +65,7 @@ public void validate() { public static MetricConfiguration loadConfiguration(String configFilePath) throws FileNotFoundException, ComposerException, RuntimeException { - Constructor constructor = new Constructor(MetricConfiguration.class); + Constructor constructor = new Constructor(MetricConfiguration.class, new LoaderOptions()); Yaml yaml = new Yaml(constructor); FileInputStream inputStream = new FileInputStream(new File(configFilePath)); MetricConfiguration config = yaml.load(inputStream); diff --git a/ts/metrics/caching_metric.py b/ts/metrics/caching_metric.py index 6dd230cea5..89f99b6d34 100644 --- a/ts/metrics/caching_metric.py +++ b/ts/metrics/caching_metric.py @@ -8,6 +8,7 @@ from ts.metrics.dimension import Dimension from ts.metrics.metric_abstract import MetricAbstract from ts.metrics.metric_type_enum import MetricTypes + logger = logging.getLogger(__name__) @@ -36,7 +37,7 @@ def __init__( unit can be one of ms, percent, count, MB, GB or a generic string dimension_names list - list of dimension names which should be strings + list of dimension name strings metric_type MetricTypes Type of metric Counter, Gauge, Histogram @@ -57,9 +58,11 @@ def _validate_and_get_dimensions( values corresponding to the metrics dimension names Returns ------- - list of dimension objects or ValueError + list of Dimension objects or ValueError """ - if dimension_values is None or len(dimension_values) != len(self.dimension_names): + if dimension_values is None or len(dimension_values) != len( + self.dimension_names + ): raise ValueError( f"Dimension values: {dimension_values} should " f"correspond to Dimension names: {self.dimension_names}" @@ -97,8 +100,10 @@ def emit_metrics( value dimension_string """ - metric_str = f"[METRICS]{self.metric_name}.{self.unit}:{value}|#{dimension_string}|" \ - f"#hostname:{socket.gethostname()},{int(time.time())}" + metric_str = ( + f"[METRICS]{self.metric_name}.{self.unit}:{value}|#{dimension_string}|" + f"#hostname:{socket.gethostname()},{int(time.time())}" + ) if request_id: logger.info(f"{metric_str},{request_id}") else: @@ -118,7 +123,7 @@ def add_or_update( value : int, float metric to be updated dimension_values : list - list of dimension values + list of dimension value strings request_id : str request id to be associated with the metric """ @@ -152,7 +157,7 @@ def update( request_id : str request id to be associated with the metric dimensions : list - list of dimension values + list of Dimension objects """ logger.warning("Overriding existing dimensions") self.dimension_names = [dim.name for dim in dimensions] diff --git a/ts/metrics/metric.py b/ts/metrics/metric.py index 7cbdc9e0f1..c52f934a67 100644 --- a/ts/metrics/metric.py +++ b/ts/metrics/metric.py @@ -5,9 +5,9 @@ import time from builtins import str from collections import OrderedDict + from ts.metrics.caching_metric import CachingMetric from ts.metrics.metric_type_enum import MetricTypes - from ts.metrics.unit import Units MetricUnit = Units() @@ -41,7 +41,7 @@ def __init__( unit: str unit can be one of ms, percent, count, MB, GB or a generic string dimensions: list - list of dimension objects + list of Dimension objects request_id: str req_id of metric metric_method: str @@ -73,13 +73,17 @@ def update(self, value): value : int, float metric to be updated """ - self._caching_metric.add_or_update(value, self.dimension_values, request_id=self.request_id) + self._caching_metric.add_or_update( + value, self.dimension_values, request_id=self.request_id + ) def reset(self): """ Reset Metric value to 0 """ - self._caching_metric.add_or_update(0, self.dimension_values, request_id=self.request_id) + self._caching_metric.add_or_update( + 0, self.dimension_values, request_id=self.request_id + ) def __str__(self): dims = ",".join([str(d) for d in self.dimensions]) diff --git a/ts/metrics/metric_abstract.py b/ts/metrics/metric_abstract.py index 287c9c08b1..99ee151d0d 100644 --- a/ts/metrics/metric_abstract.py +++ b/ts/metrics/metric_abstract.py @@ -2,8 +2,9 @@ Interface for metric class for TS """ import abc -from ts.metrics.unit import Units + from ts.metrics.metric_type_enum import MetricTypes +from ts.metrics.unit import Units MetricUnit = Units() @@ -33,7 +34,7 @@ def __init__( unit can be one of ms, percent, count, MB, GB or a generic string dimension_names list - list of dimension names which should be strings + list of dimension name strings metric_type MetricTypes Type of metric Counter, Gauge, Histogram diff --git a/ts/metrics/metric_cache_abstract.py b/ts/metrics/metric_cache_abstract.py index 2dba65075e..37abad7da1 100644 --- a/ts/metrics/metric_cache_abstract.py +++ b/ts/metrics/metric_cache_abstract.py @@ -6,8 +6,8 @@ """ import abc import os -import ts.metrics.metric_cache_errors as merrors +import ts.metrics.metric_cache_errors as merrors from ts.metrics.dimension import Dimension from ts.metrics.metric_abstract import MetricAbstract from ts.metrics.metric_type_enum import MetricTypes @@ -27,7 +27,7 @@ def __init__(self, config_file_path): Name of file to be parsed """ - self.cache = dict() + self.cache = {} self.store = [] self.request_ids = None self.model_name = None @@ -35,7 +35,9 @@ def __init__(self, config_file_path): try: os.path.exists(self.config_file_path) except Exception as exc: - raise merrors.MetricsCacheTypeError(f"Error loading {config_file_path} file: {exc}") + raise merrors.MetricsCacheTypeError( + f"Error loading {config_file_path} file: {exc}" + ) def _add_default_dims(self, idx, dimensions): dim_names = [dim.name for dim in dimensions] @@ -63,11 +65,44 @@ def _get_req(self, idx): # check if request id for the metric is given, if so use it else have a list of all. req_id = self.request_ids if isinstance(req_id, dict): - req_id = ','.join(self.request_ids.values()) + req_id = ",".join(self.request_ids.values()) if idx is not None and self.request_ids is not None and idx in self.request_ids: req_id = self.request_ids[idx] return req_id + def add_metric( + self, + name: str, + value: int or float, + unit: str, + idx: str = None, + dimensions: list = [], + metric_type: MetricTypes = MetricTypes.COUNTER, + ): + """ + Add a generic metric + Default metric type is counter + + Parameters + ---------- + name : str + metric name + value: int or float + value of the metric + unit: str + unit of metric + idx: str + request id to be associated with the metric + dimensions: list + list of Dimension objects for the metric + metric_type MetricTypes + Type of metric Counter, Gauge, Histogram + """ + req_id = self._get_req(idx) + dimensions = self._add_default_dims(req_id, dimensions) + metric = self._get_or_add_metric(name, unit, dimensions, metric_type) + metric.add_or_update(value, [dim.value for dim in dimensions], req_id) + def add_counter( self, name: str, @@ -87,7 +122,7 @@ def add_counter( idx: str request id to be associated with the metric dimensions: list - list of dimension names for the metric + list of Dimension objects for the metric """ req_id = self._get_req(idx) dimensions = self._add_default_dims(req_id, dimensions) @@ -118,7 +153,7 @@ def add_time( unit: str unit of metric, default here is ms, s is also accepted dimensions: list - list of dimension names for the metric + list of Dimension objects for the metric metric_type MetricTypes Type of metric Counter, Gauge, Histogram """ @@ -155,7 +190,7 @@ def add_size( unit: str unit of metric, default here is 'MB', 'kB', 'GB' also supported dimensions: list - list of dimensions for the metric + list of Dimension objects for the metric metric_type MetricTypes Type of metric Counter, Gauge, Histogram """ @@ -189,7 +224,7 @@ def add_percent( idx: str request id to be associated with the metric dimensions: list - list of dimensions for the metric + list of Dimension objects for the metric metric_type MetricTypes Type of metric Counter, Gauge, Histogram """ @@ -215,17 +250,19 @@ def add_error( value: int or float value of the metric dimensions: list - list of dimension objects for the metric + list of Dimension objects for the metric """ dimensions = self._add_default_dims(None, dimensions) metric = self._get_or_add_metric(name, "", dimensions, MetricTypes.COUNTER) metric.add_or_update(value, [dim.value for dim in dimensions]) - def _get_or_add_metric(self, metric_name, unit, dimensions, metric_type) -> MetricAbstract: + def _get_or_add_metric( + self, metric_name, unit, dimensions, metric_type + ) -> MetricAbstract: try: metric = self.get_metric(metric_name, metric_type) except merrors.MetricsCacheKeyError: - metric = self.add_metric( + metric = self.add_metric_to_cache( metric_name=metric_name, unit=unit, metric_type=metric_type, @@ -269,11 +306,11 @@ def get_metric( pass @abc.abstractmethod - def add_metric( + def add_metric_to_cache( self, metric_name: str, unit: str, - dimension_names: list = None, + dimension_names: list = [], metric_type: MetricTypes = MetricTypes.COUNTER, ) -> MetricAbstract: """ @@ -287,7 +324,7 @@ def add_metric( unit: str unit of metric dimension_names: list - list of dimensions for the metric + list of dimension name strings for the metric metric_type: MetricTypes Type of metric diff --git a/ts/metrics/metric_cache_yaml_impl.py b/ts/metrics/metric_cache_yaml_impl.py index 0e653db947..7206c83c30 100644 --- a/ts/metrics/metric_cache_yaml_impl.py +++ b/ts/metrics/metric_cache_yaml_impl.py @@ -2,12 +2,14 @@ Metrics Cache class for creating objects from yaml spec """ import logging + import yaml import ts.metrics.metric_cache_errors as merrors from ts.metrics.caching_metric import CachingMetric from ts.metrics.metric_cache_abstract import MetricCacheAbstract from ts.metrics.metric_type_enum import MetricTypes + logger = logging.getLogger(__name__) @@ -34,7 +36,8 @@ def _parse_yaml_file(self, config_file_path) -> None: try: self._parsed_file = yaml.safe_load( - open(config_file_path, "r", encoding="utf-8")) + open(config_file_path, "r", encoding="utf-8") + ) logging.info(f"Successfully loaded {config_file_path}.") except yaml.YAMLError as exc: raise merrors.MetricsCachePyYamlError( @@ -73,7 +76,9 @@ def initialize_cache(self) -> None: """ metrics_section = self._parse_metrics_section("model_metrics") if not metrics_section: - raise merrors.MetricsCacheValueError("Missing `model_metrics` specification") + raise merrors.MetricsCacheValueError( + "Missing `model_metrics` specification" + ) for metric_type, metrics_list in metrics_section.items(): try: metric_enum = MetricTypes(metric_type) @@ -85,16 +90,18 @@ def initialize_cache(self) -> None: metric_name = metric["name"] unit = metric["unit"] dimension_names = metric["dimensions"] - self.add_metric( + self.add_metric_to_cache( metric_name=metric_name, unit=unit, dimension_names=dimension_names, metric_type=metric_enum, ) except KeyError as k_err: - raise merrors.MetricsCacheKeyError(f"Key not found in cache spec: {k_err}") + raise merrors.MetricsCacheKeyError( + f"Key not found in cache spec: {k_err}" + ) - def add_metric( + def add_metric_to_cache( self, metric_name: str, unit: str, @@ -111,7 +118,7 @@ def add_metric( unit str unit can be one of ms, percent, count, MB, GB or a generic string dimension_names list - list of dimension keys which should be strings, or the complete log of dimensions + list of dimension name strings for the metric metric_type MetricTypes Type of metric Counter, Gauge, Histogram Returns @@ -120,16 +127,22 @@ def add_metric( """ self._check_type(metric_name, str, "`metric_name` must be a str") self._check_type(unit, str, "`unit` must be a str") - self._check_type(metric_type, MetricTypes, "`metric_type` must be a MetricTypes enum") + self._check_type( + metric_type, MetricTypes, "`metric_type` must be a MetricTypes enum" + ) if dimension_names: - self._check_type(dimension_names, list, "`dimension_names` should be a list of dimension name strings") + self._check_type( + dimension_names, + list, + "`dimension_names` should be a list of dimension name strings", + ) if metric_type not in self.cache.keys(): - self.cache[metric_type] = dict() + self.cache[metric_type] = {} metric = CachingMetric( metric_name=metric_name, unit=unit, dimension_names=dimension_names, - metric_type=metric_type + metric_type=metric_type, ) if metric_name in self.cache[metric_type].keys(): logging.warning(f"Overriding existing key {metric_type}:{metric_name}") @@ -157,7 +170,9 @@ def get_metric( Metrics object or MetricsCacheKeyError if not found """ self._check_type(metric_name, str, "`metric_name` must be a str") - self._check_type(metric_type, MetricTypes, "`metric_type` must be a MetricTypes enum") + self._check_type( + metric_type, MetricTypes, "`metric_type` must be a MetricTypes enum" + ) try: metric = self.cache[metric_type][metric_name] except KeyError: diff --git a/ts/tests/unit_tests/metrics_yaml_testing/metric_cache_unit_test.py b/ts/tests/unit_tests/metrics_yaml_testing/metric_cache_unit_test.py index 82fff1b8d5..109b0eedb0 100644 --- a/ts/tests/unit_tests/metrics_yaml_testing/metric_cache_unit_test.py +++ b/ts/tests/unit_tests/metrics_yaml_testing/metric_cache_unit_test.py @@ -3,12 +3,13 @@ ts/metrics/metric_cache_yaml_impl.py, and emit_metrics() ts/service.py """ import os -import pytest import uuid -import ts.metrics.metric_cache_errors as merrors -from ts.metrics.dimension import Dimension +import pytest + +import ts.metrics.metric_cache_errors as merrors from ts.metrics.caching_metric import CachingMetric +from ts.metrics.dimension import Dimension from ts.metrics.metric_cache_yaml_impl import MetricsCacheYamlImpl from ts.metrics.metric_type_enum import MetricTypes @@ -16,45 +17,55 @@ class TestAddMetrics: - def test_add_metric_passing(self): + def test_add_metric_to_cache_passing(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( - metric_name="test_add_metric_passing", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_passing", unit="ms", dimension_names=["ModelName", "Host"], metric_type=MetricTypes.GAUGE, ) assert MetricTypes.GAUGE in metrics_cache_obj.cache.keys() - assert "test_add_metric_passing" in metrics_cache_obj.cache[MetricTypes.GAUGE].keys() + assert ( + "test_add_metric_to_cache_passing" + in metrics_cache_obj.cache[MetricTypes.GAUGE].keys() + ) - def test_add_metric_duplicate_passing(self, caplog): + def test_add_metric_to_cache_duplicate_passing(self, caplog): caplog.set_level("INFO") metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( - metric_name="test_add_metric_duplicate_passing", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_duplicate_passing", unit="ms", dimension_names=["ModelName"], metric_type=MetricTypes.GAUGE, ) assert MetricTypes.GAUGE in metrics_cache_obj.cache.keys() - assert "test_add_metric_duplicate_passing" in metrics_cache_obj.cache[MetricTypes.GAUGE].keys() - metric = metrics_cache_obj.get_metric("test_add_metric_duplicate_passing", MetricTypes.GAUGE) + assert ( + "test_add_metric_to_cache_duplicate_passing" + in metrics_cache_obj.cache[MetricTypes.GAUGE].keys() + ) + metric = metrics_cache_obj.get_metric( + "test_add_metric_to_cache_duplicate_passing", MetricTypes.GAUGE + ) metric.add_or_update(123.5, ["dummy_model"]) assert "123.5" in caplog.text - metrics_cache_obj.add_metric( - metric_name="test_add_metric_duplicate_passing", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_duplicate_passing", unit="ms", dimension_names=["ModelName"], metric_type=MetricTypes.GAUGE, ) - metric = metrics_cache_obj.get_metric("test_add_metric_duplicate_passing", MetricTypes.GAUGE) + metric = metrics_cache_obj.get_metric( + "test_add_metric_to_cache_duplicate_passing", MetricTypes.GAUGE + ) metric.add_or_update(42.5, ["dummy_model"]) assert "42.5" in caplog.text - def test_add_metric_fail_metric_name(self): + def test_add_metric_to_cache_fail_metric_name(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( metric_name=42, unit="ms", dimension_names=["ModelName", "Host"], @@ -62,43 +73,46 @@ def test_add_metric_fail_metric_name(self): ) assert str(exc_info.value) == "`metric_name` must be a str" - def test_add_metric_fail_unit(self): + def test_add_metric_to_cache_fail_unit(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: - metrics_cache_obj.add_metric( - metric_name="test_add_metric_fail_unit", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_fail_unit", unit=["foo"], dimension_names=["ModelName", "Host"], metric_type=MetricTypes.GAUGE, ) assert str(exc_info.value) == "`unit` must be a str" - def test_add_metric_fail_dimensions(self): + def test_add_metric_to_cache_fail_dimensions(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: - metrics_cache_obj.add_metric( - metric_name="test_add_metric_fail_dimensions", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_fail_dimensions", unit="ms", dimension_names="ModelName", metric_type=MetricTypes.GAUGE, ) - assert str(exc_info.value) == "`dimension_names` should be a list of dimension name strings" + assert ( + str(exc_info.value) + == "`dimension_names` should be a list of dimension name strings" + ) - def test_add_metric_fail_type(self): + def test_add_metric_to_cache_fail_type(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: - metrics_cache_obj.add_metric( - metric_name="test_add_metric_fail_type", + metrics_cache_obj.add_metric_to_cache( + metric_name="test_add_metric_to_cache_fail_type", unit="ms", dimension_names=["ModelName", "Host"], metric_type={"key": 42}, ) assert str(exc_info.value) == "`metric_type` must be a MetricTypes enum" - def test_add_metric_fail_type_unit(self): + def test_add_metric_to_cache_fail_type_unit(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( metric_name="bar", unit=["ms"], dimension_names=["model_name", "host"], @@ -106,11 +120,29 @@ def test_add_metric_fail_type_unit(self): ) assert str(exc_info.value) == "`unit` must be a str" + def test_add_metric_passing(self): + metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) + metrics_cache_obj.add_metric( + name="test_add_metric_passing", + value=1, + unit="ms", + dimensions=[ + Dimension("ModelName", "testModel"), + Dimension("Host", "testHost"), + ], + metric_type=MetricTypes.GAUGE, + ) + assert MetricTypes.GAUGE in metrics_cache_obj.cache.keys() + assert ( + "test_add_metric_passing" + in metrics_cache_obj.cache[MetricTypes.GAUGE].keys() + ) + class TestGetMetrics: def test_get_metric_passing(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( metric_name="test_get_metric_passing", unit="ms", dimension_names=["ModelName", "Host"], @@ -126,18 +158,19 @@ def test_get_metric_passing(self): def test_get_metric_invalid_metric_type(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( metric_name="test_get_metric_invalid_metric_type", unit="ms", dimension_names=["ModelName", "Host"], metric_type=MetricTypes.GAUGE, ) with pytest.raises(merrors.MetricsCacheKeyError) as exc_info: - metrics_cache_obj.get_metric("test_get_metric_invalid_metric_type", MetricTypes.COUNTER) + metrics_cache_obj.get_metric( + "test_get_metric_invalid_metric_type", MetricTypes.COUNTER + ) assert ( - str(exc_info.value) - == '"Metric of type \'MetricTypes.COUNTER\' and ' - 'name \'test_get_metric_invalid_metric_type\' doesn\'t exist"' + str(exc_info.value) == "\"Metric of type 'MetricTypes.COUNTER' and " + "name 'test_get_metric_invalid_metric_type' doesn't exist\"" ) def test_get_metric_fail_invalid_type(self): @@ -158,70 +191,74 @@ def test_parse_expected_yaml(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) expected_dict = { "mode": "prometheus", - "dimensions": - [ - "model_name", - "host_name", - "host" - ], - "ts_metrics": - { + "dimensions": ["model_name", "host_name", "host"], + "ts_metrics": { "counter": [ - { - "name": "CounterTsMetricExample", - "unit": "ms", - "dimensions": ["model_name", "host_name"] - }], + { + "name": "CounterTsMetricExample", + "unit": "ms", + "dimensions": ["model_name", "host_name"], + } + ], "gauge": [ - { - "name": "GaugeTsMetricExample", - "unit": "ms", - "dimensions": ["model_name", "host_name"] - }], + { + "name": "GaugeTsMetricExample", + "unit": "ms", + "dimensions": ["model_name", "host_name"], + } + ], "histogram": [ - { - "name": "HistogramTsMetricExample", - "unit": "ms", - "dimensions": ["model_name", "host_name"] - }] + { + "name": "HistogramTsMetricExample", + "unit": "ms", + "dimensions": ["model_name", "host_name"], + } + ], }, - "model_metrics": - { + "model_metrics": { "counter": [ - { - "name": "InferenceTimeInMS", - "unit": "ms", - "dimensions": ["model_name", "host"] - }, - { - "name": "NumberOfMetrics", - "unit": "count", - "dimensions": ["model_name", "host_name"] - }], + { + "name": "InferenceTimeInMS", + "unit": "ms", + "dimensions": ["model_name", "host"], + }, + { + "name": "NumberOfMetrics", + "unit": "count", + "dimensions": ["model_name", "host_name"], + }, + ], "gauge": [ - { - "name": "GaugeModelMetricNameExample", - "unit": "ms", - "dimensions": ["model_name", "host"] - }], + { + "name": "GaugeModelMetricNameExample", + "unit": "ms", + "dimensions": ["model_name", "host"], + } + ], "histogram": [ - { - "name": "HistogramModelMetricNameExample", - "unit": "ms", - "dimensions": ["model_name", "host"] - }] - } + { + "name": "HistogramModelMetricNameExample", + "unit": "ms", + "dimensions": ["model_name", "host"], + } + ], + }, } assert expected_dict == metrics_cache_obj._parsed_file def test_yaml_file_passing(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - assert metrics_cache_obj.config_file_path == os.path.join(dir_path, "metrics.yaml") + assert metrics_cache_obj.config_file_path == os.path.join( + dir_path, "metrics.yaml" + ) def test_yaml_file_none_fail(self): with pytest.raises(merrors.MetricsCacheTypeError) as exc_info: MetricsCacheYamlImpl(None) - assert "stat: path should be string, bytes, os.PathLike or integer, not NoneType" in str(exc_info.value) + assert ( + "stat: path should be string, bytes, os.PathLike or integer, not NoneType" + in str(exc_info.value) + ) def test_yaml_file_non_yaml_extension_fail(self): with pytest.raises(merrors.MetricsCachePyYamlError) as exc_info: @@ -244,7 +281,9 @@ def test_pass_parse_ts_metrics(self): assert isinstance(metrics_cache_obj._parse_metrics_section("ts_metrics"), dict) def test_fail_parse_model_metrics(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_wo_model_metrics.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_wo_model_metrics.yaml") + ) with pytest.raises(merrors.MetricsCacheKeyError) as exc_info: metrics_cache_obj._parse_metrics_section() assert ( @@ -254,7 +293,9 @@ def test_fail_parse_model_metrics(self): ) def test_fail_parse_none(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_wo_model_metrics.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_wo_model_metrics.yaml") + ) with pytest.raises(merrors.MetricsCacheKeyError) as exc_info: metrics_cache_obj._parse_metrics_section(None) assert ( @@ -264,7 +305,9 @@ def test_fail_parse_none(self): ) def test_fail_parse_missing_fields(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_model_empty.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_model_empty.yaml") + ) assert metrics_cache_obj._parse_metrics_section() is None @@ -272,23 +315,25 @@ class TestYamlCacheInit: def test_yaml_to_cache_util_pass(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) metrics_cache_obj.initialize_cache() - assert(len(metrics_cache_obj.cache.keys()) == 3) + assert len(metrics_cache_obj.cache.keys()) == 3 assert metrics_cache_obj.cache_keys() == [ "counter:InferenceTimeInMS", "counter:NumberOfMetrics", "gauge:GaugeModelMetricNameExample", - "histogram:HistogramModelMetricNameExample" + "histogram:HistogramModelMetricNameExample", ] def test_yaml_to_cache_empty_dims(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_empty_dims.yml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_empty_dims.yml") + ) metrics_cache_obj.initialize_cache() assert metrics_cache_obj.cache_keys() == [ "counter:InferenceTimeInMS", "counter:NumberOfMetrics", "gauge:GaugeModelMetricNameExample", "histogram:HistogramModelMetricNameExample", - "histogram:AnotherHistogram" + "histogram:AnotherHistogram", ] for metric_type, metric in metrics_cache_obj.cache.items(): for k, v in metric.items(): @@ -300,7 +345,9 @@ def test_yaml_to_cache_empty_dims(self): assert v.dimension_names == [] def test_yaml_to_cache_util_fail_missing_fields(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_missing_types.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_missing_types.yaml") + ) with pytest.raises(merrors.MetricsCacheKeyError) as exc_info: metrics_cache_obj.initialize_cache() assert ( @@ -309,7 +356,9 @@ def test_yaml_to_cache_util_fail_missing_fields(self): ) def test_yaml_to_cache_util_fail_empty_section(self): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_model_empty.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_model_empty.yaml") + ) with pytest.raises(merrors.MetricsCacheValueError) as exc_info: metrics_cache_obj.initialize_cache() assert ( @@ -321,7 +370,7 @@ def test_yaml_to_cache_util_fail_empty_section(self): class TestManualAddMetricDimensions: def test_dimensions_metric_add_dimensions_custom_pass(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( "TempName", "count", dimension_names=["MyDim", "MyDimValue"], @@ -336,7 +385,7 @@ def test_dimensions_metric_add_dimensions_custom_pass(self): def test_dimensions_metric_add_dimensions_and_yaml_pass(self): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) - metrics_cache_obj.add_metric( + metrics_cache_obj.add_metric_to_cache( "TempName", "count", dimension_names=["ModelName"], @@ -348,16 +397,48 @@ def test_dimensions_metric_add_dimensions_and_yaml_pass(self): "counter:InferenceTimeInMS", "counter:NumberOfMetrics", "gauge:GaugeModelMetricNameExample", - "histogram:HistogramModelMetricNameExample" + "histogram:HistogramModelMetricNameExample", ] class TestAdditionalMetricMethods: + def test_add_metric_pass(self, caplog): + metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) + caplog.set_level("INFO") + dimensions = [Dimension("ModelName", "test_add_metric_pass")] + metrics_cache_obj.add_metric( + "CustomMetric", 25, unit="GB", dimensions=dimensions + ) + metric = metrics_cache_obj.get_metric("CustomMetric", MetricTypes.COUNTER) + assert metric.dimension_names == ["ModelName", "Level"] + assert ( + "[METRICS]CustomMetric.Gigabytes:25|#ModelName:test_add_metric_pass,Level:Error" + in caplog.text + ) + + def test_add_metric_non_default_type_pass(self, caplog): + metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) + caplog.set_level("INFO") + dimensions = [Dimension("ModelName", "test_add_metric_non_default_type_pass")] + metrics_cache_obj.add_metric( + "CustomMetric", + 25, + unit="GB", + dimensions=dimensions, + metric_type=MetricTypes.GAUGE, + ) + metric = metrics_cache_obj.get_metric("CustomMetric", MetricTypes.GAUGE) + assert metric.dimension_names == ["ModelName", "Level"] + assert ( + "[METRICS]CustomMetric.Gigabytes:25|#ModelName:test_add_metric_non_default_type_pass,Level:Error" + in caplog.text + ) + def test_add_counter_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) dimensions = [ Dimension("ModelName", "test_add_counter_pass"), - Dimension("Level", "Model") + Dimension("Level", "Model"), ] caplog.set_level("INFO") metrics_cache_obj.add_counter("CounterMetric", 14, dimensions=dimensions) @@ -365,7 +446,10 @@ def test_add_counter_pass(self, caplog): assert metric.dimension_names == ["ModelName", "Level"] assert metric.metric_name == "CounterMetric" assert metric.metric_type == MetricTypes.COUNTER - assert "[METRICS]CounterMetric.Count:14|#ModelName:test_add_counter_pass" in caplog.text + assert ( + "[METRICS]CounterMetric.Count:14|#ModelName:test_add_counter_pass" + in caplog.text + ) def test_add_time_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) @@ -375,13 +459,21 @@ def test_add_time_pass(self, caplog): metric = metrics_cache_obj.get_metric("TimeMetric", MetricTypes.GAUGE) assert metric.metric_name == "TimeMetric" assert metric.metric_type == MetricTypes.GAUGE - assert "[METRICS]TimeMetric.Seconds:17|#ModelName:test_add_time_pass" in caplog.text + assert ( + "[METRICS]TimeMetric.Seconds:17|#ModelName:test_add_time_pass" + in caplog.text + ) def test_add_time_diff_type_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") - metrics_cache_obj.add_time("TimeMetric", 14, unit="ms", - dimensions=[Dimension("Level", "Model")], metric_type=MetricTypes.HISTOGRAM) + metrics_cache_obj.add_time( + "TimeMetric", + 14, + unit="ms", + dimensions=[Dimension("Level", "Model")], + metric_type=MetricTypes.HISTOGRAM, + ) metric = metrics_cache_obj.get_metric("TimeMetric", MetricTypes.HISTOGRAM) assert metric.metric_name == "TimeMetric" assert metric.metric_type == MetricTypes.HISTOGRAM @@ -397,15 +489,27 @@ def test_add_size_pass(self, caplog): metrics_cache_obj.add_size("SizeMetric", 25, unit="GB", dimensions=dimensions) metric = metrics_cache_obj.get_metric("SizeMetric", MetricTypes.GAUGE) assert metric.dimension_names == ["ModelName", "Level"] - assert "[METRICS]SizeMetric.Gigabytes:25|#ModelName:test_add_size_pass,Level:Error" in caplog.text + assert ( + "[METRICS]SizeMetric.Gigabytes:25|#ModelName:test_add_size_pass,Level:Error" + in caplog.text + ) def test_add_size_diff_metric_type_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") dimensions = [Dimension("Level", "test_add_size_diff_metric_type_pass")] - metrics_cache_obj.add_size("SizeMetric", 5, unit="kB", dimensions=dimensions, metric_type=MetricTypes.COUNTER) + metrics_cache_obj.add_size( + "SizeMetric", + 5, + unit="kB", + dimensions=dimensions, + metric_type=MetricTypes.COUNTER, + ) metric = metrics_cache_obj.get_metric("SizeMetric", MetricTypes.COUNTER) - assert "[METRICS]SizeMetric.Kilobytes:5|#Level:test_add_size_diff_metric_type_pass" in caplog.text + assert ( + "[METRICS]SizeMetric.Kilobytes:5|#Level:test_add_size_diff_metric_type_pass" + in caplog.text + ) def test_add_percent_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) @@ -416,24 +520,36 @@ def test_add_percent_pass(self, caplog): request_id_map = {1: uid1, 2: uid2} metrics_cache_obj.set_request_ids(request_id_map) metrics_cache_obj.add_percent("PercentMetric", 11, uid1, dimensions) - assert "[METRICS]PercentMetric.Percent:11|#ModelName:test_add_percent_pass,Level:Model|" in caplog.text + assert ( + "[METRICS]PercentMetric.Percent:11|#ModelName:test_add_percent_pass,Level:Model|" + in caplog.text + ) assert str(uid1) in caplog.text metric = metrics_cache_obj.get_metric("PercentMetric", MetricTypes.GAUGE) metric.update(22, request_id=uid2, dimensions=dimensions) - assert "[METRICS]PercentMetric.Percent:22|#ModelName:test_add_percent_pass,Level:Model|" in caplog.text + assert ( + "[METRICS]PercentMetric.Percent:22|#ModelName:test_add_percent_pass,Level:Model|" + in caplog.text + ) assert str(uid2) in caplog.text def test_add_percent_diff_metric_type_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") - metrics_cache_obj.add_percent("PercentMetric", 72, - dimensions=[Dimension("Host", "hostname")], metric_type=MetricTypes.HISTOGRAM) + metrics_cache_obj.add_percent( + "PercentMetric", + 72, + dimensions=[Dimension("Host", "hostname")], + metric_type=MetricTypes.HISTOGRAM, + ) assert "[METRICS]PercentMetric.Percent:72|#Host:hostname" in caplog.text def test_add_error_pass(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") - metrics_cache_obj.add_error("ErrorName", 72, dimensions=[Dimension("ModelName", "hostname")]) + metrics_cache_obj.add_error( + "ErrorName", 72, dimensions=[Dimension("ModelName", "hostname")] + ) assert "[METRICS]ErrorName.unit:72|#ModelName:hostname" in caplog.text @@ -441,78 +557,129 @@ class TestIncrementDecrementMetrics: def test_add_counter_dimensions_empty(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") - metrics_cache_obj.add_counter("LoopCount", 7, uuid.uuid4(), [ - Dimension("ModelName", "test_add_counter_dimensions_empty") - ]) + metrics_cache_obj.add_counter( + "LoopCount", + 7, + uuid.uuid4(), + [Dimension("ModelName", "test_add_counter_dimensions_empty")], + ) counter_metric = metrics_cache_obj.get_metric("LoopCount", MetricTypes.COUNTER) counter_metric.add_or_update(14, ["test_add_counter_dimensions_empty", "Host"]) - assert "LoopCount.Count:7|#ModelName:test_add_counter_dimensions_empty,Level:Error|" in caplog.text - assert "LoopCount.Count:14|#ModelName:test_add_counter_dimensions_empty,Level:Host|" in caplog.text + assert ( + "LoopCount.Count:7|#ModelName:test_add_counter_dimensions_empty,Level:Error|" + in caplog.text + ) + assert ( + "LoopCount.Count:14|#ModelName:test_add_counter_dimensions_empty,Level:Host|" + in caplog.text + ) def test_add_counter_dimensions_filled(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") dimensions = [ Dimension("ModelName", "model_name_a"), - Dimension("Level", "level_a") + Dimension("Level", "level_a"), ] metrics_cache_obj.add_counter("LoopCount", 71, dimensions=dimensions) metric = metrics_cache_obj.get_metric("LoopCount", MetricTypes.COUNTER) metric.add_or_update(19, ["model_name_b", "level_b"]) - assert "LoopCount.Count:71|#ModelName:model_name_a,Level:level_a|" in caplog.text - assert "LoopCount.Count:19|#ModelName:model_name_b,Level:level_b|" in caplog.text + assert ( + "LoopCount.Count:71|#ModelName:model_name_a,Level:level_a|" in caplog.text + ) + assert ( + "LoopCount.Count:19|#ModelName:model_name_b,Level:level_b|" in caplog.text + ) def test_add_error_dimensions_filled(self, caplog): metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics.yaml")) caplog.set_level("INFO") dimensions = [ Dimension("ModelName", "model_name_a"), - Dimension("Level", "level_a") + Dimension("Level", "level_a"), ] metrics_cache_obj.add_error("LoopCountError", 2, dimensions=dimensions) metric = metrics_cache_obj.get_metric("LoopCountError", MetricTypes.COUNTER) metric.add_or_update(4, ["model_name_b", "level_b"]) - assert "LoopCountError.unit:2|#ModelName:model_name_a,Level:level_a|" in caplog.text - assert "LoopCountError.unit:4|#ModelName:model_name_b,Level:level_b|" in caplog.text + assert ( + "LoopCountError.unit:2|#ModelName:model_name_a,Level:level_a|" + in caplog.text + ) + assert ( + "LoopCountError.unit:4|#ModelName:model_name_b,Level:level_b|" + in caplog.text + ) class TestAPIAndYamlParse: def test_yaml_then_api(self, caplog): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_api.yaml")) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_api.yaml") + ) metrics_cache_obj.initialize_cache() - metrics_cache_obj.add_metric("InferenceTimeInMS", "ms", ["ModelName"]) - metrics_cache_obj.add_metric("GaugeModelMetricNameExample", "MB", ["Level"], MetricTypes.GAUGE) - metrics_cache_obj.add_metric("LoopCountError", "") + metrics_cache_obj.add_metric_to_cache("InferenceTimeInMS", "ms", ["ModelName"]) + metrics_cache_obj.add_metric_to_cache( + "GaugeModelMetricNameExample", "MB", ["Level"], MetricTypes.GAUGE + ) + metrics_cache_obj.add_metric_to_cache("LoopCountError", "") print(metrics_cache_obj.cache_keys()) assert len(metrics_cache_obj.cache_keys()) == 5 - counter_metric = metrics_cache_obj.get_metric("InferenceTimeInMS", MetricTypes.COUNTER) - gauge_metric = metrics_cache_obj.get_metric("GaugeModelMetricNameExample", MetricTypes.GAUGE) - error_metric = metrics_cache_obj.get_metric("LoopCountError", MetricTypes.COUNTER) + counter_metric = metrics_cache_obj.get_metric( + "InferenceTimeInMS", MetricTypes.COUNTER + ) + gauge_metric = metrics_cache_obj.get_metric( + "GaugeModelMetricNameExample", MetricTypes.GAUGE + ) + error_metric = metrics_cache_obj.get_metric( + "LoopCountError", MetricTypes.COUNTER + ) caplog.set_level("INFO") counter_metric.add_or_update(2.7, ["test_yaml_then_api"]) gauge_metric.add_or_update(25.17, ["model"]) error_metric.add_or_update(5) - assert "InferenceTimeInMS.Milliseconds:2.7|#ModelName:test_yaml_then_api|" in caplog.text - assert "GaugeModelMetricNameExample.Megabytes:25.17|#Level:model|" in caplog.text + assert ( + "InferenceTimeInMS.Milliseconds:2.7|#ModelName:test_yaml_then_api|" + in caplog.text + ) + assert ( + "GaugeModelMetricNameExample.Megabytes:25.17|#Level:model|" in caplog.text + ) assert "LoopCountError.unit:5|#|" in caplog.text def test_api_then_yaml(self, caplog): - metrics_cache_obj = MetricsCacheYamlImpl(os.path.join(dir_path, "metrics_api.yaml")) - metrics_cache_obj.add_metric("InferenceTimeInMS", "count", ["ModelName"]) - metrics_cache_obj.add_metric("GaugeModelMetricNameExample", "MB", ["Level"], MetricTypes.GAUGE) - counter_metric = metrics_cache_obj.get_metric("InferenceTimeInMS", MetricTypes.COUNTER) + metrics_cache_obj = MetricsCacheYamlImpl( + os.path.join(dir_path, "metrics_api.yaml") + ) + metrics_cache_obj.add_metric_to_cache( + "InferenceTimeInMS", "count", ["ModelName"] + ) + metrics_cache_obj.add_metric_to_cache( + "GaugeModelMetricNameExample", "MB", ["Level"], MetricTypes.GAUGE + ) + counter_metric = metrics_cache_obj.get_metric( + "InferenceTimeInMS", MetricTypes.COUNTER + ) caplog.set_level("INFO") counter_metric.add_or_update(24.7, ["test_api_then_yaml"]) - assert "InferenceTimeInMS.Count:24.7|#ModelName:test_api_then_yaml|" in caplog.text + assert ( + "InferenceTimeInMS.Count:24.7|#ModelName:test_api_then_yaml|" in caplog.text + ) counter_metric.add_or_update(42.5, ["updated"]) assert "InferenceTimeInMS.Count:42.5|#ModelName:updated|" in caplog.text metrics_cache_obj.initialize_cache() assert len(metrics_cache_obj.cache_keys()) == 4 - counter_metric = metrics_cache_obj.get_metric("InferenceTimeInMS", MetricTypes.COUNTER) - spec_metric = metrics_cache_obj.get_metric("NumberOfMetrics", MetricTypes.COUNTER) + counter_metric = metrics_cache_obj.get_metric( + "InferenceTimeInMS", MetricTypes.COUNTER + ) + spec_metric = metrics_cache_obj.get_metric( + "NumberOfMetrics", MetricTypes.COUNTER + ) counter_metric.add_or_update(4.1, ["test_api_then_yaml", "model"]) spec_metric.add_or_update(2, ["test_api_then_yaml"]) - assert "InferenceTimeInMS.Milliseconds:4.1|#model_name:test_api_then_yaml,level:model|" in caplog.text + assert ( + "InferenceTimeInMS.Milliseconds:4.1|#model_name:test_api_then_yaml,level:model|" + in caplog.text + ) assert "NumberOfMetrics.Count:2|#model_name:test_api_then_yaml" in caplog.text