[V2 Logger] Feature branch (#1516) (#1614)

* [V2 Logger] Feature branch (#1516)

* logger feature beanch

* [V2 Logger]  utils for import (#1536)

* utils for import

* clean up + tests

* [V2 Logger] registry for default func/class (#1539)

* registry for default func/class

* prometheus logger

* clean up

* clean up

* remove ListLogger import, moved to tests

* add base logger for type checking

* exmaple

* [V2 Logger] Filters (#1540)

* filters

* cleaner logic

* fix bug -- if config has duplicated tag.func with different freq

* delete config test

* move unravel_value_as_generator to pattern.py

* tests

* doc string

* exact match

* async submitter (#1538)

* manager (#1541)

* [V2 Loggers] logger manager patch (#1560)

* manager

* doc string

* [V2 Loggers] config file (#1533)

* config file

* test

* comments

* comments

* pipeline tests (#1553)

* [V2 Logger] root logger (#1542)

* root logger

* only get func's with respect to the tag

* fix bug for duplicated tag.log_type

* clena up

* doc strings:

* loosen up rules for capture

* [V2 Logger]  factory (#1537)

* factory

* docstring

* [V2 Logger] logger middleware (#1543)

* logger middleware

* yield individual eleeents in a list

* comments

* edit state

* polish, passes tests

* pass middleware

* edit condition to add logger to inference state

* set default logger manager

* delete og prometheus logger test

* fix in test_basic_logger

* move loggers to legacy and pass tests, circular imports

* move tests/deepsparse/loggers to tests/deepsparse/legacy/loggers and pass tests

* move loggers_v2 to logger for src and tests, pass logger tests

* fix tests and rename legacy logger tests to _legacy_

* pass tests, wait for async logs to complete'

* doc string typo and change default to re:.*

* fix frequency test bug on text gen

* wait for async loggers to finish before counting

* get rid of capture, inconsistent number of fields per log calls cause error

* fix src. reference (#1607)

---------

Co-authored-by: Benjamin Fineran <bfineran@users.noreply.github.com>

src/deepsparse/__init__.py

@@ -34,7 +34,6 @@
 from .pipeline_config import *
 from .tasks import *
 from .pipeline import *
-from .loggers import *
 from .version import __version__, is_release
 from .analytics import deepsparse_analytics as _analytics
 from .subgraph_execute import *

src/deepsparse/legacy/base_pipeline.py

@@ -19,10 +19,10 @@
 from pydantic import BaseModel
 
 from deepsparse import Context
+from deepsparse.legacy.loggers.base_logger import BaseLogger
+from deepsparse.legacy.loggers.build_logger import logger_from_config
+from deepsparse.legacy.loggers.constants import validate_identifier
 from deepsparse.legacy.tasks import SupportedTasks, dynamic_import_task
-from deepsparse.loggers.base_logger import BaseLogger
-from deepsparse.loggers.build_logger import logger_from_config
-from deepsparse.loggers.constants import validate_identifier
 
 
 __all__ = [

src/deepsparse/legacy/loggers/__init__.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# flake8: noqa
+# isort: skip_file
+
+# base modules
+from .base_logger import *
+from .constants import *
+
+
+# logger implementations
+from .async_logger import *
+from .function_logger import *
+from .multi_logger import *
+from .prometheus_logger import *
+from .python_logger import *
+
+# functions for creating complex loggers
+from .build_logger import *

src/deepsparse/loggers/async_logger.py → src/deepsparse/legacy/loggers/async_logger.py

@@ -21,7 +21,7 @@
 from concurrent.futures import Executor, ThreadPoolExecutor
 from typing import Any
 
-from deepsparse.loggers import BaseLogger, MetricCategories
+from deepsparse.legacy.loggers import BaseLogger, MetricCategories
 
 
 __all__ = ["AsyncLogger"]

src/deepsparse/loggers/build_logger.py → src/deepsparse/legacy/loggers/build_logger.py

@@ -25,7 +25,7 @@
 
 import yaml
 
-from deepsparse.loggers import (
+from deepsparse.legacy.loggers import (
     FROM_PREDEFINED,
     AsyncLogger,
     BaseLogger,
@@ -34,14 +34,14 @@
     PrometheusLogger,
     PythonLogger,
 )
-from deepsparse.loggers.config import (
+from deepsparse.legacy.loggers.config import (
     MetricFunctionConfig,
     PipelineLoggingConfig,
     SystemLoggingConfig,
     SystemLoggingGroup,
 )
-from deepsparse.loggers.helpers import get_function_and_function_name
-from deepsparse.loggers.metric_functions.registry import DATA_LOGGING_REGISTRY
+from deepsparse.legacy.loggers.helpers import get_function_and_function_name
+from deepsparse.legacy.loggers.metric_functions.registry import DATA_LOGGING_REGISTRY
 
 
 __all__ = [

src/deepsparse/legacy/loggers/config.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, validator
+
+
+"""
+Implements schemas for the configs pertaining to logging
+"""
+
+__all__ = [
+    "MetricFunctionConfig",
+    "SystemLoggingGroup",
+    "SystemLoggingConfig",
+    "PipelineLoggingConfig",
+]
+
+
+class MetricFunctionConfig(BaseModel):
+    """
+    Holds logging configuration for a metric function
+    """
+
+    func: str = Field(
+        description="The name that specifies the metric function to be applied. "
+        "It can be: "
+        "1) a built-in function name "
+        "2) a dynamic import function of the form "
+        "'<path_to_the_python_script>:<function_name>' "
+        "3) a framework function (e.g. np.mean or torch.mean)"
+    )
+
+    frequency: int = Field(
+        description="Specifies how often the function should be applied"
+        "(measured in numbers of inference calls).",
+        default=1,
+    )
+
+    target_loggers: List[str] = Field(
+        default=[],
+        description="Overrides the global logger configuration."
+        "If not an empty list, this configuration stops logging data "
+        "to globally specified loggers, and will only use "
+        "the subset of loggers (specified here by a list of their names).",
+    )
+
+    @validator("frequency")
+    def non_zero_frequency(cls, frequency: int) -> int:
+        if frequency <= 0:
+            raise ValueError(
+                f"Passed frequency: {frequency}, but "
+                "frequency must be a positive integer greater equal 1"
+            )
+        return frequency
+
+
+class SystemLoggingGroup(BaseModel):
+    """
+    Holds the configuration for a single system logging group.
+    """
+
+    enable: bool = Field(
+        default=False,
+        description="Whether to enable the system logging group. Defaults to False",
+    )
+
+    target_loggers: List[str] = Field(
+        default=[],
+        description="The list of target loggers to log to. "
+        "If None, logs to all the available loggers",
+    )
+
+
+class SystemLoggingConfig(BaseModel):
+    # Global Logging Config
+    enable: bool = Field(
+        default=True, description="Whether to enable system logging. Defaults to True"
+    )
+
+
+class PipelineSystemLoggingConfig(SystemLoggingConfig):
+    """
+    Holds the configuration for the system logging
+    in the context of a single pipeline
+    """
+
+    # Pipeline System Logging Groups
+    inference_details: SystemLoggingGroup = Field(
+        default=SystemLoggingGroup(enable=False),
+        description="The configuration group for the inference details "
+        "logging group. By default this group is disabled.",
+    )
+    prediction_latency: SystemLoggingGroup = Field(
+        default=SystemLoggingGroup(enable=True),
+        description="The configuration group for the prediction latency "
+        "logging group. By default this group is enabled.",
+    )
+
+
+class PipelineLoggingConfig(BaseModel):
+    """
+    Holds the complete configuration for the logging
+    in the context of a single pipeline
+    """
+
+    loggers: Dict[str, Optional[Dict[str, Any]]] = Field(
+        default={},
+        description=(
+            "Optional dictionary of logger integration names to initialization kwargs."
+            "Set to {} for no loggers. Default is {}."
+        ),
+    )
+
+    system_logging: PipelineSystemLoggingConfig = Field(
+        default=PipelineSystemLoggingConfig(),
+        description="A model that holds the system logging configuration. "
+        "If not specified explicitly in the yaml config, the "
+        "default SystemLoggingConfig model is used.",
+    )
+
+    data_logging: Optional[Dict[str, List[MetricFunctionConfig]]] = Field(
+        default=None,
+        description="Specifies the rules for the data logging. "
+        "It relates a key (name of the logging target) "
+        "to a list of metric functions that are to be applied"
+        "to this target prior to logging.",
+    )

src/deepsparse/legacy/loggers/constants.py

@@ -0,0 +1,70 @@
+"""
+Holds logging-related objects with constant values
+"""
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+__all__ = [
+    "MetricCategories",
+    "validate_identifier",
+    "SystemGroups",
+    "FROM_PREDEFINED",
+]
+
+UNSUPPORTED_IDENTIFIER_CHARS = {".", "[", "]"}
+FROM_PREDEFINED = "predefined"
+
+
+class MetricCategories(Enum):
+    """
+    Metric Taxonomy [for reference]
+        CATEGORY - category of metric (System/Data)
+            GROUP - logical group of metrics
+                METRIC - individual metric
+    """
+
+    # Categories
+    SYSTEM = "system"
+    DATA = "data"
+
+
+@dataclass(frozen=True)
+class SystemGroups:
+    # Pipeline System Groups
+    INFERENCE_DETAILS: str = "inference_details"
+    PREDICTION_LATENCY: str = "prediction_latency"
+    # Server System Groups
+    REQUEST_DETAILS: str = "request_details"
+    RESOURCE_UTILIZATION: str = "resource_utilization"
+
+
+def validate_identifier(identifier: str):
+    """
+    Makes sure that the identifier does not contain any
+    of the characters that would introduce ambiguity
+    when parsing the identifier
+
+    :param identifier: a string that is used
+        to identify a log
+    """
+    for char in UNSUPPORTED_IDENTIFIER_CHARS:
+        if char in identifier:
+            raise ValueError(
+                f"Logging identifier: {identifier} "
+                f"contains unsupported character {char}"
+            )

src/deepsparse/loggers/function_logger.py → src/deepsparse/legacy/loggers/function_logger.py

@@ -18,8 +18,12 @@
 import textwrap
 from typing import Any, Callable
 
-from deepsparse.loggers import BaseLogger, MetricCategories
-from deepsparse.loggers.helpers import NO_MATCH, finalize_identifier, match_and_extract
+from deepsparse.legacy.loggers import BaseLogger, MetricCategories
+from deepsparse.legacy.loggers.helpers import (
+    NO_MATCH,
+    finalize_identifier,
+    match_and_extract,
+)
 
 
 __all__ = ["FunctionLogger"]

src/deepsparse/loggers/helpers.py → src/deepsparse/legacy/loggers/helpers.py

@@ -24,9 +24,9 @@
 
 import numpy
 
-import deepsparse.loggers.metric_functions as built_ins
-from deepsparse.loggers import MetricCategories
-from deepsparse.loggers.metric_functions.utils import BatchResult
+import deepsparse.legacy.loggers.metric_functions as built_ins
+from deepsparse.legacy.loggers import MetricCategories
+from deepsparse.legacy.loggers.metric_functions.utils import BatchResult
 
 
 __all__ = [

src/deepsparse/loggers/metric_functions/built_ins.py → src/deepsparse/legacy/loggers/metric_functions/built_ins.py

@@ -16,10 +16,10 @@
 """
 from typing import Any, List, Union
 
-from deepsparse.loggers.metric_functions.registry import (
+from deepsparse.legacy.loggers.metric_functions.registry import (
     register as register_metric_function,
 )
-from deepsparse.loggers.metric_functions.utils import BatchResult
+from deepsparse.legacy.loggers.metric_functions.utils import BatchResult
 
 
 __all__ = ["identity", "predicted_classes", "predicted_top_score"]

src/deepsparse/loggers/metric_functions/computer_vision/built_ins.py → src/deepsparse/legacy/loggers/metric_functions/computer_vision/built_ins.py

@@ -19,10 +19,10 @@
 
 import numpy
 
-from deepsparse.loggers.metric_functions.registry import (
+from deepsparse.legacy.loggers.metric_functions.registry import (
     register as register_metric_function,
 )
-from deepsparse.loggers.metric_functions.utils import BatchResult
+from deepsparse.legacy.loggers.metric_functions.utils import BatchResult
 
 
 __all__ = [

src/deepsparse/loggers/metric_functions/helpers/config_generation.py → src/deepsparse/legacy/loggers/metric_functions/helpers/config_generation.py

@@ -25,9 +25,9 @@
 
 import yaml
 
-from deepsparse.loggers.build_logger import parse_out_predefined_function_groups
-from deepsparse.loggers.config import MetricFunctionConfig
-from deepsparse.loggers.metric_functions.registry import DATA_LOGGING_REGISTRY
+from deepsparse.legacy.loggers.build_logger import parse_out_predefined_function_groups
+from deepsparse.legacy.loggers.config import MetricFunctionConfig
+from deepsparse.legacy.loggers.metric_functions.registry import DATA_LOGGING_REGISTRY
 
 
 _WHITESPACE = "  "

src/deepsparse/loggers/metric_functions/natural_language_processing/built_ins.py → src/deepsparse/legacy/loggers/metric_functions/natural_language_processing/built_ins.py

@@ -16,10 +16,10 @@
 """
 from typing import List, Union
 
-from deepsparse.loggers.metric_functions.registry import (
+from deepsparse.legacy.loggers.metric_functions.registry import (
     register as register_metric_function,
 )
-from deepsparse.loggers.metric_functions.utils import BatchResult
+from deepsparse.legacy.loggers.metric_functions.utils import BatchResult
 
 
 __all__ = ["string_length", "percent_unknown_tokens"]

src/deepsparse/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py → src/deepsparse/legacy/loggers/metric_functions/natural_language_processing/question_answering/built_ins.py

@@ -15,10 +15,10 @@
 Set of functions for logging metrics from the question answering pipeline
 """
 
-from deepsparse.loggers.metric_functions.natural_language_processing import (
+from deepsparse.legacy.loggers.metric_functions.natural_language_processing import (
     string_length,
 )
-from deepsparse.loggers.metric_functions.registry import (
+from deepsparse.legacy.loggers.metric_functions.registry import (
     register as register_metric_function,
 )