Merge branch 'main' into rename-llm-variables

src/deepsparse/server/helpers.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
+
+import numpy
+from pydantic import BaseModel
 
 from deepsparse import (
     BaseLogger,
@@ -24,7 +27,7 @@
 from deepsparse.server.config import EndpointConfig, ServerConfig
 
 
-__all__ = ["server_logger_from_config"]
+__all__ = ["server_logger_from_config", "prep_outputs_for_serialization"]
 
 
 def server_logger_from_config(config: ServerConfig) -> BaseLogger:
@@ -58,6 +61,26 @@ def server_logger_from_config(config: ServerConfig) -> BaseLogger:
     )
 
 
+def prep_outputs_for_serialization(pipeline_outputs: Any):
+    """
+    Prepares a pipeline output for JSON serialization by converting any numpy array
+    field to a list. For large numpy arrays, this operation will take a while to run.
+
+    :param pipeline_outputs: output data to clean
+    :return: cleaned pipeline_outputs
+    """
+    if isinstance(pipeline_outputs, BaseModel):
+        for field_name in pipeline_outputs.__fields__.keys():
+            field_value = getattr(pipeline_outputs, field_name)
+            if isinstance(field_value, numpy.ndarray):
+                # numpy arrays aren't JSON serializable
+                setattr(pipeline_outputs, field_name, field_value.tolist())
+    elif isinstance(pipeline_outputs, numpy.ndarray):
+        pipeline_outputs = pipeline_outputs.tolist()
+
+    return pipeline_outputs
+
+
 def _extract_system_logging_from_endpoints(
     endpoints: List[EndpointConfig],
 ) -> Dict[str, SystemLoggingGroup]:

src/deepsparse/server/server.py

@@ -34,7 +34,10 @@
     SystemLoggingConfig,
 )
 from deepsparse.server.config_hot_reloading import start_config_watcher
-from deepsparse.server.helpers import server_logger_from_config
+from deepsparse.server.helpers import (
+    prep_outputs_for_serialization,
+    server_logger_from_config,
+)
 from deepsparse.server.system_logging import (
     SystemLoggingMiddleware,
     log_system_information,
@@ -261,6 +264,7 @@ def _predict(request: pipeline.input_schema):
                 server_logger=server_logger,
                 system_logging_config=system_logging_config,
             )
+        pipeline_outputs = prep_outputs_for_serialization(pipeline_outputs)
         return pipeline_outputs
 
     def _predict_from_files(request: List[UploadFile]):