Merge branch 'main' into bug-benchmark-int-ext-inconsistent-values

setup.py

@@ -309,7 +309,7 @@ def _setup_entry_points() -> Dict:
             f"deepsparse.image_classification.eval={ic_eval}",
             "deepsparse.license=deepsparse.license:main",
             "deepsparse.validate_license=deepsparse.license:validate_license_cli",
-            "deepsparse.eval=deepsparse.evaluate.evaluator.cli:main",
+            "deepsparse.eval=deepsparse.evaluation.cli:main",
         ]
     }
 

src/deepsparse/evaluation/cli.py

@@ -115,7 +115,7 @@
 @click.option(
     "-e",
     "--engine_type",
-    type=click.Choice([DEEPSPARSE_ENGINE, ORT_ENGINE, TORCHSCRIPT_ENGINE, None]),
+    type=click.Choice([DEEPSPARSE_ENGINE, ORT_ENGINE, TORCHSCRIPT_ENGINE]),
     default=DEEPSPARSE_ENGINE,
     help="The engine to use for the evaluation. The default is the "
     "DeepSparse engine. If the evaluation should be run without "

src/deepsparse/schedulers/continuous_batching_scheduler.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import asyncio
 from concurrent.futures import Future
 from threading import Lock
 from typing import List
@@ -113,7 +113,10 @@ def submit(self, *args, operator: Operator, **kwargs) -> Future:
                 f"found {type(inputs)}"
             )
 
-        future = Future()
+        # asyncio.Future() is used when we run the pipeline using async/excecute
+        # operators using asyncio. Outside of the async pathway
+        # (i.e. outside of the server), we use concurrent.Future()
+        future = asyncio.Future() if kwargs.get("loop") else Future()
         self._queues.add_queue_item(key=operator, item=inputs, future=future)
 
         return future

src/deepsparse/server/server.py

@@ -238,6 +238,7 @@ async def predict(
         system_logging_config: SystemLoggingConfig,
         raw_request: Request,
     ):
+
         if hasattr(proxy_pipeline.pipeline, "run_async"):
             inference_state = InferenceState()
             inference_state.create_state({})

tests/deepsparse/schedulers/test_continuous_batching_scheduler.py

@@ -31,7 +31,7 @@ def test_continuous_batching_executor_thread():
         "zoo:mobilenet_v2-1.0-imagenet-base",
     )
 
-    scheduler.add_engine_operator(engine_operator, [1])
+    scheduler.add_engine_operator(engine_operator, [2])
 
     # submit job to scheduler and expect future to be returned
     engine_input = engine_operator.input_schema(