Fixes for using files v2 (#1234)

* draft

* pass  tests/test_benchmark.py

* fix tests/test_pipeline_benchmark.py

* fix test_pipeline_benchmark.py

* pass

* pass test_python_logger.py

* pass test_predefined.py

* pass test_transformers.py

* pass tests/deepsparse/image_classification/test_pipelines.py

* pass tests/server/test_endpoints.py

* add test_loggers.py

* indexing error tests/server/test_endpoints.py

* indexing test_run_inference_ner

* lint

* pass test_run_inference.py

* remove comments

* lint

src/deepsparse/transformers/pipelines/embedding_extraction.py

@@ -100,10 +100,7 @@ def to_list(cls) -> List[str]:
 @Pipeline.register(
     task="transformers_embedding_extraction",
     task_aliases=[],
-    default_model_path=(
-        "zoo:nlp/masked_language_modeling/bert-base/pytorch/huggingface/"
-        "wikipedia_bookcorpus/pruned80_quant-none-vnni"
-    ),
+    default_model_path=("zoo:bert-large-wikipedia_bookcorpus-pruned90"),
 )
 class TransformersEmbeddingExtractionPipeline(TransformersPipeline):
     """

src/deepsparse/transformers/pipelines/question_answering.py

@@ -80,10 +80,7 @@ class QuestionAnsweringOutput(BaseModel):
 @Pipeline.register(
     task="question_answering",
     task_aliases=["qa"],
-    default_model_path=(
-        "zoo:nlp/question_answering/bert-base/pytorch/huggingface/"
-        "squad/12layer_pruned80_quant-none-vnni"
-    ),
+    default_model_path=("zoo:bert-large-squad_wikipedia_bookcorpus-pruned90_quantized"),
 )
 class QuestionAnsweringPipeline(TransformersPipeline):
     """

src/deepsparse/transformers/pipelines/text_classification.py

@@ -81,8 +81,7 @@ class TextClassificationOutput(BaseModel):
     task="text_classification",
     task_aliases=["glue", "sentiment_analysis"],
     default_model_path=(
-        "zoo:nlp/sentiment_analysis/bert-base/pytorch/huggingface/"
-        "sst2/12layer_pruned80_quant-none-vnni"
+        "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized"
     ),
 )
 class TextClassificationPipeline(TransformersPipeline):

src/deepsparse/transformers/pipelines/token_classification.py

@@ -137,10 +137,7 @@ class TokenClassificationOutput(BaseModel):
 @Pipeline.register(
     task="token_classification",
     task_aliases=["ner"],
-    default_model_path=(
-        "zoo:nlp/token_classification/bert-base/pytorch/huggingface/"
-        "conll2003/12layer_pruned80_quant-none-vnni"
-    ),
+    default_model_path=("zoo:distilbert-conll2003_wikipedia_bookcorpus-pruned90"),
 )
 class TokenClassificationPipeline(TransformersPipeline):
     """

tests/deepsparse/image_classification/test_pipelines.py

@@ -34,8 +34,7 @@
     "zoo_stub,image_size,num_samples",
     [
         (
-            "zoo:cv/classification/resnet_v1-50/pytorch/sparseml/imagenet/"
-            "pruned85_quant-none-vnni",
+            "zoo:mobilenet_v1-1.0-imagenet-pruned.4block_quantized",
             224,
             5,
         )
@@ -59,10 +58,12 @@ def test_image_classification_pipeline_preprocessing(
     ic_pipeline = Pipeline.create("image_classification", model_path=zoo_stub)
     zoo_model = Model(zoo_stub)
     data_originals_path = None
+
     if zoo_model.sample_inputs is not None:
         if not zoo_model.sample_inputs.files:
             zoo_model.sample_inputs.unzip()
         data_originals_path = zoo_model.sample_inputs.path
+
     for idx, sample in enumerate(load_numpy_list(data_originals_path)):
         image_raw = list(sample.values())[0].astype(numpy.uint8)
         image_raw = numpy.transpose(image_raw, axes=[1, 2, 0])

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/sentiment_analysis.txt

@@ -1,3 +1,3 @@
 identifier:text_classification/pipeline_inputs.sequences__string_length, value:21, category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['LABEL_0'], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.5907761454582214, category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['entailment'], category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.44569113850593567, category:MetricCategories.DATA

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/sentiment_analysis_1.txt

@@ -1,3 +1,3 @@
 identifier:text_classification/pipeline_inputs.sequences__string_length, value:[21, 19], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['LABEL_0', 'LABEL_0'], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.5907761454582214, category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['entailment', 'entailment'], category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.44569113850593567, category:MetricCategories.DATA

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/sentiment_analysis_2.txt

@@ -1,3 +1,3 @@
 identifier:text_classification/pipeline_inputs.sequences__string_length, value:[[21], [19]], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['LABEL_0', 'LABEL_0'], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.5907761454582214, category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['entailment', 'entailment'], category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.44569113850593567, category:MetricCategories.DATA

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/text_classification.txt

@@ -1,2 +1,2 @@
-identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['LABEL_0'], category:MetricCategories.DATA
-identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.5907761454582214, category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.labels__predicted_classes, value:['entailment'], category:MetricCategories.DATA
+identifier:text_classification/pipeline_outputs.scores__predicted_top_score, value:0.44569113850593567, category:MetricCategories.DATA

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/token_classification.txt

@@ -1,3 +1,3 @@
 identifier:token_classification/pipeline_inputs.inputs__string_length, value:21, category:MetricCategories.DATA
-identifier:token_classification/pipeline_outputs__percent_zero_labels, value:[0.5], category:MetricCategories.DATA
-identifier:token_classification/pipeline_outputs__mean_score, value:[0.16483906656503677], category:MetricCategories.DATA
+identifier:token_classification/pipeline_outputs__percent_zero_labels, value:[0.0], category:MetricCategories.DATA
+identifier:token_classification/pipeline_outputs__mean_score, value:[0.170909084379673], category:MetricCategories.DATA

tests/deepsparse/loggers/metric_functions/predefined/predefined_logs/token_classification_1.txt

@@ -1,3 +1,3 @@
 identifier:token_classification/pipeline_inputs.inputs__string_length, value:[21, 19], category:MetricCategories.DATA
-identifier:token_classification/pipeline_outputs__percent_zero_labels, value:[0.5, 0.5], category:MetricCategories.DATA
-identifier:token_classification/pipeline_outputs__mean_score, value:[0.16483906656503677, 0.16483906656503677], category:MetricCategories.DATA
+identifier:token_classification/pipeline_outputs__percent_zero_labels, value:[0.0, 0.0], category:MetricCategories.DATA
+identifier:token_classification/pipeline_outputs__mean_score, value:[0.170909084379673, 0.170909084379673], category:MetricCategories.DATA

tests/deepsparse/pipelines/test_transformers.py

@@ -41,27 +41,21 @@ def test_token_classification_pipeline(engine_mock):
         ]
     )
     assert len(output.predictions) == 2
-    assert len(output.predictions[0]) == 9
-    assert len(output.predictions[1]) == 10
+    assert len(output.predictions[0]) == 6
+    assert len(output.predictions[1]) == 7
 
     assert [p.word for p in output.predictions[0]] == [
-        "my",
         "name",
-        "is",
         "bob",
-        "and",
         "i",
         "live",
         "in",
         "boston",
     ]
 
     assert [p.word for p in output.predictions[1]] == [
-        "sally",
         "bob",
-        "joe",
         ".",
-        "california",
         ",",
         "united",
         "states",

tests/server/test_endpoints.py

@@ -165,10 +165,7 @@ def test_add_endpoint_with_no_route_specified(self, app):
 class TestActualModelEndpoints:
     @pytest.fixture(scope="class")
     def client(self):
-        stub = (
-            "zoo:nlp/text_classification/distilbert-none/"
-            "pytorch/huggingface/qqp/pruned80_quant-none-vnni"
-        )
+        stub = "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized"
         server_config = ServerConfig(
             num_cores=1,
             num_workers=1,

tests/server/test_loggers.py

@@ -35,7 +35,7 @@
 
 
 logger_identifier = "tests/deepsparse/loggers/helpers.py:ListLogger"
-stub = "zoo:nlp/sentiment_analysis/bert-base/pytorch/huggingface/sst2/12layer_pruned80_quant-none-vnni"  # noqa E501
+stub = "zoo:distilbert-sst2_wikipedia_bookcorpus-pruned90"  # noqa E501
 task = "text-classification"
 name = "endpoint_name"
 

tests/test_benchmark.py

@@ -39,51 +39,44 @@ def test_benchmark_help():
             ["-shapes", "[1,128],[1,128],[1,128]"],
         ),
         (
-            "zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/"
-            "pruned-aggressive_98",
+            "zoo:bert-base_cased-squad_wikipedia_bookcorpus-pruned90",
             ["-s", "sync"],
         ),
         (
-            "zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/"
-            "pruned_quant-aggressive_95",
+            "zoo:bert-base_cased-squad_wikipedia_bookcorpus-pruned90",
             ["-s", "async", "-nstreams", "4"],
         ),
         (
-            "zoo:nlp/masked_language_modeling/bert-base/pytorch/huggingface/"
-            "bookcorpus_wikitext/base-none",
+            "zoo:bert-base-wikipedia_bookcorpus-pruned90",
             ["-t", "20"],
         ),
         pytest.param(
-            "zoo:nlp/masked_language_modeling/bert-base/pytorch/huggingface/"
-            "bookcorpus_wikitext/12layer_pruned90-none",
+            "zoo:bert-base-wikipedia_bookcorpus-pruned90",
             [],
             marks=pytest.mark.smoke,
         ),
         (
-            "zoo:cv/classification/resnet_v1-50/pytorch/sparseml/imagenet/base-none",
+            "zoo:mobilenet_v1-1.0-imagenet-pruned.4block_quantized",
             ["-x", "results.json"],
         ),
         (
-            "zoo:cv/classification/resnet_v1-50/pytorch/sparseml/imagenet/"
-            "pruned-moderate",
+            "zoo:mobilenet_v1-1.0-imagenet-pruned.4block_quantized",
             ["-ncores", "4"],
         ),
         (
-            "zoo:cv/classification/resnet_v1-50/pytorch/sparseml/imagenet/"
-            "pruned_quant-moderate",
+            "zoo:mobilenet_v1-1.0-imagenet-pruned.4block_quantized",
             ["-pin", "none"],
         ),
         (
-            "zoo:cv/detection/yolov5-s/pytorch/ultralytics/coco/base-none",
+            "zoo:yolo_v3-spp-coco-pruned",
             ["-pin", "numa", "-shapes", "[1,3,640,640]"],
         ),
         (
-            "zoo:cv/detection/yolov5-s/pytorch/ultralytics/coco/pruned-aggressive_96",
+            "zoo:yolo_v3-spp-coco-pruned",
             ["-q"],
         ),
         (
-            "zoo:cv/detection/yolov5-s/pytorch/ultralytics/coco/"
-            "pruned_quant-aggressive_94",
+            "zoo:yolo_v3-spp-coco-pruned",
             ["-b", "64"],
         ),
     ],
@@ -114,8 +107,8 @@ def test_benchmark(
     ("model_stub"),
     [
         (
-            "zoo:nlp/masked_language_modeling/bert-base/pytorch/huggingface/"
-            "bookcorpus_wikitext/12layer_pruned90-none"
+            "zoo:nlp/masked_language_modeling/bert-large/pytorch/"
+            "huggingface/wikipedia_bookcorpus/pruned90-none"
         ),
     ],
 )

tests/test_data/bert-ner-test-input.csv

@@ -1,2 +1,2 @@
 inputs
-Red color
+Canadian embassy

tests/test_data/bert-ner-test-input.json

@@ -1 +1 @@
-{"inputs":"Red color"}
+{"inputs":"Canadian embassy"}

tests/test_data/bert-ner-test-input.txt

@@ -1 +1 @@
-Red color
+Canadian embassy

tests/test_run_inference.py

@@ -41,8 +41,7 @@ def test_run_inference_ner(cleanup: Dict[str, List]):
         "--task",
         "ner",
         "--model-path",
-        "zoo:nlp/token_classification/bert-base/pytorch/huggingface/conll2003/"
-        "12layer_pruned80_quant-none-vnni",
+        "zoo:bert-large-conll2003_wikipedia_bookcorpus-pruned80.4block_quantized",
         "--data",
         "tests/test_data/bert-ner-test-input.json",
         "--output-file",
@@ -60,7 +59,7 @@ def test_run_inference_ner(cleanup: Dict[str, List]):
     assert "fail" not in res.stdout.lower()
 
     # light validation of output file
-    expected = "red"
+    expected = "canadian"
     assert os.path.exists("output.json")
     with open("output.json") as f:
         data = json.load(f)
@@ -73,15 +72,13 @@ def test_run_inference_ner(cleanup: Dict[str, List]):
     [
         pytest.param(
             "csv",
-            "zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/"
-            "pruned_6layers-aggressive_98",
+            "zoo:bert-base-squad_wikipedia_bookcorpus-pruned90",
             True,
             marks=pytest.mark.smoke,
         ),
         (
             "json",
-            "zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/"
-            "pruned_6layers-aggressive_98",
+            "zoo:bert-base-squad_wikipedia_bookcorpus-pruned90",
             False,
         ),
     ],
@@ -131,32 +128,32 @@ def test_run_inference_qa(
     [
         (
             "csv",
-            "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
+            "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized",
             False,
             ["--batch-size", "1", "--engine-type", "onnxruntime"],
         ),
         (
             "txt",
-            "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
+            "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized",
             True,
             ["--num-cores", "4", "--engine-type", "onnxruntime"],
         ),
         pytest.param(
             "csv",
-            "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
+            "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized",
             True,
             [],
             marks=pytest.mark.smoke,
         ),
         (
             "json",
-            "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
+            "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized",
             True,
             ["--batch-size", "5", "--engine-type", "deepsparse"],
         ),
         (
             "txt",
-            "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
+            "zoo:bert-large-mnli_wikipedia_bookcorpus-pruned80.4block_quantized",
             True,
             ["--batch-size", "10", "--num-cores", "4"],
         ),