5.0b3 Release (#1285)

* 5.0b3 Release

* Update wheel path in GitLab CI yaml

* Skip test for macOS 11

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -9,7 +9,7 @@ assignees: ''
 
 ## 🌱 Describe your Feature Request
 - A clear and concise description of what the problem is.
-- CoreML / iOS version you are using?
+- Core ML / iOS version you are using?
 - Are you interested in contributing?
 
 ## Use cases

.gitlab-ci.yml

@@ -128,7 +128,7 @@ test_macos11_py37_coremltools_test:
   dependencies:
     - build_wheel_macos_py37
   variables:
-    WHEEL_PATH: build/dist/*cp37*10_16*
+    WHEEL_PATH: build/dist/*cp37*10_15*
     TEST_PACKAGE: coremltools.test
     PYTHON: "3.7"
 
@@ -141,7 +141,7 @@ test_macos11_py37_pytorch:
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil.frontend.torch
-    WHEEL_PATH: build/dist/*cp37*10_16*
+    WHEEL_PATH: build/dist/*cp37*10_15*
 
 test_macos11_py37_tf1:
   <<: *test_macos_pkg
@@ -152,7 +152,7 @@ test_macos11_py37_tf1:
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil.frontend.tensorflow
-    WHEEL_PATH: build/dist/*cp37*10_16*
+    WHEEL_PATH: build/dist/*cp37*10_15*
 
 test_macos11_py37_tf2:
   <<: *test_macos_pkg_with_reqs
@@ -164,7 +164,7 @@ test_macos11_py37_tf2:
     PYTHON: "3.7"
     REQUIREMENTS: reqs/test_tf2.pip
     TEST_PACKAGE: coremltools.converters.mil.frontend.tensorflow2
-    WHEEL_PATH: build/dist/*cp37*10_16*
+    WHEEL_PATH: build/dist/*cp37*10_15*
 
 test_macos11_py37_mil:
   <<: *test_macos_pkg
@@ -175,7 +175,7 @@ test_macos11_py37_mil:
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil
-    WHEEL_PATH: build/dist/*cp37*10_16*
+    WHEEL_PATH: build/dist/*cp37*10_15*
 
 #########################################################################
 ##
@@ -190,7 +190,7 @@ test_macos11_py38_coremltools_smoke_test:
   dependencies:
     - build_wheel_macos_py38
   variables:
-    WHEEL_PATH: build/dist/*cp38*10_16*
+    WHEEL_PATH: build/dist/*cp38*10_15*
     TEST_PACKAGE: coremltools.test.neural_network.test_simple_nn_inference
     PYTHON: "3.8"
 

CMakeLists.txt

@@ -194,7 +194,15 @@ endif()
 
 set(PYTHON_TAG "cp${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}")
 if(APPLE)
-  set(PLAT_NAME "macosx_10_16_intel;macosx_10_15_intel;macosx_10_14_intel;macosx_10_13_intel;macosx_10_12_intel")
+  execute_process(COMMAND uname -m OUTPUT_VARIABLE HARDWARE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(${HARDWARE_NAME} MATCHES "x86_64")
+    set(MIN_MAC_OS "10_15")
+  elseif(${HARDWARE_NAME} MATCHES "arm64")
+    set(MIN_MAC_OS "11_0")
+  else()
+    message(FATAL_ERROR "Unsupported hardware type. On macOS, x86_64 and arm64 are supported.")
+  endif()
+  set(PLAT_NAME "macosx_${MIN_MAC_OS}_${HARDWARE_NAME}")
 elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
   set(PLAT_NAME "manylinux1_x86_64")
 else()

README.md

@@ -34,7 +34,7 @@ The coremltools 5 package offers several performance improvements over previous
 To install coremltools 5, use the following command:
 
 ```shell
-pip install coremltools==5.0b2
+pip install coremltools==5.0b3
 ```
 
 

coremlpython/CoreMLPython.h

@@ -1,3 +1,7 @@
+// Copyright (c) 2021, Apple Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-3-clause license that can be
+// found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wexit-time-destructors"
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -22,7 +26,7 @@ namespace CoreML {
             Model(const Model&) = delete;
             Model& operator=(const Model&) = delete;
             ~Model();
-            explicit Model(const std::string& urlStr, bool useCPUOnly);
+            explicit Model(const std::string& urlStr, const std::string& computeUnits);
             py::dict predict(const py::dict& input, bool useCPUOnly);
             static py::bytes autoSetSpecificationVersion(const py::bytes& modelBytes);
             static int32_t maximumSupportedSpecificationVersion();

coremlpython/CoreMLPython.mm

@@ -1,3 +1,7 @@
+// Copyright (c) 2021, Apple Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-3-clause license that can be
+// found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 #import <CoreML/CoreML.h>
 #import "CoreMLPythonArray.h"
 #import "CoreMLPython.h"
@@ -27,7 +31,7 @@
     }
 }
 
-Model::Model(const std::string& urlStr, bool useCPUOnly) {
+Model::Model(const std::string& urlStr, const std::string& computeUnits) {
     @autoreleasepool {
 
         // Compile the model
@@ -58,8 +62,13 @@
 
         if (@available(macOS 10.14, *)) {
             MLModelConfiguration *configuration = [MLModelConfiguration new];
-            if (useCPUOnly){
+            if (computeUnits == "CPU_ONLY") {
                 configuration.computeUnits = MLComputeUnitsCPUOnly;
+            } else if (computeUnits == "CPU_AND_GPU") {
+                configuration.computeUnits = MLComputeUnitsCPUAndGPU;
+            } else {
+                assert(computeUnits == "ALL");
+                configuration.computeUnits = MLComputeUnitsAll;
             }
             m_model = [MLModel modelWithContentsOfURL:compiledUrl configuration:configuration error:&error];
         } else {
@@ -141,7 +150,7 @@
     py::module m("libcoremlpython", "CoreML.Framework Python bindings");
 
     py::class_<Model>(m, "_MLModelProxy")
-        .def(py::init<const std::string&, bool>())
+        .def(py::init<const std::string&, const std::string&>())
         .def("predict", &Model::predict)
         .def_static("auto_set_specification_version", &Model::autoSetSpecificationVersion)
         .def_static("maximum_supported_specification_version", &Model::maximumSupportedSpecificationVersion);

coremltools/__init__.py

@@ -21,9 +21,11 @@
 
 For more information: http://developer.apple.com/documentation/coreml
 """
+from enum import Enum as _Enum
+from logging import getLogger as _getLogger
+
 
 # Backup root logger handlers
-from logging import getLogger as _getLogger
 _root_logger = _getLogger()
 _root_logger_handlers_backup = _root_logger.handlers.copy()
 
@@ -58,6 +60,14 @@
 # New versions for iOS 15.0
 _SPECIFICATION_VERSION_IOS_15 = 6
 
+class ComputeUnit(_Enum):
+    '''
+    The set of processing-unit configurations the model can use to make predictions.
+    '''
+    ALL = 1  # Allows the model to use all compute units available, including the neural engine
+    CPU_AND_GPU = 2 # Allows the model to use both the CPU and GPU, but not the neural engine
+    CPU_ONLY = 3 # Limit the model to only use the CPU
+
 # expose sub packages as directories
 from . import converters
 from . import proto

coremltools/_deps/__init__.py

@@ -229,6 +229,13 @@ def __get_sklearn_version(version):
     _HAS_ONNX = False
 MSG_ONNX_NOT_FOUND = "ONNX not found."
 
+try:
+    import scipy
+except:
+    _HAS_SCIPY = False
+else:
+    _HAS_SCIPY = True
+
 # General utils
 def version_ge(module, target_version):
     """

coremltools/converters/_converters_entry.py

@@ -4,7 +4,9 @@
 # found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 import gc
 import collections
+import warnings
 
+from coremltools import ComputeUnit as _ComputeUnit
 from coremltools.converters.mil.mil.passes.quantization_passes import AbstractQuantizationPass, FP16ComputePrecision
 from coremltools.converters.mil.mil.passes.quantization_passes import ComputePrecision as precision
 from coremltools.converters.mil.input_types import InputType, ClassifierConfig
@@ -43,6 +45,7 @@ def convert(
     convert_to=None,
     compute_precision=None,
     skip_model_load=False,
+    compute_units=_ComputeUnit.ALL,
     **kwargs
 ):
     """
@@ -199,10 +202,9 @@ def convert(
 
                   The above casts all the float32 tensors to be float16, except
                   the input/output tensors to any ``linear`` op.
-            - If ``None``, the parameter defaults to ``coremltools.precision.FLOAT32``.
-        - TODO: rdar://74140243.
-            - Before coremltools 5.0 release, change the default
-              to coremltools.precision.FLOAT16 when convert_to="mlprogram"
+            - If ``None``,
+                - when convert_to="mlprogram", compute_precision parameter defaults to ``coremltools.precision.FLOAT16``.
+                - when convert_to="neuralnetwork", compute_precision parameter needs to be None and has no meaning.
 
     skip_model_load : bool
         Set to True to prevent coremltools from calling into the Core ML framework
@@ -216,6 +218,14 @@ def convert(
         can only be compiled and loaded from macOS12+.
         Defaults to False.
 
+    compute_units: coremltools.ComputeUnit
+        A enum with three possible values:
+            - coremltools.ComputeUnit.ALL - use all compute units available, including the
+                  neural engine.
+            - coremltools.ComputeUnit.CPU_ONLY - limit the model to only use the CPU.
+            - coremltools.ComputeUnit.CPU_AND_GPU - use both the CPU and GPU, but not the
+                  neural engine.
+
     Returns
     -------
     model : ``coremltools.models.MLModel`` or ``coremltools.converters.mil.Program``
@@ -272,13 +282,14 @@ def convert(
     _validate_inputs(model, exact_source, inputs, outputs, classifier_config, compute_precision,
                      exact_target, **kwargs)
 
+    if "useCPUOnly" in kwargs and kwargs["useCPUOnly"]:
+        warnings.warn('The "useCPUOnly" parameter is deprecated and will be removed in 6.0. '
+                      'Use the compute_units parameter: "compute_units=coremotools.ComputeUnits.CPU_ONLY".')
+        compute_units = _ComputeUnit.CPU_ONLY
+
 
     if compute_precision is None:
-        # TODO: rdar://74140243
-        # Before 5.0 release,
-        # map "None" to "fp32" for "neuralnetwork"
-        # and to "fp16" for "mlprogram"
-        transforms = list()
+        transforms = [FP16ComputePrecision(op_selector=lambda op: True)] if convert_to != "neuralnetwork" else list()
     elif compute_precision == precision.FLOAT32:
         transforms = list()
     elif compute_precision == precision.FLOAT16:
@@ -295,8 +306,9 @@ def convert(
         inputs=inputs,
         outputs=outputs,
         classifier_config=classifier_config,
-        transforms=transforms,
+        transforms=tuple(transforms),
         skip_model_load=skip_model_load,
+        compute_units=compute_units,
         **kwargs
     )
 
@@ -355,12 +367,11 @@ def raise_if_duplicated(input_list):
             msg = '"classifier_config" must be of type ClassifierConfig'
             raise ValueError(msg)
 
-    if convert_to.lower() == 'neuralnetwork':
-        if compute_precision is not None:
-            if compute_precision != precision.FLOAT32:
-                msg = "'compute_precision' must be coremltools.precision.FLOAT32 when " \
-                    "the target is 'neuralnetwork' (i.e. deployment target is less than iOS15)"
-                raise ValueError(msg)
+    if convert_to.lower() == 'neuralnetwork' and compute_precision is not None:
+        msg = "compute_precision is only supported for mlprogram target and must be None if target=='neuralnetwork'.\n" \
+              "Note that target may be implicitly set depending on the minimum_deployment_target.\n" \
+              "See minimum_deployment_target for more details."
+        raise ValueError(msg)
 
     if compute_precision is not None:
         if compute_precision not in [precision.FLOAT32, precision.FLOAT16]:

coremltools/converters/mil/backend/mil/load.py

@@ -4,7 +4,7 @@
 import tempfile
 
 from coremltools.converters.mil.backend.mil.helper import *
-from coremltools.converters.mil.backend.mil.passes.mil_passes import mil_backend_passes
+import coremltools.converters.mil.backend.mil.passes.mil_passes as mil_passes
 import coremltools.proto.MIL_pb2 as pm
 from coremltools.converters.mil.mil import types
 from coremltools.converters.mil.mil import Function
@@ -249,7 +249,7 @@ def load(prog, weights_dir, resume_on_errors=False, **kwargs):
     if "main" not in prog.functions:
         raise ValueError("main function not found in program")
 
-    mil_backend_passes(prog)
+    mil_passes.mil_backend_passes(prog)
 
     # if user has specified "ClassifierConfig", then add the "classify" op to the prog
     classifier_config = kwargs.get("classifier_config", None)

coremltools/converters/mil/backend/mil/passes/adjust_io_to_supported_types.py

@@ -68,6 +68,12 @@ def adjust_io_to_supported_types(prog):
 #####
 # Main Function
 #####
+def _adjust_var_dtype_helper(var, dtype):
+    if (_types.is_scalar(var.sym_type)):
+        var._sym_type = dtype
+    else:
+        var._sym_type = _types.tensor(dtype, var.sym_type.get_shape())
+
 def _adjust_main_inputs(func):
     first_op = func.operations[0] if len(func.operations) > 0 else None
     for input_name, input_var in func.inputs.items():
@@ -81,13 +87,13 @@ def _adjust_main_inputs(func):
                                ". Only integer variables of bit width 32 are supported by the CoreML runtime. " +\
                                "This input will be assigned a dType of int32. " +\
                                "No cast will be inserted; the previous dtype will be replaced.")
-                input_var._sym_type = _types.tensor(_types.int32, input_var.sym_type.get_shape())
+                _adjust_var_dtype_helper(input_var, _types.int32)
             elif input_var.dtype == _types.fp64:
                 # Replace float64 input type with fp32.
                 _warnings.warn("Input" + input_var.name + " is of dtype fp64. 64 bit float inputs are " +\
                                "not supported by ML program models. This input will be assigned a dType " +\
                                "of fp32. No cast will be inserted; the previous dtype will be replaced.")
-                input_var._sym_type = _types.tensor(_types.fp32, input_var.sym_type.get_shape())
+                _adjust_var_dtype_helper(input_var, _types.fp32)
             else:
                 # This is some other dType. Change the type to fp32 and add a cast.
                 # This is only a limitation of main--other functions do not represent CoreML model inputs
@@ -100,8 +106,7 @@ def _adjust_main_inputs(func):
                 with func:
                     casted_input_var = _mb.cast(x=input_var, dtype=input_dtype_str, before_op=first_op)
                     func.replace_uses_of_var_after_op(anchor_op=casted_input_var.op, old_var=input_var, new_var=casted_input_var)
-                    input_var._sym_type = _types.tensor(_types.fp32, input_var.sym_type.get_shape())
-
+                    _adjust_var_dtype_helper(input_var, _types.fp32)
 
 def _adjust_main_outputs(func):
     new_outputs = []
@@ -145,14 +150,14 @@ def _adjust_var(var):
                            ". Only integer variables of bit width 32 are supported by the CoreML runtime. " +\
                            "This input will be assigned a dType of int32. " +\
                            "No cast will be inserted; the previous dtype will be replaced.")
-            var._sym_type = _types.tensor(_types.int32, var.sym_type.get_shape())
+            _adjust_var_dtype_helper(var, _types.int32)
         else:
             # This is some other unsupported dType. Change the input type to fp32.
             _warnings.warn("Var " + var.name + " is of dType " + dtype_str + ". The CoreML runtime " +\
                            "does not support this dType (only fp16, fp32, bool, and int32 are supported). " +\
                            "This input will be assigned a dType of fp32. No cast will be inserted; " +\
                            "the previous dtype will be replaced.")
-            var._sym_type = _types.tensor(_types.fp32, var.sym_type.get_shape())
+            _adjust_var_dtype_helper(var, _types.fp32)
 
 
 def _adjust_func_inputs(func):

coremltools/converters/mil/backend/mil/passes/mil_passes.py

@@ -14,6 +14,8 @@ def mil_backend_passes(prog):
         "mil_backend::adjust_io_to_supported_types",
         "mil_backend::insert_image_preprocessing_ops",
         "mil_backend::fuse_activation_silu",
+        "mil_backend::rank0_expand_dims_swap",
+        "common::const_elimination", # rank0_expand_dims_swap might introduce some new const tensor
         # TODO: Right now, "const elimination" pass CANNOT be done after the "homogenize_input_dtypes" pass.
         # Remove this requirement in rdar://76032946.
         # Right now due to a bug in the PYMIL const op, which is that it can only produce FP32 and INT32 types tensors (e.g. it can't produce int64),