6.0b2 Release (#1566)

CMakeLists.txt

@@ -1,6 +1,5 @@
 cmake_minimum_required(VERSION 3.10.2)
 
-set(CMAKE_OSX_ARCHITECTURES x86_64)
 set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
 
 project(coremltools)

coremlpython/CoreMLPython.mm

@@ -2,12 +2,14 @@
 //
 // Use of this source code is governed by a BSD-3-clause license that can be
 // found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
 #import <CoreML/CoreML.h>
 #import "CoreMLPythonArray.h"
 #import "CoreMLPython.h"
 #import "CoreMLPythonUtils.h"
 #import "Globals.hpp"
 #import "Utils.hpp"
+#import <AvailabilityMacros.h>
 #import <fstream>
 #import <vector>
 
@@ -18,10 +20,19 @@
 #error "ARC is off"
 #endif
 
+#ifndef BUILT_WITH_MACOS13_SDK
+#define BUILT_WITH_MACOS13_SDK (MAC_OS_X_VERSION_MAX_ALLOWED >= 130000)
+#endif
+
 namespace py = pybind11;
 
 using namespace CoreML::Python;
 
+bool usingMacOS13OrHigher() {
+    // MLProgram class was introduced in macOS 13.
+    return (NSProtocolFromString(@"MLProgram") != nil);
+}
+
 Model::~Model() {
     @autoreleasepool {
         NSFileManager *fileManager = [NSFileManager defaultManager];
@@ -60,20 +71,27 @@
             throw std::runtime_error(errmsg.str());
         }
 
-        if (@available(macOS 10.14, *)) {
-            MLModelConfiguration *configuration = [MLModelConfiguration new];
-            if (computeUnits == "CPU_ONLY") {
-                configuration.computeUnits = MLComputeUnitsCPUOnly;
-            } else if (computeUnits == "CPU_AND_GPU") {
-                configuration.computeUnits = MLComputeUnitsCPUAndGPU;
+        // Set compute unit
+        MLModelConfiguration *configuration = [MLModelConfiguration new];
+        if (computeUnits == "CPU_ONLY") {
+            configuration.computeUnits = MLComputeUnitsCPUOnly;
+        } else if (computeUnits == "CPU_AND_GPU") {
+            configuration.computeUnits = MLComputeUnitsCPUAndGPU;
+        } else if (computeUnits == "CPU_AND_NE") {
+            if (usingMacOS13OrHigher()) {
+#if BUILT_WITH_MACOS13_SDK
+                configuration.computeUnits = MLComputeUnitsCPUAndNeuralEngine;
+#endif // BUILT_WITH_MACOS13_SDK
             } else {
-                assert(computeUnits == "ALL");
-                configuration.computeUnits = MLComputeUnitsAll;
+                throw std::runtime_error("CPU_AND_NE is only available on macOS >= 13.0");
             }
-            m_model = [MLModel modelWithContentsOfURL:compiledUrl configuration:configuration error:&error];
         } else {
-            m_model = [MLModel modelWithContentsOfURL:compiledUrl error:&error];
+            assert(computeUnits == "ALL");
+            configuration.computeUnits = MLComputeUnitsAll;
         }
+
+        // Create MLModel
+        m_model = [MLModel modelWithContentsOfURL:compiledUrl configuration:configuration error:&error];
         Utils::handleError(error);
     }
 }

coremlpython/CoreMLPythonArray.h

@@ -1,3 +1,8 @@
+// Copyright (c) 2021, Apple Inc. All rights reserved.
+//
+// Use of this source code is governed by a BSD-3-clause license that can be
+// found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wexit-time-destructors"
 #pragma clang diagnostic ignored "-Wdocumentation"

coremlpython/CoreMLPythonUtils.mm

@@ -7,6 +7,9 @@
 
 #include <pybind11/eval.h>
 #include <pybind11/numpy.h>
+#include <iomanip> // for std::setfill etc
+
+#import <Accelerate/Accelerate.h>
 
 #if PY_MAJOR_VERSION < 3
 
@@ -219,6 +222,8 @@ static void handleCVReturn(CVReturn status) {
         format = kCVPixelFormatType_32BGRA;
     } else if (formatStr == "L") {
         format = kCVPixelFormatType_OneComponent8;
+    } else if (formatStr == "F") {
+        format = kCVPixelFormatType_OneComponent16Half;
     } else {
         std::stringstream msg;
         msg << "Unsupported image type " << formatStr << ". ";
@@ -236,7 +241,6 @@ static void handleCVReturn(CVReturn status) {
     Py_ssize_t bytesLength = PyBytes_Size(bytesResult.ptr());
     assert(bytesLength >= 0);
     const char *bytesPtr = PyBytes_AsString(bytesResult.ptr());
-    std::string bytes(bytesPtr, static_cast<size_t>(bytesLength));
 
     // copy data into the CVPixelBuffer
     status = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
@@ -245,71 +249,63 @@ static void handleCVReturn(CVReturn status) {
     assert(baseAddress != nullptr);
     assert(!CVPixelBufferIsPlanar(pixelBuffer));
     size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
-    const char *srcPointer = bytes.data();
+    const char *srcPointer = bytesPtr;
+
+    vImage_Buffer srcBuffer;
+    memset(&srcBuffer, 0, sizeof(srcBuffer));
+    srcBuffer.data = const_cast<char *>(srcPointer);
+    srcBuffer.width = width;
+    srcBuffer.height = height;
 
+    vImage_Buffer dstBuffer;
+    memset(&dstBuffer, 0, sizeof(dstBuffer));
+    dstBuffer.data = baseAddress;
+    dstBuffer.width = width;
+    dstBuffer.height = height;
+
     if (formatStr == "RGB") {
 
         // convert RGB to BGRA
-        assert(bytes.size() == width * height * 3);
-        for (size_t row = 0; row < height; row++) {
-            char *dstPointer = static_cast<char *>(baseAddress) + (row * bytesPerRow);
-
-            for (size_t col = 0; col < width; col++) {
-
-                char R = *srcPointer++;
-                char G = *srcPointer++;
-                char B = *srcPointer++;
-
-                *dstPointer++ = B;
-                *dstPointer++ = G;
-                *dstPointer++ = R;
-                *dstPointer++ = 0; // A
-
-            }
-            assert(bytesPerRow >= width * 4);
-        }
-        assert(srcPointer == bytes.data() + bytes.size());
+        assert(bytesLength == width * height * 3);
+
+        srcBuffer.rowBytes = width * 3;
+        dstBuffer.rowBytes = bytesPerRow;
+        vImageConvert_RGB888toBGRA8888(&srcBuffer, NULL, 255, &dstBuffer, false, 0);
 
     } else if (formatStr == "RGBA") {
 
         // convert RGBA to BGRA
-        assert(bytes.size() == width * height * 4);
-        for (size_t row = 0; row < height; row++) {
-            char *dstPointer = static_cast<char *>(baseAddress) + (row * bytesPerRow);
-
-            for (size_t col = 0; col < width; col++) {
-
-                char R = *srcPointer++;
-                char G = *srcPointer++;
-                char B = *srcPointer++;
-                char A = *srcPointer++;
-
-                *dstPointer++ = B;
-                *dstPointer++ = G;
-                *dstPointer++ = R;
-                *dstPointer++ = A;
-
-            }
-            assert(bytesPerRow >= width * 4);
-        }
-        assert(srcPointer == bytes.data() + bytes.size());
+        assert(bytesLength == width * height * 4);
+        srcBuffer.rowBytes = width * 4;
+        dstBuffer.rowBytes = bytesPerRow;
+        uint8_t permuteMap[4] = { 2, 1, 0, 3 };
+        vImagePermuteChannels_ARGB8888(&srcBuffer, &dstBuffer, permuteMap, 0);
 
-    } else {
+    } else if (formatStr == "L") {
 
-        // assume 8 bit grayscale (the only other case)
-        assert(formatStr == "L");
-        assert(bytes.size() == width * height);
+        // 8 bit grayscale.
+        assert(bytesLength == width * height);
+
+        srcBuffer.rowBytes = width;
+        dstBuffer.rowBytes = bytesPerRow;
+        vImageCopyBuffer(&srcBuffer, &dstBuffer, 1, 0);
+
+    } else if (formatStr == "F") {
 
-        for (size_t row = 0; row < height; row++) {
-            char *dstPointer = static_cast<char *>(baseAddress) + (row * bytesPerRow);
-
-            std::memcpy(dstPointer, srcPointer, width);
-            srcPointer += width;
-        }
+        // convert Float32 to Float16.
+        assert(bytesLength == width * height * sizeof(Float32));
+
+        srcBuffer.rowBytes = width * sizeof(Float32);
+        dstBuffer.rowBytes = bytesPerRow;
+        vImageConvert_PlanarFtoPlanar16F(&srcBuffer, &dstBuffer, 0);
+
+    } else {
+        std::stringstream msg;
+        msg << "Unsupported image type " << formatStr << ". ";
+        msg << "Supported types are: RGB, RGBA, L.";
+        throw std::runtime_error(msg.str());
     }
 
-    assert(srcPointer == bytes.data() + bytes.size());
-
 #ifdef COREML_SHOW_PIL_IMAGES
     if (formatStr == "RGB") {
         // for debugging purposes, convert back to PIL image and show it
@@ -509,40 +505,70 @@ static size_t sizeOfArrayElement(MLMultiArrayDataType type) {
 
     // supports grayscale and BGRA format types
     auto formatType = CVPixelBufferGetPixelFormatType(value);
-    assert(formatType == kCVPixelFormatType_32BGRA || formatType == kCVPixelFormatType_OneComponent8);
-
+    assert(formatType == kCVPixelFormatType_32BGRA
+           || formatType == kCVPixelFormatType_OneComponent8
+           || formatType == kCVPixelFormatType_OneComponent16Half);
+
+    auto height = CVPixelBufferGetHeight(value);
+    auto width = CVPixelBufferGetWidth(value);
+
+    py::str mode;
+    size_t dstBytesPerRow = 0;
+    if (formatType == kCVPixelFormatType_32BGRA) {
+        mode = "RGBA";
+        dstBytesPerRow = width * 4;
+    } else if (formatType == kCVPixelFormatType_OneComponent8) {
+        mode = "L";
+        dstBytesPerRow = width * sizeof(uint8_t);
+    } else if (formatType == kCVPixelFormatType_OneComponent16Half) {
+        mode = "F";
+        dstBytesPerRow = width * sizeof(Float32);
+    } else {
+        std::stringstream msg;
+        msg << "Unsupported pixel format type: " << std::hex << std::setfill('0') << std::setw(4) << formatType << ". ";
+        throw std::runtime_error(msg.str());
+    }
+
+    PyObject *dstPyBytes = PyBytes_FromStringAndSize(NULL, height * dstBytesPerRow);
+    if (!dstPyBytes) {
+        throw std::bad_alloc();
+    }
+
     auto result = CVPixelBufferLockBaseAddress(value, kCVPixelBufferLock_ReadOnly);
     assert(result == kCVReturnSuccess);
 
     uint8_t *src = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddress(value));
     assert(src != nullptr);
 
-    auto height = CVPixelBufferGetHeight(value);
-    auto width = CVPixelBufferGetWidth(value);
     size_t srcBytesPerRow = CVPixelBufferGetBytesPerRow(value);
-    // Initializing this for Xcode warnings
-    size_t dstBytesPerRow = 0;
-    py::str mode;
+
+    // Prepare for vImage blitting
+    vImage_Buffer srcBuffer;
+    memset(&srcBuffer, 0, sizeof(srcBuffer));
+    srcBuffer.data = src;
+    srcBuffer.width = width;
+    srcBuffer.height = height;
+    srcBuffer.rowBytes = srcBytesPerRow;
+
+    vImage_Buffer dstBuffer;
+    memset(&dstBuffer, 0, sizeof(dstBuffer));
+    dstBuffer.data = PyBytes_AS_STRING(dstPyBytes);
+    dstBuffer.width = width;
+    dstBuffer.height = height;
+    dstBuffer.rowBytes = dstBytesPerRow;
+
     if (formatType == kCVPixelFormatType_32BGRA) {
-        dstBytesPerRow = width * 4;
-        mode = "RGBA";
+        // convert BGRA to RGBA
+        uint8_t permuteMap[4] = { 2, 1, 0, 3 };
+        vImagePermuteChannels_ARGB8888(&srcBuffer, &dstBuffer, permuteMap, 0);
     } else if (formatType == kCVPixelFormatType_OneComponent8) {
-        dstBytesPerRow = width;
-        mode = "L";
-    }
-    std::string array(height * dstBytesPerRow, 0);
-    for (size_t i=0; i<height; i++) {
-        for (size_t j=0; j<width; j++) {
-            if (formatType == kCVPixelFormatType_32BGRA) {
-                // convert BGRA to RGBA
-                array[(i * dstBytesPerRow) + (j * 4) + 0] = static_cast<char>(src[(i * srcBytesPerRow) + (j * 4) + 2]);
-                array[(i * dstBytesPerRow) + (j * 4) + 1] = static_cast<char>(src[(i * srcBytesPerRow) + (j * 4) + 1]);
-                array[(i * dstBytesPerRow) + (j * 4) + 2] = static_cast<char>(src[(i * srcBytesPerRow) + (j * 4) + 0]);
-                array[(i * dstBytesPerRow) + (j * 4) + 3] = static_cast<char>(src[(i * srcBytesPerRow) + (j * 4) + 3]);
-            } else if (formatType == kCVPixelFormatType_OneComponent8) {
-                array[(i * dstBytesPerRow) + j] = static_cast<char>(src[(i * srcBytesPerRow) + j]);
-            }
-        }
+        vImageCopyBuffer(&srcBuffer, &dstBuffer, 1, 0);
+    } else if (formatType == kCVPixelFormatType_OneComponent16Half) {
+        vImageConvert_Planar16FtoPlanarF(&srcBuffer, &dstBuffer, 0);
+    } else {
+        std::stringstream msg;
+        msg << "Unsupported pixel format type: " << std::hex << std::setfill('0') << std::setw(4) << formatType << ". ";
+        throw std::runtime_error(msg.str());
     }
 
     result = CVPixelBufferUnlockBaseAddress(value, kCVPixelBufferLock_ReadOnly);
@@ -552,7 +578,8 @@ static size_t sizeOfArrayElement(MLMultiArrayDataType type) {
     py::eval<py::eval_single_statement>("import PIL.Image", scope);
     py::object pilImage = py::eval<py::eval_expr>("PIL.Image", scope);
     py::object frombytes = pilImage.attr("frombytes");
-    py::object img = frombytes(mode, py::make_tuple(width, height), py::bytes(array));
+    py::bytes dstBytes = py::reinterpret_steal<py::bytes>(dstPyBytes); // transfer ownership of `dstPyBytes` to `dstBytes`
+    py::object img = frombytes(mode, py::make_tuple(width, height), dstBytes);
     return img;
 }
 

coremltools/__init__.py

@@ -70,9 +70,13 @@ class ComputeUnit(_Enum):
     ALL = 1  # Allows the model to use all compute units available, including the neural engine
     CPU_AND_GPU = 2 # Allows the model to use both the CPU and GPU, but not the neural engine
     CPU_ONLY = 3 # Limit the model to only use the CPU
+    CPU_AND_NE = 4 # Allows the model to use both the CPU and neural engine, but not the GPU.
+                   # Only available on macOS >= 13.0
 
 # A dictionary that maps the CoreML model specification version to the MLProgram/MIL opset string
 _OPSET = {
+    _SPECIFICATION_VERSION_IOS_13: "CoreML3",
+    _SPECIFICATION_VERSION_IOS_14: "CoreML4",
     _SPECIFICATION_VERSION_IOS_15: "CoreML5",
     _SPECIFICATION_VERSION_IOS_16: "CoreML6",
 }

coremltools/converters/_converters_entry.py

@@ -92,6 +92,7 @@ def convert(
             - `HDF5 file path <https://keras.io/api/models/model_saving_apis/>`_ (``.h5``)
             - `SavedModel <https://www.tensorflow.org/guide/saved_model>`_ directory path
             - A `concrete function <https://www.tensorflow.org/guide/concrete_function>`_
+            - A `GraphDef <https://www.tensorflow.org/api_docs/python/tf/compat/v1/GraphDef>`_
 
         * PyTorch
         
@@ -344,6 +345,8 @@ def skip_real_div_ops(op):
             - ``coremltools.ComputeUnit.CPU_ONLY``: Limit the model to only use the CPU.
             - ``coremltools.ComputeUnit.CPU_AND_GPU``: Use both the CPU and GPU, but not the
               neural engine.
+            - ``coremltools.ComputeUnit.CPU_AND_NE``: Use both the CPU and neural engine, but
+              not the GPU. Only available on macOS >= 13.0.
 
     package_dir : str
         Post conversion, the model is saved at a temporary location and

coremltools/converters/mil/_deployment_compatibility.py

@@ -40,6 +40,12 @@ class AvailableTarget(IntEnum):
     tvOS14 = _SPECIFICATION_VERSION_IOS_14
     tvOS15 = _SPECIFICATION_VERSION_IOS_15
     tvOS16 = _SPECIFICATION_VERSION_IOS_16
+
+    # customized __str__
+    def __str__(self):
+        original_str = super().__str__()
+        new_str = original_str.replace(type(self).__name__, "coremltools.target")
+        return new_str
 
 
 _get_features_associated_with = {}
@@ -133,7 +139,7 @@ def check_deployment_compatibility(spec, representation, deployment_target):
 
     for any_target in AvailableTarget:
 
-        if any_target.value > deployment_target.value and any_target in _get_features_associated_with:
+        if any_target > deployment_target and any_target in _get_features_associated_with:
             missing_features = _get_features_associated_with[any_target](spec)
 
             if missing_features:

coremltools/converters/mil/backend/mil/load.py

@@ -281,7 +281,7 @@ def load(prog, weights_dir, resume_on_errors=False, specification_version=_SPECI
     if "main" not in prog.functions:
         raise ValueError("main function not found in program")
 
-    mil_passes.mil_backend_passes(prog, specification_version)
+    mil_passes.mil_backend_passes(prog)
 
     # if user has specified "ClassifierConfig", then add the "classify" op to the prog
     classifier_config = kwargs.get("classifier_config", None)