diff --git a/ports/onnxruntime/fix-clang-cl-simd-compile.patch b/ports/onnxruntime/fix-clang-cl-simd-compile.patch deleted file mode 100644 index 18c598c547ad75..00000000000000 --- a/ports/onnxruntime/fix-clang-cl-simd-compile.patch +++ /dev/null @@ -1,80 +0,0 @@ -diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake -index bee83ff07c..04b4cf42b7 100644 ---- a/cmake/onnxruntime_mlas.cmake -+++ b/cmake/onnxruntime_mlas.cmake -@@ -159,15 +159,27 @@ function(setup_mlas_source_for_windows) - ) - set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "/arch:AVX2") - -+ file(GLOB_RECURSE mlas_platform_srcs_avx512 CONFIGURE_DEPENDS -+ "${MLAS_SRC_DIR}/intrinsics/avx512/*.cpp" -+ ) -+ set(mlas_platform_srcs_amx "${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp") -+ -+ # clang-cl requires us to enable the platform feature flags explicitly to compile the intrinsics code -+ # unlike MSVC. See: https://github.com/llvm/llvm-project/issues/53520 -+ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") -+ set_source_files_properties(${mlas_platform_srcs_avx512} PROPERTIES COMPILE_FLAGS "/arch:AVX512") -+ set_source_files_properties(${mlas_platform_srcs_amx} PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mamx-tile -mamx-int8") -+ endif() -+ - target_sources(onnxruntime_mlas PRIVATE - ${MLAS_SRC_DIR}/dgemm.cpp - ${mlas_platform_srcs_avx} - ${mlas_platform_srcs_avx2} -- ${MLAS_SRC_DIR}/qgemm_kernel_amx.cpp -+ ${mlas_platform_srcs_avx512} -+ ${mlas_platform_srcs_amx} - ${MLAS_SRC_DIR}/qgemm_kernel_avx2.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp - ${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp -- ${MLAS_SRC_DIR}/intrinsics/avx512/quantize_avx512f.cpp - ${MLAS_SRC_DIR}/amd64/QgemmU8S8KernelAmx.asm - ${MLAS_SRC_DIR}/amd64/QgemmU8S8KernelAvx2.asm - ${MLAS_SRC_DIR}/amd64/QgemmU8U8KernelAvx2.asm -@@ -205,9 +217,15 @@ function(setup_mlas_source_for_windows) - ${MLAS_SRC_DIR}/amd64/ErfKernelFma3.asm - ) - if (NOT onnxruntime_ORT_MINIMAL_BUILD) -- target_sources(onnxruntime_mlas PRIVATE -+ set(onnxruntime_mlas_q4gemm_avx512 - ${MLAS_SRC_DIR}/q4gemm_avx512.cpp - ) -+ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") -+ # clang-cl requires us to enable the platform feature flags explicitly to compile the intrinsics code -+ # unlike MSVC. See: https://github.com/llvm/llvm-project/issues/53520 -+ set_source_files_properties(${onnxruntime_mlas_q4gemm_avx512} PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mavx512vnni") -+ endif() -+ target_sources(onnxruntime_mlas PRIVATE ${onnxruntime_mlas_q4gemm_avx512}) - endif() - else() - target_sources(onnxruntime_mlas PRIVATE -diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake -index 5c294b5..a466c77 100644 ---- a/cmake/onnxruntime_mlas.cmake -+++ b/cmake/onnxruntime_mlas.cmake -@@ -169,6 +169,9 @@ function(setup_mlas_source_for_windows) - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set_source_files_properties(${mlas_platform_srcs_avx512} PROPERTIES COMPILE_FLAGS "/arch:AVX512") - set_source_files_properties(${mlas_platform_srcs_amx} PROPERTIES COMPILE_FLAGS "/arch:AVX512 -mamx-tile -mamx-int8") -+ # https://clang.llvm.org/docs/UsersManual.html#cpu-architectures-features-and-limitations -+ set_source_files_properties(${MLAS_SRC_DIR}/qgemm_kernel_sse.cpp PROPERTIES COMPILE_FLAGS "-march=x86-64") -+ set_source_files_properties(${MLAS_SRC_DIR}/qgemm_kernel_sse41.cpp PROPERTIES COMPILE_FLAGS "-march=x86-64-v2") - endif() - - target_sources(onnxruntime_mlas PRIVATE -diff --git a/onnxruntime/core/mlas/lib/qgemm_kernel_sse41.cpp b/onnxruntime/core/mlas/lib/qgemm_kernel_sse41.cpp -index 68931c5..6c095bd 100644 ---- a/onnxruntime/core/mlas/lib/qgemm_kernel_sse41.cpp -+++ b/onnxruntime/core/mlas/lib/qgemm_kernel_sse41.cpp -@@ -16,6 +16,10 @@ Abstract: - - #include "mlasi.h" - #include "qgemm.h" -+#if defined(__clang__) -+#include -+#include -+#endif - - // N.B. MSVC does not require turning on SSE 4.1 intrinsics and the current use - // for this code is Windows only, so restrict this kernel to that environment. diff --git a/ports/onnxruntime/fix-cmake.patch b/ports/onnxruntime/fix-cmake.patch index 318fa9f3e6817d..b0a31fbfd8c831 100644 --- a/ports/onnxruntime/fix-cmake.patch +++ b/ports/onnxruntime/fix-cmake.patch @@ -1,18 +1,25 @@ diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt -index 755e20f..d668069 100644 +index 8edbb6a..9e6c664 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt -@@ -572,7 +572,8 @@ get_filename_component(ORTTRAINING_ROOT "${ORTTRAINING_ROOT}" ABSOLUTE) +@@ -38,6 +38,7 @@ include(CheckLanguage) + include(CMakeDependentOption) + include(FetchContent) + include(CheckFunctionExists) ++include(GNUInstallDirs) # onnxruntime_providers_* require CMAKE_INSTALL_* variables + + # TODO: update this once all system adapt c++20 + if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") +@@ -583,7 +584,7 @@ get_filename_component(ORTTRAINING_ROOT "${ORTTRAINING_ROOT}" ABSOLUTE) get_filename_component(REPO_ROOT "${REPO_ROOT}" ABSOLUTE) set(ONNXRUNTIME_INCLUDE_DIR ${REPO_ROOT}/include/onnxruntime) -include(external/onnxruntime_external_deps.cmake) -+include(external/onnxruntime_vcpkg_deps.cmake) # see onnxruntime/portfile.cmake -+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/external") ++include(external/onnxruntime_vcpkg_deps.cmake) set(ORT_WARNING_FLAGS) if (WIN32) -@@ -904,7 +905,7 @@ function(onnxruntime_set_compile_flags target_name) +@@ -956,7 +957,7 @@ function(onnxruntime_set_compile_flags target_name) target_compile_definitions(${target_name} PRIVATE ORT_NEURAL_SPEED) endif() @@ -21,35 +28,7 @@ index 755e20f..d668069 100644 if (onnxruntime_USE_CUDA) # Suppress a "conversion_function_not_usable" warning in gsl/span target_compile_options(${target_name} PRIVATE "$<$:SHELL:-Xcudafe \"--diag_suppress=conversion_function_not_usable\">") -@@ -967,7 +968,7 @@ function(onnxruntime_set_compile_flags target_name) - target_compile_options(${target_name} PRIVATE "-Wno-unused-parameter") - endif() - target_compile_definitions(${target_name} PUBLIC -DNSYNC_ATOMIC_CPP11) -- onnxruntime_add_include_to_target(${target_name} nsync::nsync_cpp) -+ onnxruntime_add_include_to_target(${target_name} nsync::nsync_cpp Eigen3::Eigen) - endif() - foreach(ORT_FLAG ${ORT_PROVIDER_FLAGS}) - target_compile_definitions(${target_name} PRIVATE ${ORT_FLAG}) -@@ -1098,9 +1099,7 @@ endfunction() - function(onnxruntime_add_include_to_target dst_target) - foreach(src_target ${ARGN}) - if(TARGET ${src_target}) -- target_include_directories(${dst_target} PRIVATE $) -- target_compile_definitions(${dst_target} PRIVATE $) -- target_sources(${dst_target} PRIVATE $) -+ target_link_libraries(${dst_target} PRIVATE ${src_target}) - endif() - endforeach() - endfunction() -@@ -1721,6 +1720,7 @@ if (WIN32 AND NOT GDK_PLATFORM) - endif() - endif() - -+include(GNUInstallDirs) # some targets requires this... - foreach(target_name ${ONNXRUNTIME_CMAKE_FILES}) - include(${target_name}.cmake) - endforeach() -@@ -1767,20 +1767,16 @@ if (onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS) +@@ -1801,20 +1802,16 @@ if (onnxruntime_ENABLE_EXTERNAL_CUSTOM_OP_SCHEMAS) ) endif() @@ -73,12 +52,12 @@ index 755e20f..d668069 100644 write_basic_package_version_file( diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake -index c900f4d..8d5fcd6 100644 +index e15c8a0..598a38c 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake -@@ -303,6 +303,9 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) - # If it's an onnxruntime library, extract .o files to a separate directory for each library to avoid any clashes - # with filenames (e.g. utils.o) +@@ -308,6 +308,9 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) + # If it's an onnxruntime library, extract .o files from the original cmake build path to a separate directory for + # each library to avoid any clashes with filenames (e.g. utils.o) foreach(_LIB ${onnxruntime_INTERNAL_LIBRARIES} ) + if(NOT TARGET ${_LIB}) # if we didn't build from source. it may not a target + continue() @@ -86,7 +65,7 @@ index c900f4d..8d5fcd6 100644 GET_TARGET_PROPERTY(_LIB_TYPE ${_LIB} TYPE) if(_LIB_TYPE STREQUAL "STATIC_LIBRARY") set(CUR_STATIC_LIB_OBJ_DIR ${STATIC_LIB_TEMP_DIR}/$) -@@ -317,6 +320,9 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) +@@ -338,6 +341,9 @@ if(onnxruntime_BUILD_APPLE_FRAMEWORK) # for external libraries we create a symlink to the .a file foreach(_LIB ${onnxruntime_EXTERNAL_LIBRARIES}) @@ -97,10 +76,10 @@ index c900f4d..8d5fcd6 100644 if(_LIB_TYPE STREQUAL "STATIC_LIBRARY") add_custom_command(TARGET onnxruntime POST_BUILD diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake -index 43d5fa9..c441304 100644 +index 69d8f5f..374e439 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake -@@ -215,7 +215,7 @@ endif() +@@ -212,7 +212,7 @@ endif() if (NOT onnxruntime_BUILD_SHARED_LIB) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/common DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core) @@ -164,10 +143,10 @@ index 4d51325..fba1a68 100644 LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake -index 17de2aa..9b284ac 100644 +index 682dcfc..107a8b6 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake -@@ -632,7 +632,7 @@ if (WIN32) +@@ -651,7 +651,7 @@ if (PLATFORM_NAME STREQUAL "macabi") endif() if (NOT onnxruntime_BUILD_SHARED_LIB) @@ -176,7 +155,7 @@ index 17de2aa..9b284ac 100644 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -@@ -655,7 +655,7 @@ block() +@@ -674,7 +674,7 @@ block() endblock() @@ -198,43 +177,8 @@ index f15d5b8..c994fce 100644 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -diff --git a/cmake/onnxruntime_providers_coreml.cmake b/cmake/onnxruntime_providers_coreml.cmake -index aa8c355..fe6d397 100644 ---- a/cmake/onnxruntime_providers_coreml.cmake -+++ b/cmake/onnxruntime_providers_coreml.cmake -@@ -14,7 +14,7 @@ - "${COREML_PROTO_ROOT}/*.proto" - ) - onnxruntime_add_static_library(onnxruntime_coreml_proto ${coreml_proto_srcs}) -- target_include_directories(onnxruntime_coreml_proto PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}") -+ target_include_directories(onnxruntime_coreml_proto PUBLIC $ PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") - target_compile_definitions(onnxruntime_coreml_proto PUBLIC $) - set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility=hidden") - set_target_properties(onnxruntime_coreml_proto PROPERTIES COMPILE_FLAGS "-fvisibility-inlines-hidden") -@@ -27,7 +27,7 @@ - ) - - if (NOT onnxruntime_BUILD_SHARED_LIB) -- install(TARGETS onnxruntime_coreml_proto -+ install(TARGETS onnxruntime_coreml_proto EXPORT ${PROJECT_NAME}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -@@ -99,9 +99,9 @@ - set_target_properties(onnxruntime_providers_coreml PROPERTIES LINKER_LANGUAGE CXX) - - if (NOT onnxruntime_BUILD_SHARED_LIB) -- install(TARGETS onnxruntime_providers_coreml -+ install(TARGETS onnxruntime_providers_coreml EXPORT ${PROJECT_NAME}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR}) -- endif() -\ No newline at end of file -+ endif() diff --git a/cmake/onnxruntime_providers_cpu.cmake b/cmake/onnxruntime_providers_cpu.cmake -index b81a5c7..6f01091 100644 +index b211c02..f97b68b 100644 --- a/cmake/onnxruntime_providers_cpu.cmake +++ b/cmake/onnxruntime_providers_cpu.cmake @@ -60,7 +60,7 @@ if(NOT onnxruntime_DISABLE_CONTRIB_OPS) @@ -255,7 +199,7 @@ index b81a5c7..6f01091 100644 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -@@ -268,9 +268,9 @@ if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD +@@ -268,7 +268,7 @@ if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD endif() if (NOT onnxruntime_BUILD_SHARED_LIB) @@ -264,59 +208,8 @@ index b81a5c7..6f01091 100644 ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR}) --endif() -\ No newline at end of file -+endif() -diff --git a/cmake/onnxruntime_providers_nnapi.cmake b/cmake/onnxruntime_providers_nnapi.cmake -index 5ac25a3..cae13aa 100644 ---- a/cmake/onnxruntime_providers_nnapi.cmake -+++ b/cmake/onnxruntime_providers_nnapi.cmake -@@ -76,9 +76,9 @@ - endif() - - if (NOT onnxruntime_BUILD_SHARED_LIB) -- install(TARGETS onnxruntime_providers_nnapi -+ install(TARGETS onnxruntime_providers_nnapi EXPORT ${PROJECT_NAME}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR}) -- endif() -\ No newline at end of file -+ endif() -diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake -index 5876b2b..cfca76b 100644 ---- a/cmake/onnxruntime_providers_openvino.cmake -+++ b/cmake/onnxruntime_providers_openvino.cmake -@@ -42,8 +42,8 @@ - target_compile_options(onnxruntime_providers_openvino PRIVATE "-Wno-parentheses") - endif() - add_dependencies(onnxruntime_providers_openvino onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) -- target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${eigen_INCLUDE_DIRS} ${OpenVINO_INCLUDE_DIR} ${OPENVINO_INCLUDE_DIR_LIST} ${PYTHON_INCLUDE_DIRS} $ENV{OPENCL_INCS} $ENV{OPENCL_INCS}/../../cl_headers/) -- target_link_libraries(onnxruntime_providers_openvino ${ONNXRUNTIME_PROVIDERS_SHARED} Boost::mp11 ${OPENVINO_LIB_LIST} ${ABSEIL_LIBS}) -+ target_include_directories(onnxruntime_providers_openvino SYSTEM PRIVATE ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${eigen_INCLUDE_DIRS} ${OpenVINO_INCLUDE_DIR} ${OPENVINO_INCLUDE_DIR_LIST} ${PYTHON_INCLUDE_DIRS}) -+ target_link_libraries(onnxruntime_providers_openvino PUBLIC ${ONNXRUNTIME_PROVIDERS_SHARED} Boost::mp11 ${OPENVINO_LIB_LIST} ${ABSEIL_LIBS}) - - target_compile_definitions(onnxruntime_providers_openvino PRIVATE VER_MAJOR=${VERSION_MAJOR_PART}) - target_compile_definitions(onnxruntime_providers_openvino PRIVATE VER_MINOR=${VERSION_MINOR_PART}) -@@ -72,12 +72,12 @@ - endif() - - if (CMAKE_OPENVINO_LIBRARY_INSTALL_DIR) -- install(TARGETS onnxruntime_providers_openvino -+ install(TARGETS onnxruntime_providers_openvino EXPORT ${PROJECT_NAME}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_OPENVINO_LIBRARY_INSTALL_DIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) - else() -- install(TARGETS onnxruntime_providers_openvino -+ install(TARGETS onnxruntime_providers_openvino EXPORT ${PROJECT_NAME}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake -index 79bee3b..c49653b 100644 +index 79bee3b..9f5c400 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -63,7 +63,7 @@ endif() diff --git a/ports/onnxruntime/fix-onnxruntime-pr-19966.patch b/ports/onnxruntime/fix-onnxruntime-pr-19966.patch deleted file mode 100644 index 874965d774dd7d..00000000000000 --- a/ports/onnxruntime/fix-onnxruntime-pr-19966.patch +++ /dev/null @@ -1,2284 +0,0 @@ -diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt -index 02b568abdf8dad08b142c87418a4111340c710d8..6700c96d84cb411ce3100cb75a38babf6a3ac65b 100644 ---- a/cmake/CMakeLists.txt -+++ b/cmake/CMakeLists.txt -@@ -1290,34 +1290,6 @@ if (onnxruntime_USE_OPENVINO) - - add_definitions(-DUSE_OPENVINO=1) - -- if (EXISTS "$ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/version.txt") -- file(READ $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/version.txt VER) -- endif() -- -- if (NOT DEFINED ENV{INTEL_OPENVINO_DIR}) -- message(FATAL_ERROR "[Couldn't locate OpenVINO] OpenVINO may not have been initialized") -- endif() -- -- # Check OpenVINO version for support -- if ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.0") -- set(OPENVINO_VERSION "2023.0") -- add_definitions(-DOPENVINO_2023_0=1) -- elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.1") -- set(OPENVINO_VERSION "2023.1") -- add_definitions(-DOPENVINO_2023_1=1) -- elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.2") -- set(OPENVINO_VERSION "2023.2") -- add_definitions(-DOPENVINO_2023_2=1) -- elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "2023.3") -- set(OPENVINO_VERSION "2023.3") -- add_definitions(-DOPENVINO_2023_3=1) -- elseif ($ENV{INTEL_OPENVINO_DIR} MATCHES "openvino") -- set(OPENVINO_VERSION "2023.3") -- add_definitions(-DOPENVINO_2023_3=1) -- else() -- message(FATAL_ERROR "Unsupported OpenVINO version: ${INTEL_OPENVINO_DIR}") -- endif() -- - if (onnxruntime_USE_OPENVINO_GPU_FP32) - add_definitions(-DOPENVINO_CONFIG_GPU_FP32=1) - endif() -@@ -1334,6 +1306,10 @@ if (onnxruntime_USE_OPENVINO) - add_definitions(-DOPENVINO_CONFIG_CPU_FP16=1) - endif() - -+ if (onnxruntime_USE_OPENVINO_NPU) -+ add_definitions(-DOPENVINO_CONFIG_NPU=1) -+ endif() -+ - if (onnxruntime_USE_OPENVINO_GPU_FP32_NP) - add_definitions(-DOPENVINO_CONFIG_GPU_FP32=1) - add_definitions(-DOPENVINO_DISABLE_GRAPH_PARTITION=1) -@@ -1354,6 +1330,11 @@ if (onnxruntime_USE_OPENVINO) - add_definitions(-DOPENVINO_DISABLE_GRAPH_PARTITION=1) - endif() - -+ if (onnxruntime_USE_OPENVINO_NPU_NP) -+ add_definitions(-DOPENVINO_CONFIG_NPU=1) -+ add_definitions(-DOPENVINO_DISABLE_GRAPH_PARTITION=1) -+ endif() -+ - if (onnxruntime_USE_OPENVINO_HETERO) - add_definitions(-DOPENVINO_CONFIG_HETERO=1) - add_definitions(-DDEVICE_NAME="${onnxruntime_USE_OPENVINO_DEVICE}") -diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake -index e26f0bfc0b75183dd490bcebfbe4042055ae35f3..5876b2b5c448bcafdbd067f408be6b6b1993a3f5 100644 ---- a/cmake/onnxruntime_providers_openvino.cmake -+++ b/cmake/onnxruntime_providers_openvino.cmake -@@ -16,23 +16,19 @@ - endif() - - # Header paths -- find_package(InferenceEngine REQUIRED) -- find_package(ngraph REQUIRED) -- -- if (OPENVINO_2022_1 OR OPENVINO_2022_2) - find_package(OpenVINO REQUIRED COMPONENTS Runtime ONNX) -- list (OV_20_LIBS openvino::frontend::onnx openvino::runtime) -+ if(OpenVINO_VERSION VERSION_LESS 2023.0) -+ message(FATAL_ERROR "OpenVINO 2023.0 and newer are supported. Please, latest OpenVINO release") - endif() - - if (WIN32) - unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) - endif() - -+ list(APPEND OPENVINO_LIB_LIST openvino::frontend::onnx openvino::runtime ${PYTHON_LIBRARIES}) - if ((DEFINED ENV{OPENCL_LIBS}) AND (DEFINED ENV{OPENCL_INCS})) - add_definitions(-DIO_BUFFER_ENABLED=1) -- list(APPEND OPENVINO_LIB_LIST $ENV{OPENCL_LIBS} ${OV_20_LIBS} ${InferenceEngine_LIBRARIES} ${NGRAPH_LIBRARIES} ngraph::onnx_importer ${PYTHON_LIBRARIES}) -- else() -- list(APPEND OPENVINO_LIB_LIST ${OV_20_LIBS} ${InferenceEngine_LIBRARIES} ${NGRAPH_LIBRARIES} ngraph::onnx_importer ${PYTHON_LIBRARIES}) -+ list(APPEND OPENVINO_LIB_LIST $ENV{OPENCL_LIBS}) - endif() - - source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_openvino_cc_srcs}) -@@ -75,7 +71,14 @@ - message(FATAL_ERROR "onnxruntime_providers_openvino unknown platform, need to specify shared library exports for it") - endif() - -- install(TARGETS onnxruntime_providers_openvino -- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -- RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -\ No newline at end of file -+ if (CMAKE_OPENVINO_LIBRARY_INSTALL_DIR) -+ install(TARGETS onnxruntime_providers_openvino -+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -+ LIBRARY DESTINATION ${CMAKE_OPENVINO_LIBRARY_INSTALL_DIR} -+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+ else() -+ install(TARGETS onnxruntime_providers_openvino -+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -+ endif() -diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino -index 78d04a51ba162a2fd36425fb519d27878e335204..049916fac92f118722aba9ae6ab4cb734030ac6a 100644 ---- a/dockerfiles/Dockerfile.openvino -+++ b/dockerfiles/Dockerfile.openvino -@@ -1,9 +1,9 @@ - #------------------------------------------------------------------------- --# Copyright(C) 2021-2023 Intel Corporation. -+# Copyright(C) 2021-2024 Intel Corporation. - # SPDX-License-Identifier: MIT - #-------------------------------------------------------------------------- - --ARG OPENVINO_VERSION=2023.0.0 -+ARG OPENVINO_VERSION=2024.0.0 - - - # Build stage -@@ -17,7 +17,7 @@ ARG DEVICE=CPU_FP32 - ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git - ARG ONNXRUNTIME_BRANCH=main - --ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake -+ENV OpenVINO_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake - - USER root - RUN apt update; apt install -y git protobuf-compiler libprotobuf-dev -diff --git a/dockerfiles/Dockerfile.openvino-centos7 b/dockerfiles/Dockerfile.openvino-centos7 -deleted file mode 100755 -index 697db44801e3bc5b87d187a71bec797e977b2e4d..0000000000000000000000000000000000000000 ---- a/dockerfiles/Dockerfile.openvino-centos7 -+++ /dev/null -@@ -1,105 +0,0 @@ --#------------------------------------------------------------------------- --# Copyright(C) 2021 Intel Corporation. --# SPDX-License-Identifier: MIT --#-------------------------------------------------------------------------- -- --FROM centos:7.8.2003 -- --WORKDIR /code -- --ARG MY_ROOT=/code --ARG YUM_OV_PACKAGE=intel-openvino-runtime-centos7-2021.4.752.x86_64 --ARG DEVICE=CPU_FP32 --ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime --ARG ONNXRUNTIME_BRANCH=main -- --ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.4.752 --ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/share --ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/lib/intel64 --ENV ngraph_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/ngraph/cmake --ENV LD_LIBRARY_PATH=/opt/intel/opencl:${INTEL_OPENVINO_DIR}/inference_engine/external/gna/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/mkltiny_lnx/lib:$INTEL_OPENVINO_DIR/deployment_tools/ngraph/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/omp/lib:${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/tbb/lib:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH} --ENV OpenCV_DIR=${INTEL_OPENVINO_DIR}/opencv/share/OpenCV --ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/opencv/lib:${INTEL_OPENVINO_DIR}/opencv/share/OpenCV/3rdparty/lib:${LD_LIBRARY_PATH} --ENV HDDL_INSTALL_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl --ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl/lib:$LD_LIBRARY_PATH --ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64:/lib64:$LD_LIBRARY_PATH -- --# Install packages --RUN yum update -y && \ -- yum groupinstall "Development Tools" -y && \ -- yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-devel libffi-devel openssl-devel boost-devel-1.53.0 && \ -- yum clean packages && yum clean all && rm -rf /var/cache/yum && \ --# Install cmake -- cd $MY_ROOT && \ -- wget https://github.com/Kitware/CMake/releases/download/v3.27.3/cmake-3.27.3.tar.gz && \ -- tar -zxvf cmake-3.27.3.tar.gz && rm -rf cmake-3.27.3.tar.gz && \ -- cd cmake-3.27.3 && \ -- ./bootstrap && \ -- make && \ -- make install && \ -- cd $MY_ROOT && \ --# libusb1.0.22 -- cd /opt/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \ -- unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /opt/libusb-1.0.22 && \ --# bootstrap steps -- ./bootstrap.sh && \ -- ./configure --disable-udev --enable-shared && \ -- make -j4 && \ --# configure libusb1.0.22 -- cd /opt/libusb-1.0.22/libusb && \ -- /bin/mkdir -p '/usr/local/lib' && \ -- /bin/bash ../libtool --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \ -- /bin/mkdir -p '/usr/local/include/libusb-1.0' && \ -- /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \ -- /bin/mkdir -p '/usr/local/lib/pkgconfig' && \ --# Install openvino -- yum-config-manager --add-repo https://yum.repos.intel.com/openvino/2021/setup/intel-openvino-2021.repo && \ -- rpm --import https://yum.repos.intel.com/openvino/2021/setup/RPM-GPG-KEY-INTEL-OPENVINO-2021 && \ -- yum update -y && yum list intel-openvino* && \ -- yum install -y $YUM_OV_PACKAGE && \ -- cd ${INTEL_OPENVINO_DIR}/install_dependencies/ && ./install_openvino_dependencies.sh -y && \ -- printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2021.4.752/bin/setupvars.sh && \ -- cd /opt/libusb-1.0.22 && \ -- /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \ -- cp /opt/intel/openvino_2021/deployment_tools/inference_engine/external/97-myriad-usbboot.rules /etc/udev/rules.d/ && \ -- ldconfig && \ --# Install GPU runtime and drivers -- cd ${MY_ROOT} && \ -- mkdir /tmp/opencl && \ -- cd /tmp/opencl && \ -- yum install -y epel-release && \ -- yum install -y ocl-icd ocl-icd-devel && \ -- wget -O intel-igc-core-1.0.2597-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-core-1.0.2597-1.el7.x86_64.rpm/download && \ -- wget -O intel-opencl-19.41.14441-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-opencl-19.41.14441-1.el7.x86_64.rpm/download && \ -- wget -O intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-devel-1.0.2597-1.el7.x86_64.rpm/download && \ -- wget -O intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-igc-opencl-1.0.2597-1.el7.x86_64.rpm/download && \ -- wget -O intel-gmmlib-19.3.2-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-19.3.2-1.el7.x86_64.rpm/download && \ -- wget -O intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm https://sourceforge.net/projects/intel-compute-runtime/files/19.41.14441/centos-7/intel-gmmlib-devel-19.3.2-1.el7.x86_64.rpm/download && \ -- rpm -i /tmp/opencl/*.rpm && \ -- ldconfig && \ -- rm -rf /tmp/opencl && \ --# Installing gcc-10 -- yum install -y centos-release-scl && \ -- yum install -y devtoolset-10-gcc* && \ -- echo 'source scl_source enable devtoolset-10' >> ~/.bashrc && \ --# python installation -- source scl_source enable devtoolset-10 && \ -- cd /code/ && \ -- wget https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz && tar xvf Python-3.8.3.tgz && \ -- cd Python-3.8*/ && ./configure && make && make install && \ -- cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.3/ /usr/bin/Python38 && \ --# installing dependancies -- yum install -y python3-lxml python3-six libusb.x86_64 && \ -- yum clean packages && yum clean all && rm -rf /var/cache/yum && \ --# Build onnxruntime -- cd $MY_ROOT && \ -- pip3 install numpy wheel setuptools cython && \ -- git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} && \ -- pip3 install onnx && \ -- cd /code/onnxruntime && ./build.sh --allow_running_as_root --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel && \ -- pip3 install /code/onnxruntime/build/Linux/Release/dist/*-linux_x86_64.whl && \ --# Clean up -- cd $MY_ROOT && rm -rf onnxruntime Python-3* && \ -- cd ${MY_ROOT}/ && rm -rf cmake* && \ -- cd /usr/share/ && rm -rf gcc* && cd /usr/lib/ && rm -rf gcc cd && rm -rf .cache && \ -- cd ${INTEL_OPENVINO_DIR}/ && rm -rf documentation data_processing && cd deployment_tools/ && rm -rf tools -diff --git a/dockerfiles/Dockerfile.openvino-csharp b/dockerfiles/Dockerfile.openvino-csharp -deleted file mode 100644 -index 2529ef4b7320925b9ebeb66e3b5209b1c1623979..0000000000000000000000000000000000000000 ---- a/dockerfiles/Dockerfile.openvino-csharp -+++ /dev/null -@@ -1,90 +0,0 @@ --#------------------------------------------------------------------------- --# Copyright(C) 2021-2023 Intel Corporation. --# SPDX-License-Identifier: MIT --#-------------------------------------------------------------------------- -- --ARG OPENVINO_VERSION=2023.0.0 -- --# Build stage --FROM openvino/ubuntu20_runtime:${OPENVINO_VERSION} AS base -- --ENV WORKDIR_PATH=/home/openvino --WORKDIR $WORKDIR_PATH --ENV DEBIAN_FRONTEND noninteractive -- --USER root --RUN apt update; apt install -y --no-install-recommends wget gnupg && \ -- rm -rf /var/lib/apt/lists/* -- --# Install Mono --RUN wget http://download.mono-project.com/repo/xamarin.gpg && apt-key add xamarin.gpg && rm xamarin.gpg && \ -- echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | tee /etc/apt/sources.list.d/mono-official-stable.list && \ -- apt update -y && \ -- apt install -y mono-devel -- --# Install nuget.exe --RUN wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe && \ -- mv nuget.exe /usr/local/bin/nuget.exe && \ -- echo 'mono /usr/local/bin/nuget.exe $@' > /usr/local/bin/nuget && \ -- chmod a+x /usr/local/bin/nuget -- --# Install .NET core --RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \ -- dpkg -i packages-microsoft-prod.deb && \ -- apt-get update -y &&\ -- apt-get install -y apt-transport-https && \ -- apt-get update -y && \ -- apt-get install -y dotnet-sdk-5.0 -- --# Build stage --FROM base AS builder -- --ENV WORKDIR_PATH=/home/openvino --WORKDIR $WORKDIR_PATH --ENV DEBIAN_FRONTEND noninteractive -- --ARG DEVICE=CPU_FP32 --ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git --ARG ONNXRUNTIME_BRANCH=main -- --ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake --ENV LANG en_US.UTF-8 -- --USER root --RUN apt update; apt install -y --no-install-recommends git protobuf-compiler libprotobuf-dev ca-certificates unattended-upgrades && \ -- unattended-upgrade && \ -- rm -rf /var/lib/apt/lists/* -- --RUN git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} --RUN /bin/sh onnxruntime/dockerfiles/scripts/install_common_deps.sh --RUN ln -s cmake-* cmake-dir --RUN python3 -m pip install wheel --ENV PATH=${WORKDIR_PATH}/cmake-dir/bin:$PATH --RUN pip3 install onnx --RUN ln -s /usr/bin/python3 /usr/bin/python --RUN apt install locales && \ -- locale-gen en_US en_US.UTF-8 && \ -- dpkg-reconfigure locales --RUN cd onnxruntime && ./build.sh --allow_running_as_root --config Release --update --build --parallel --use_openvino ${DEVICE} --build_nuget --build_shared_lib --RUN cp /home/openvino/onnxruntime/build/Linux/Release/Microsoft.ML.OnnxRuntime.Managed* /home/openvino/onnxruntime/build/Linux/Release/nuget-artifacts -- --# Deploy stage --FROM base -- --ENV DEBIAN_FRONTEND noninteractive --USER root -- --RUN apt update; apt install -y unattended-upgrades fonts-freefont-ttf && \ -- unattended-upgrade --ARG BUILD_UID=1001 --ARG BUILD_USER=onnxruntimedev --RUN adduser --uid $BUILD_UID $BUILD_USER --RUN usermod -a -G video,users ${BUILD_USER} --ENV WORKDIR_PATH /home/${BUILD_USER} --WORKDIR ${WORKDIR_PATH} --COPY --from=builder /home/openvino/onnxruntime/build/Linux/Release/nuget-artifacts ${WORKDIR_PATH}/nuget-artifacts -- --USER ${BUILD_USER} --ENV PATH=${WORKDIR_PATH}/miniconda/bin:${WORKDIR_PATH}/cmake-dir/bin:$PATH --ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/runtime/lib/intel64 --ENV LD_LIBRARY_PATH=/opt/intel/opencl:${INTEL_OPENVINO_DIR}/runtime/3rdparty/tbb/lib:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH} -diff --git a/dockerfiles/Dockerfile.openvino-rhel8 b/dockerfiles/Dockerfile.openvino-rhel8 -deleted file mode 100644 -index 5c504cfa553a17eb58b46c27d04fb2c5ff5d969d..0000000000000000000000000000000000000000 ---- a/dockerfiles/Dockerfile.openvino-rhel8 -+++ /dev/null -@@ -1,87 +0,0 @@ --# Build stage --FROM registry.access.redhat.com/ubi8/ubi:8.4 -- --WORKDIR /code -- --ARG MY_ROOT=/code --ARG DEVICE=CPU_FP32 --ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime --ARG ONNXRUNTIME_BRANCH=main -- --ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2022.3.0 -- --ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake --ENV IE_PLUGINS_PATH=${INTEL_OPENVINO_DIR}/runtime/lib/intel64/ --ENV ngraph_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake --ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/runtime/3rdparty/tbb/lib/:${IE_PLUGINS_PATH}:${LD_LIBRARY_PATH} --ENV OpenCV_DIR=${INTEL_OPENVINO_DIR}/extras/opencv/cmake --ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/extras/opencv/lib:${LD_LIBRARY_PATH} --ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64:/lib64:${LD_LIBRARY_PATH} --ENV PATH=${MY_ROOT}/cmake-dir/bin:$PATH -- --# Install packages --RUN yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-devel libffi-devel openssl-devel git make gcc && \ -- yum clean packages && yum clean all && rm -rf /var/cache/yum && \ --# Install python 3.8 -- cd $MY_ROOT && \ -- wget https://www.python.org/ftp/python/3.8.9/Python-3.8.9.tgz && tar xvf Python-3.8.9.tgz && rm -rf Python-3.8.9.tgz && \ -- cd Python-3.8*/ && ./configure && make && make install && \ -- cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.9/ /usr/bin/Python38 && ln -s /usr/bin/pip3 /usr/bin/pip && \ --# libusb1.0.22 -- cd /opt/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \ -- unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /opt/libusb-1.0.22 && \ --# bootstrap steps -- ./bootstrap.sh && \ -- ./configure --disable-udev --enable-shared && \ -- make -j4 && \ --# configure libusb1.0.22 -- cd /opt/libusb-1.0.22/libusb && \ -- /bin/mkdir -p '/usr/local/lib' && \ -- /bin/bash ../libtool --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \ -- /bin/mkdir -p '/usr/local/include/libusb-1.0' && \ -- /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \ -- /bin/mkdir -p '/usr/local/lib/pkgconfig' && \ --# Install openvino -- cd /opt/ && mkdir intel/ && cd intel && \ -- wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ -- tar xvf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ -- rm -rf l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz && \ -- mv l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64 openvino_2022.3.0 && \ -- cd ${INTEL_OPENVINO_DIR}/install_dependencies/ && ./install_openvino_dependencies.sh -y && ./install_NEO_OCL_driver.sh -y && \ -- printf "\nexport LD_LIBRARY_PATH=\${LD_LIBRARY_PATH}:/usr/local/lib\n" >> /opt/intel/openvino_2022.3.0/setupvars.sh && \ -- cd /opt/libusb-1.0.22 && \ -- /usr/bin/install -c -m 644 libusb-1.0.pc '/usr/local/lib/pkgconfig' && \ -- # MYRIAD plugins are not available for openvino 2022.3.0 release -- #cp /opt/intel/openvino_2022.3.0/install_dependencies/97-myriad-usbboot.rules /etc/udev/rules.d/ && \ -- ldconfig && \ --#Install protobuf -- cd $MY_ROOT && \ -- git clone https://github.com/protocolbuffers/protobuf.git && \ -- cd protobuf && \ -- git checkout v3.16.0 && \ -- git submodule update --init --recursive && \ -- mkdir build_source && cd build_source && \ -- cmake ../cmake -DCMAKE_INSTALL_LIBDIR=lib64 -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release && \ -- make -j$(nproc) && \ -- make install && \ --# Build onnxruntime -- cd $MY_ROOT && \ -- pip3 install numpy wheel setuptools cython onnx && \ -- git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} && \ -- bash onnxruntime/dockerfiles/scripts/install_common_deps.sh && \ -- ln -s cmake-* cmake-dir && \ -- source /opt/intel/openvino_2022.3.0/setupvars.sh && \ -- cd /code/onnxruntime && ./build.sh --allow_running_as_root --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel && \ -- pip3 install /code/onnxruntime/build/Linux/Release/dist/*-linux_x86_64.whl && \ --# Clean up -- cd ${MY_ROOT} && rm -rf onnxruntime && rm -rf Python-3.8.9 && rm -rf protobuf -- --# Deploy stage --ARG BUILD_UID=1001 --ARG BUILD_USER=onnxruntimedev --RUN adduser --uid $BUILD_UID $BUILD_USER --RUN usermod -a -G video,users,render ${BUILD_USER} --ENV WORKDIR_PATH /home/${BUILD_USER} -- --WORKDIR ${WORKDIR_PATH} --USER ${BUILD_USER} -diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc -index 330b464ffd1bb8804e0b9aa6084c0912931d7126..3252603e333897f8ba9ee5e26041201e77267d2b 100644 ---- a/onnxruntime/core/providers/openvino/backend_manager.cc -+++ b/onnxruntime/core/providers/openvino/backend_manager.cc -@@ -1,8 +1,9 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include - #include -+#include - - #include "core/providers/shared_library/provider_api.h" - #include "contexts.h" -@@ -24,15 +25,6 @@ BackendManager::BackendManager(const GlobalContext& global_context, - global_context_ = global_context; - - auto prec_str = GetGlobalContext().precision_str; -- if (prec_str == "FP32") { -- subgraph_context_.precision = "FP32"; -- } else if (prec_str == "FP16") { -- subgraph_context_.precision = "FP16"; -- } else if (prec_str == "U8") { -- subgraph_context_.precision = "U8"; -- } else { -- throw std::string("Invalid OpenVINO Precision type: " + prec_str); -- } - - // Save the indexes of graph inputs among fused_node's inputDefs - // (which also contains initializers). -@@ -47,7 +39,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, - for (auto input : graph_inputs) { - auto it = subgraph_context_.input_names.find(input->Name()); - if (it == subgraph_context_.input_names.end()) { -- throw std::string("Input not found in the input defs list"); -+ ORT_THROW("Input not found in the input defs list"); - } - int index = it->second; - subgraph_context_.input_indexes.push_back(index); -@@ -61,6 +53,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, - } - subgraph_context_.subgraph_name = fused_node.Name(); - model_proto_ = GetModelProtoFromFusedNode(fused_node, subgraph, logger); -+ std::string device_type = openvino_ep::BackendManager::GetGlobalContext().device_type; - - if (ModelHasSymbolicInputDims(subgraph)) { - subgraph_context_.has_dynamic_input_shape = true; -@@ -75,7 +68,7 @@ BackendManager::BackendManager(const GlobalContext& global_context, - GetGlobalContext(), - subgraph_context_); - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " - << "Backend created for graph " << subgraph_context_.subgraph_name; -@@ -87,12 +80,29 @@ BackendManager::BackendManager(const GlobalContext& global_context, - << subgraph_context_.subgraph_name; - - subgraph_context_.has_dynamic_input_shape = false; -+ -+ // OV NPU plugin is supported with fallback to OV CPU upon compilation failures. - try { - concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, - GetGlobalContext(), - subgraph_context_); -- } catch (std::string const& msg) { -- throw msg; -+ } catch (const OnnxRuntimeException& ex) { -+ if (device_type.find("NPU") != std::string::npos) { -+ LOGS_DEFAULT(WARNING) << ex.what(); -+ LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." -+ << "Falling back to OV CPU for execution"; -+ GetGlobalContext().device_type = "CPU"; -+ GetGlobalContext().precision_str = "FP32"; -+ try { -+ concrete_backend_ = BackendFactory::MakeBackend(*model_proto_, -+ GetGlobalContext(), -+ subgraph_context_); -+ } catch (std::string const& msg) { -+ ORT_THROW(msg); -+ } -+ } else { -+ ORT_THROW(ex.what()); -+ } - } - } - } -@@ -254,8 +264,13 @@ void BackendManager::Compute(OrtKernelContext* context) { - LOGS_DEFAULT(INFO) << "Start Compute"; - } - #endif -+ // OV NPU doesn't support dynamic shaped model inference. -+ // if disable_dynamic_shapes is set to true then execution of dynamic model is done -+ // by rewriting the model to static shaped model at runtime based on input shape. -+ // disable_dynamic_shapes is always set to true for OV NPU plugin. - bool use_dynamic_backend = true; -- if (!GetGlobalContext().disable_dynamic_shapes && subgraph_context_.has_dynamic_input_shape && -+ if (subgraph_context_.has_dynamic_input_shape && -+ !GetGlobalContext().disable_dynamic_shapes && - (GetGlobalContext().device_type.find("CPU") != std::string::npos || - GetGlobalContext().device_type.find("GPU") != std::string::npos)) { - concrete_backend_->Infer(context); -@@ -263,12 +278,11 @@ void BackendManager::Compute(OrtKernelContext* context) { - } else if (use_dynamic_backend && subgraph_context_.has_dynamic_input_shape) { - std::vector> tensor_shapes = GetInputTensorShapes(ctx); - auto key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type); -- - std::shared_ptr dynamic_backend; - auto search = backend_map_.find(key); - if (search == backend_map_.end()) { - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " -- << "Creating concrete backend for key: " << key; -+ << "Creating dynamic backend for key: " << key; - LOGS_DEFAULT(INFO) << "[OpenVINO-EP] " - << "Backend created for graph " << subgraph_context_.subgraph_name; - auto modelproto_with_concrete_shapes = ReWriteInputShapeInfo(*model_proto_, tensor_shapes); -@@ -276,8 +290,22 @@ void BackendManager::Compute(OrtKernelContext* context) { - dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes, - GetGlobalContext(), - subgraph_context_); -- } catch (std::string const& msg) { -- throw msg; -+ } catch (const OnnxRuntimeException& ex) { -+ if (GetGlobalContext().device_type.find("NPU") != std::string::npos) { -+ LOGS_DEFAULT(WARNING) << ex.what(); -+ LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU." -+ << "Falling back to OV CPU for execution"; -+ GetGlobalContext().device_type = "CPU"; -+ GetGlobalContext().precision_str = "FP32"; -+ key = MakeMapKeyString(tensor_shapes, GetGlobalContext().device_type); -+ try { -+ dynamic_backend = BackendFactory::MakeBackend(*modelproto_with_concrete_shapes, -+ GetGlobalContext(), -+ subgraph_context_); -+ } catch (std::string const& msg) { -+ ORT_THROW(msg); -+ } -+ } - } - backend_map_.insert({key, dynamic_backend}); - } else { -diff --git a/onnxruntime/core/providers/openvino/backend_manager.h b/onnxruntime/core/providers/openvino/backend_manager.h -index 59bda7ca640ee3c130fa1d2193110b1c320408c4..376ebea225a2bef7b285e9a6580ef7c03ae04076 100644 ---- a/onnxruntime/core/providers/openvino/backend_manager.h -+++ b/onnxruntime/core/providers/openvino/backend_manager.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -diff --git a/onnxruntime/core/providers/openvino/backend_utils.cc b/onnxruntime/core/providers/openvino/backend_utils.cc -index 50c839017df2a00ca18104596e40d1d4a5d35bd3..32b5ad7d5b66debf18faa4fddb22abd1f4ed3dc3 100644 ---- a/onnxruntime/core/providers/openvino/backend_utils.cc -+++ b/onnxruntime/core/providers/openvino/backend_utils.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include -@@ -11,12 +11,7 @@ - #include "core/providers/shared_library/provider_api.h" - #include "backend_utils.h" - --#if defined(OV_API_20) - using Exception = ov::Exception; --#else --using Exception = InferenceEngine::details::InferenceEngineException; --using WaitMode = InferenceEngine::IInferRequest::WaitMode; --#endif - - namespace onnxruntime { - namespace openvino_ep { -@@ -47,7 +42,6 @@ struct static_cast_int64 { - - std::shared_ptr - CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext& global_context, -- const SubGraphContext& subgraph_context, - std::map>& const_outputs_map) { - if (IsCILogEnabled()) { - std::cout << "CreateNgraphFunc" << std::endl; -@@ -55,28 +49,6 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext - const std::string model = model_proto.SerializeAsString(); - try { - auto cnn_network = global_context.ie_core.ReadModel(model, global_context.onnx_model_path_name); -- if ((subgraph_context.precision == "FP16") && -- (global_context.device_type.find("NPU") == std::string::npos)) { -- // FP16 transformations -- ov::pass::ConvertFP32ToFP16 pass_obj; -- pass_obj.run_on_model(cnn_network); -- cnn_network->validate_nodes_and_infer_types(); -- -- auto proc = ov::preprocess::PrePostProcessor(cnn_network); -- for (size_t i = 0; i < cnn_network->inputs().size(); i++) { -- if (cnn_network->inputs()[i].get_element_type() == ov::element::f16) { -- proc.input(i).tensor().set_element_type(ov::element::f32); -- proc.input(i).preprocess().convert_element_type(ov::element::f16); -- } -- } -- -- for (size_t i = 0; i < cnn_network->outputs().size(); i++) { -- if (cnn_network->outputs()[i].get_element_type() == ov::element::f16) { -- proc.output(i).postprocess().convert_element_type(ov::element::f32); -- } -- } -- cnn_network = proc.build(); -- } - - // Check for Constant Folding - if (!global_context.is_wholly_supported_graph) { -@@ -103,7 +75,7 @@ CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, const GlobalContext - #endif - return cnn_network; - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - } - -@@ -127,7 +99,7 @@ GetOutputTensor(Ort::KernelContext& context, size_t batch_size, - } - auto it = output_names.find(output_name); - if (it == output_names.end()) { -- throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX"); -+ ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); - } - int index = it->second; - return context.GetOutput(index, output_shape.get(), num_dims); -@@ -145,7 +117,7 @@ GetOutputTensor(Ort::KernelContext& context, - - auto it = output_names.find(output_name); - if (it == output_names.end()) { -- throw std::string(log_tag + "Output names mismatch between OpenVINO and ONNX"); -+ ORT_THROW(log_tag + "Output names mismatch between OpenVINO and ONNX"); - } - int index = it->second; - auto shape = node->get_shape(); -@@ -204,7 +176,7 @@ void FillOutputsWithConstantData(std::shared_ptr node, Ort::UnownedVal - break; - } - default: -- throw std::string(log_tag + "Unsupported output data type"); -+ ORT_THROW(log_tag + "Unsupported output data type"); - } - } - -@@ -232,7 +204,7 @@ void FillInputBlob(OVTensorPtr inputBlob, size_t batch_slice_idx, - auto tensor = context.GetInput(subgraph_context.input_names.at(input_name)); - auto mem_info = tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { -- throw std::string(log_tag + "IO Buffering is not enabled, Please enable Input on CPU"); -+ ORT_THROW(log_tag + "IO Buffering is not enabled, Please enable Input on CPU"); - } - // Copy input data into OpenVINO's input buffer - const char* tensor_data = tensor.GetTensorData(); -diff --git a/onnxruntime/core/providers/openvino/backend_utils.h b/onnxruntime/core/providers/openvino/backend_utils.h -index 82b0351e87da5d901e58249ae942d210fb4603de..93fa874774469214be3afafd3c46714046483707 100644 ---- a/onnxruntime/core/providers/openvino/backend_utils.h -+++ b/onnxruntime/core/providers/openvino/backend_utils.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -65,7 +65,6 @@ void FillOutputBlob(OVTensorPtr outputBlob, Ort::UnownedValue& output_tensor, - std::shared_ptr - CreateOVModel(const ONNX_NAMESPACE::ModelProto& model_proto, - const GlobalContext& global_context, -- const SubGraphContext& subgraph_context, - std::map>& const_outputs_map); - - void printPerformanceCounts(const std::vector& performanceMap, -diff --git a/onnxruntime/core/providers/openvino/backends/backend_factory.cc b/onnxruntime/core/providers/openvino/backends/backend_factory.cc -index c586dd8b38af98198476e474aa9315c58b016664..a0f4ce8f843b060c1c10e49f3f69c344323be096 100644 ---- a/onnxruntime/core/providers/openvino/backends/backend_factory.cc -+++ b/onnxruntime/core/providers/openvino/backends/backend_factory.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include -@@ -24,11 +24,11 @@ BackendFactory::MakeBackend(const ONNX_NAMESPACE::ModelProto& model_proto, - try { - concrete_backend_ = std::make_shared(model_proto, global_context, subgraph_context); - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - return concrete_backend_; - } else { -- throw std::string("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type); -+ ORT_THROW("[OpenVINO-EP] Backend factory error: Unknown backend type: " + type); - } - } - } // namespace openvino_ep -diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc -index 0779940983aea0133e67b08c7a90b5066a6b5b28..69d234a7c55efd95451cfb75f74c05edfda81da4 100644 ---- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc -+++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include -@@ -79,20 +79,20 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, - subgraph_context_.subgraph_name); - LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; - } else { -- ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); -+ ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.LoadNetwork( - ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name); - LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; - } - #endif - } else { -- ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); -+ ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_); - exe_network_ = global_context_.ie_core.LoadNetwork( - ie_cnn_network_, hw_target, device_config, subgraph_context_.subgraph_name); - LOGS_DEFAULT(INFO) << log_tag << "Loaded model to the plugin"; - } - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - - inferRequestsQueue_ = std::unique_ptr(new InferRequestsQueue(exe_network_, 1)); -@@ -125,21 +125,17 @@ void BasicBackend::PopulateConfigValue(ov::AnyMap& device_config) { - if (global_context_.device_type.find("NPU") != std::string::npos) { - std::pair device_property; - device_property = std::make_pair("NPU_COMPILER_TYPE", "DRIVER"); -+ -+ const std::string env_npu_compiler_type = onnxruntime::GetEnvironmentVar("ORT_OPENVINO_NPU_COMPILER_TYPE"); -+ if (!env_npu_compiler_type.empty()) { -+ device_property = std::make_pair("NPU_COMPILER_TYPE", env_npu_compiler_type); -+ } - device_config.emplace(ov::device::properties("NPU", device_property)); - } - } - - void BasicBackend::EnableCaching() { - if (!global_context_.cache_dir.empty()) { -- if (global_context_.is_wholly_supported_graph) { --#if defined(OPENVINO_2022_3) --#if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) || defined(__BORLANDC__) -- _putenv_s("OV_GPU_CACHE_MODEL", "1"); --#else -- setenv("OV_GPU_CACHE_MODEL", "1", 1); --#endif --#endif -- } - LOGS_DEFAULT(INFO) << log_tag << "Enables Caching"; - global_context_.ie_core.SetCache(global_context_.cache_dir); - } -@@ -162,7 +158,7 @@ void BasicBackend::EnableStreams() { - (global_context_.device_type.find("HETERO") != std::string::npos) || - (global_context_.device_type.find("AUTO") != std::string::npos)) { - if (global_context_.num_streams != 1) { -- throw(log_tag + "Cannot set NUM_STREAMS to " + std::to_string(global_context_.num_streams) + " for device " + global_context_.device_type); -+ ORT_THROW(log_tag + "Cannot set NUM_STREAMS to " + std::to_string(global_context_.num_streams) + " for device " + global_context_.device_type); - } - // Do nothing - } else { -@@ -198,9 +194,9 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque - if (input_names.find(onnx_input_name) != input_names.end()) { - input_name = onnx_input_name; - } else { -- throw(log_tag + -- "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + -- " doesn't exist in the list of OpenVINO input tensor names"); -+ ORT_THROW(log_tag + -+ "Input names mismatch between OpenVINO and ONNX. " + onnx_input_name + -+ " doesn't exist in the list of OpenVINO input tensor names"); - } - size_t batch_slice_idx = 0; - if (subgraph_context_.has_dynamic_input_shape && -@@ -232,14 +228,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque - try { - infer_request->SetTensor(input_name, tensor_ptr); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - } else { - OVTensorPtr graph_input_blob; - try { - graph_input_blob = infer_request->GetTensor(input_name); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - FillInputBlob(graph_input_blob, batch_slice_idx, input_name, context, subgraph_context_); - } -@@ -248,7 +244,7 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque - // Start Async inference - infer_request->StartAsync(); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - } - -@@ -274,10 +270,10 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe - if (input_names.find(onnx_input_name) != input_names.end()) { - input_name = onnx_input_name; - } else { -- throw(log_tag + -- "Input names mismatch between OpenVINO and ONNX. " + -- onnx_input_name + -- " doesn't exist in the list of OpenVINO input tensor names"); -+ ORT_THROW(log_tag + -+ "Input names mismatch between OpenVINO and ONNX. " + -+ onnx_input_name + -+ " doesn't exist in the list of OpenVINO input tensor names"); - } - input_idx++; - // Kernel Context Input Buffer -@@ -322,7 +318,7 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe - } - } - if (!output_name_found) { -- throw std::string( -+ ORT_THROW( - log_tag + - "Output names mismatch between OpenVINO and ONNX. [ONNX Output: ] " + - onnx_output_name + " doesn't exist in the list of OpenVINO output tensor names"); -@@ -344,7 +340,7 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe - try { - infer_request->SetTensor(output_name, tensor_ptr); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - } - } -@@ -352,7 +348,7 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe - // Start Async inference - infer_request->StartAsync(); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - } - #endif -@@ -382,17 +378,18 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe - } - } - if (!output_name_found) { -- throw(log_tag + -- "Output names mismatch between OpenVINO and ONNX. " -- "[ONNX Output: ] " + -- onnx_output_name + -- " doesn't exist in the " -- "list of OpenVINO output tensor names"); -+ ORT_THROW( -+ log_tag + -+ "Output names mismatch between OpenVINO and ONNX. " -+ "[ONNX Output: ] " + -+ onnx_output_name + -+ " doesn't exist in the " -+ "list of OpenVINO output tensor names"); - } - try { - graph_output_blob = infer_request->GetTensor(output_name); - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - size_t batch_size = 1; - auto output_tensor = -@@ -413,14 +410,14 @@ void BasicBackend::CompleteAsyncInference(Ort::KernelContext& context, OVInferRe - auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); - auto mem_info = output_tensor.GetTensorMemoryInfo(); - if (mem_info.GetAllocatorName() == OpenVINO_GPU) { -- throw(log_tag + "IO Buffering is not supported for constant subgraphs"); -+ ORT_THROW(log_tag + "IO Buffering is not supported for constant subgraphs"); - } else { - FillOutputsWithConstantData(node, output_tensor); - } - } - } - } catch (const char* msg) { -- throw(msg); -+ ORT_THROW(msg); - } - } - -@@ -440,7 +437,7 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { - auto output_tensor = GetOutputTensor(context, out_name, subgraph_context_.output_names, node); - FillOutputsWithConstantData(node, output_tensor); - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - } - // Get Output tensors -@@ -461,26 +458,26 @@ void BasicBackend::Infer(OrtKernelContext* ctx) { - try { - StartRemoteAsyncInference(context, infer_request); - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - } else { - try { - StartAsyncInference(context, infer_request); - } catch (std::string const& msg) { -- throw msg; -+ ORT_THROW(msg); - } - } - #else - try { - StartAsyncInference(context, infer_request); -- } catch (std::string const& msg) { -- throw msg; -+ } catch (const std::runtime_error& e) { -+ ORT_THROW(log_tag + " Exception at StartAsyncInference: " + e.what()); - } - #endif - try { - CompleteAsyncInference(context, infer_request); -- } catch (std::string const& msg) { -- throw msg; -+ } catch (const std::runtime_error& e) { -+ ORT_THROW(log_tag + " Exception at CompleteAsyncInference: " + e.what()); - } - - // Get Output tensors -diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.h b/onnxruntime/core/providers/openvino/backends/basic_backend.h -index aa96dadbf0e2d49563e52045029185a93fd6a976..3502f660bbb2094a7604f2c70cc52e44b649ead0 100644 ---- a/onnxruntime/core/providers/openvino/backends/basic_backend.h -+++ b/onnxruntime/core/providers/openvino/backends/basic_backend.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h -index 5f19c71683f24a9d897dc310a6756dd98704454e..8701d9f676ffdc6ae15d0897e966b2a894f3d6fb 100644 ---- a/onnxruntime/core/providers/openvino/contexts.h -+++ b/onnxruntime/core/providers/openvino/contexts.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -31,6 +31,7 @@ struct GlobalContext { - int onnx_opset_version; - void* context = 0; - bool use_api_2; -+ std::vector OpenVINO_Version = {}; // Ov Major and OV minor version from OV headers - }; - - // Holds context specific to subgraph. -@@ -44,7 +45,6 @@ struct SubGraphContext { - std::vector input_indexes; - std::unordered_map input_names; - std::unordered_map output_names; -- std::string precision; - }; - - } // namespace openvino_ep -diff --git a/onnxruntime/core/providers/openvino/ibackend.h b/onnxruntime/core/providers/openvino/ibackend.h -index 8aacce19c14d5909a2a8fac80903f06d5c9e6419..ece855c6167c6c1c1b7c6f9541e854d16b598bbf 100644 ---- a/onnxruntime/core/providers/openvino/ibackend.h -+++ b/onnxruntime/core/providers/openvino/ibackend.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc -index e3948cc94b348db80e60ad2bd14569f067ee5a38..913440d2fb6ea72a7ffd170f8eff0d38a20e38cf 100644 ---- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc -+++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include "core/providers/shared_library/provider_api.h" -@@ -6,6 +6,7 @@ - #include "contexts.h" - #include "backend_manager.h" - #include "ov_versions/capability.h" -+#include "openvino/core/version.hpp" - - #define MEMCPY_S(dest, src, destsz, srcsz) memcpy(dest, src, std::min(destsz, srcsz)) - -@@ -25,6 +26,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv - global_context_->enable_opencl_throttling = info.enable_opencl_throttling_; - global_context_->disable_dynamic_shapes = info.disable_dynamic_shapes_; - global_context_->num_of_threads = info.num_of_threads_; -+ global_context_->OpenVINO_Version = {OPENVINO_VERSION_MAJOR, OPENVINO_VERSION_MINOR}; - - // to check if target device is available - // using ie_core capability GetAvailableDevices to fetch list of devices plugged in -@@ -50,8 +52,7 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv - device_found = true; - break; - } -- if ((info.device_type_.find("NPU") != std::string::npos) && -- (info.precision_ == "FP16" || info.precision_ == "U8")) { -+ if (info.device_type_.find("NPU") != std::string::npos) { - device_found = true; - break; - } -@@ -113,27 +114,10 @@ OpenVINOExecutionProvider::GetCapability(const GraphViewer& graph_viewer, - global_context_->onnx_opset_version = - graph_viewer.DomainToVersionMap().at(kOnnxDomain); - --#if defined(OPENVINO_2023_0) - openvino_ep::GetCapability obj(graph_viewer, - global_context_->device_type, -- global_context_->precision_str, "V_2023_0"); -+ global_context_->precision_str); - result = obj.Execute(); --#elif defined(OPENVINO_2023_1) -- openvino_ep::GetCapability obj(graph_viewer, -- global_context_->device_type, -- global_context_->precision_str, "V_2023_1"); -- result = obj.Execute(); --#elif defined(OPENVINO_2023_2) -- openvino_ep::GetCapability obj(graph_viewer, -- global_context_->device_type, -- global_context_->precision_str, "V_2023_2"); -- result = obj.Execute(); --#elif defined(OPENVINO_2023_3) -- openvino_ep::GetCapability obj(graph_viewer, -- global_context_->device_type, -- global_context_->precision_str, "V_2023_3"); -- result = obj.Execute(); --#endif - - global_context_->is_wholly_supported_graph = obj.IsWhollySupportedGraph(); - -diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h -index b0c92828d8a38d9ee4303ffb64ae3dd3e93e967f..b0dc881c36f3320682556854bf56c6c6f53a37bb 100644 ---- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h -+++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -20,7 +20,7 @@ static void print_build_options() { - << "you want to build" - << std::endl; - std::cout << "The different hardware devices that can be added with HETERO/MULTI/AUTO build " -- << "are ['CPU','GPU']" -+ << "are ['CPU','GPU','NPU']" - << std::endl; - std::cout << "An example of how to specify the HETERO or MULTI or AUTO build type. " - << "Ex: HETERO:GPU,CPU Ex: MULTI:GPU,CPU Ex: AUTO:GPU,CPU" -@@ -48,7 +48,7 @@ static std::vector parseDevices(const std::string& device_string) { - print_build_options(); - ORT_THROW("Invalid device string: " + device_string); - } -- std::vector dev_options = {"CPU", "GPU"}; -+ std::vector dev_options = {"CPU", "GPU", "NPU"}; - for (std::string dev : devices) { - if (!std::count(dev_options.begin(), dev_options.end(), dev)) { - print_build_options(); -@@ -98,12 +98,9 @@ struct OpenVINOExecutionProviderInfo { - #elif defined OPENVINO_CONFIG_GPU_FP16 - device_type_ = "GPU"; - precision_ = "FP16"; --#elif defined OPENVINO_CONFIG_NPU_FP16 -+#elif defined OPENVINO_CONFIG_NPU - device_type_ = "NPU"; -- precision_ = "FP16"; --#elif defined OPENVINO_CONFIG_NPU_U8 -- device_type_ = "NPU"; -- precision_ = "U8"; -+ precision_ = ""; - #elif defined OPENVINO_CONFIG_HETERO || defined OPENVINO_CONFIG_MULTI || defined OPENVINO_CONFIG_AUTO - #ifdef DEVICE_NAME - #define DEVICE DEVICE_NAME -@@ -142,12 +139,9 @@ struct OpenVINOExecutionProviderInfo { - } else if (dev_type == "GPU.1_FP16") { - device_type_ = "GPU.1"; - precision_ = "FP16"; -- } else if (dev_type == "NPU_FP16") { -- device_type_ = "NPU"; -- precision_ = "FP16"; -- } else if (dev_type == "NPU_U8") { -+ } else if (dev_type == "NPU") { - device_type_ = "NPU"; -- precision_ = "U8"; -+ precision_ = ""; - } else if (dev_type.find("HETERO") == 0 || dev_type.find("MULTI") == 0) { - std::vector devices = parseDevices(dev_type); - precision_ = "FP16"; -diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc -index 068456777beceb900511f66fdae92fd5828b477f..17511c54aab860140eec58c5b03144be5ddb089e 100644 ---- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc -+++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include "core/providers/shared_library/provider_api.h" -@@ -78,7 +78,6 @@ struct OpenVINO_Provider : Provider { - // with this value at runtime. - bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU - // device (Reduces CPU Utilization when using GPU) -- bool disable_dynamic_shapes = false; // [disable_dynamic_shapes]: Execute model with default static shape for optimal performance. - void* context = nullptr; - - if (provider_options_map.find("device_type") != provider_options_map.end()) { -@@ -86,7 +85,7 @@ struct OpenVINO_Provider : Provider { - - std::set ov_supported_device_types = {"CPU_FP32", "CPU_FP16", "GPU_FP32", - "GPU.0_FP32", "GPU.1_FP32", "GPU_FP16", -- "GPU.0_FP16", "GPU.1_FP16"}; -+ "GPU.0_FP16", "GPU.1_FP16", "NPU"}; - if (!((ov_supported_device_types.find(device_type) != ov_supported_device_types.end()) || - (device_type.find("HETERO:") == 0) || - (device_type.find("MULTI:") == 0) || -@@ -94,7 +93,7 @@ struct OpenVINO_Provider : Provider { - ORT_THROW( - "[ERROR] [OpenVINO] You have selcted wrong configuration value for the key 'device_type'. " - "Select from 'CPU_FP32', 'CPU_FP16', 'GPU_FP32', 'GPU.0_FP32', 'GPU.1_FP32', 'GPU_FP16', " -- "'GPU.0_FP16', 'GPU.1_FP16' or from" -+ "'GPU.0_FP16', 'GPU.1_FP16', 'NPU' or from" - " HETERO/MULTI/AUTO options available. \n"); - } - } -@@ -147,12 +146,24 @@ struct OpenVINO_Provider : Provider { - bool_flag = ""; - } - -+ // [disable_dynamic_shapes]: Rewrite dynamic shaped models to static shape at runtime and execute. -+ // Always true for NPU plugin. -+ bool disable_dynamic_shapes = false; -+ if (device_type.find("NPU") != std::string::npos) { -+ disable_dynamic_shapes = true; -+ } - if (provider_options_map.find("disable_dynamic_shapes") != provider_options_map.end()) { - bool_flag = provider_options_map.at("disable_dynamic_shapes"); - if (bool_flag == "true" || bool_flag == "True") - disable_dynamic_shapes = true; -- else if (bool_flag == "false" || bool_flag == "False") -- disable_dynamic_shapes = false; -+ else if (bool_flag == "false" || bool_flag == "False") { -+ if (device_type.find("NPU") != std::string::npos) { -+ disable_dynamic_shapes = true; -+ LOGS_DEFAULT(INFO) << "[OpenVINO-EP] The value for the key 'disable_dynamic_shapes' will be set to TRUE for NPU backend.\n "; -+ } else { -+ disable_dynamic_shapes = false; -+ } -+ } - } - return std::make_shared(const_cast(device_type.c_str()), - enable_npu_fast_compile, -diff --git a/onnxruntime/core/providers/openvino/ov_interface.cc b/onnxruntime/core/providers/openvino/ov_interface.cc -index ea481791111fc42d866b72608d2332cdc1ff2a2c..d7c6654c90f812117b6b75671ebacb9d3697fdee 100644 ---- a/onnxruntime/core/providers/openvino/ov_interface.cc -+++ b/onnxruntime/core/providers/openvino/ov_interface.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include "ov_interface.h" -@@ -8,12 +8,7 @@ - #include "core/providers/shared_library/provider_api.h" - #include "backend_utils.h" - --#if defined(OV_API_20) - using Exception = ov::Exception; --#else --using Exception = InferenceEngine::details::InferenceEngineException; --using WaitMode = InferenceEngine::IInferRequest::WaitMode; --#endif - - namespace onnxruntime { - namespace openvino_ep { -@@ -36,9 +31,9 @@ std::shared_ptr OVCore::ReadModel(const std::string& model, const std - } - return FE->convert(inputModel); - } catch (const Exception& e) { -- throw std::string(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what())); -+ ORT_THROW(log_tag + "[OpenVINO-EP] Exception while Reading network: " + std::string(e.what())); - } catch (...) { -- throw std::string(log_tag + "[OpenVINO-EP] Unknown exception while Reading network"); -+ ORT_THROW(log_tag + "[OpenVINO-EP] Unknown exception while Reading network"); - } - } - -@@ -81,9 +76,9 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& ie_cnn_network, - OVExeNetwork exe(obj); - return exe; - } catch (const Exception& e) { -- throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what()); -+ ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what()); - } catch (...) { -- throw std::string(log_tag + " Exception while Loading Network for graph " + name); -+ ORT_THROW(log_tag + " Exception while Loading Network for graph " + name); - } - } - -@@ -113,9 +108,9 @@ OVExeNetwork OVCore::LoadNetwork(std::shared_ptr& model, OVRemoteCont - auto obj = oe.compile_model(model, *context); - return OVExeNetwork(obj); - } catch (const Exception& e) { -- throw std::string(log_tag + " Exception while Loading Network for graph: " + name + e.what()); -+ ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what()); - } catch (...) { -- throw std::string(log_tag + " Exception while Loading Network for graph " + name); -+ ORT_THROW(log_tag + " Exception while Loading Network for graph " + name); - } - } - #endif -@@ -135,9 +130,9 @@ OVInferRequest OVExeNetwork::CreateInferRequest() { - OVInferRequest inf_obj(infReq); - return inf_obj; - } catch (const Exception& e) { -- throw std::string(log_tag + "Exception while creating InferRequest object: " + e.what()); -+ ORT_THROW(log_tag + "Exception while creating InferRequest object: " + e.what()); - } catch (...) { -- throw std::string(log_tag + "Exception while creating InferRequest object."); -+ ORT_THROW(log_tag + "Exception while creating InferRequest object."); - } - } - -@@ -147,9 +142,9 @@ OVTensorPtr OVInferRequest::GetTensor(const std::string& input_name) { - OVTensorPtr blob = std::make_shared(tobj); - return blob; - } catch (const Exception& e) { -- throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); -+ ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name + e.what()); - } catch (...) { -- throw std::string(log_tag + " Cannot access IE Blob for input: " + input_name); -+ ORT_THROW(log_tag + " Cannot access IE Blob for input: " + input_name); - } - } - -@@ -157,9 +152,9 @@ void OVInferRequest::SetTensor(const std::string& name, OVTensorPtr& blob) { - try { - ovInfReq.set_tensor(name, *(blob.get())); - } catch (const Exception& e) { -- throw std::string(log_tag + " Cannot set Remote Blob for output: " + name + e.what()); -+ ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name + e.what()); - } catch (...) { -- throw std::string(log_tag + " Cannot set Remote Blob for output: " + name); -+ ORT_THROW(log_tag + " Cannot set Remote Blob for output: " + name); - } - } - -@@ -167,9 +162,9 @@ void OVInferRequest::StartAsync() { - try { - ovInfReq.start_async(); - } catch (const Exception& e) { -- throw std::string(log_tag + " Couldn't start Inference: " + e.what()); -+ ORT_THROW(log_tag + " Couldn't start Inference: " + e.what()); - } catch (...) { -- throw std::string(log_tag + " In Error Couldn't start Inference"); -+ ORT_THROW(log_tag + " In Error Couldn't start Inference"); - } - } - -@@ -177,9 +172,9 @@ void OVInferRequest::Infer() { - try { - ovInfReq.infer(); - } catch (const Exception& e) { -- throw std::string(log_tag + " Couldn't start Inference: " + e.what()); -+ ORT_THROW(log_tag + " Couldn't start Inference: " + e.what()); - } catch (...) { -- throw std::string(log_tag + " In Error Couldn't start Inference"); -+ ORT_THROW(log_tag + " In Error Couldn't start Inference"); - } - } - -@@ -187,9 +182,9 @@ void OVInferRequest::WaitRequest() { - try { - ovInfReq.wait(); - } catch (const Exception& e) { -- throw std::string(log_tag + " Wait Model Failed: " + e.what()); -+ ORT_THROW(log_tag + " Wait Model Failed: " + e.what()); - } catch (...) { -- throw std::string(log_tag + " Wait Mode Failed"); -+ ORT_THROW(log_tag + " Wait Mode Failed"); - } - } - -diff --git a/onnxruntime/core/providers/openvino/ov_interface.h b/onnxruntime/core/providers/openvino/ov_interface.h -index cf4d867d4df5520919fd315ce7ddd5c893c786da..2a13fafb99fd303f64d02e60657747b07dd4836e 100644 ---- a/onnxruntime/core/providers/openvino/ov_interface.h -+++ b/onnxruntime/core/providers/openvino/ov_interface.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -6,14 +6,11 @@ - #include - #include - --#define OV_API_20 - #include "openvino/openvino.hpp" - #include "openvino/pass/convert_fp32_to_fp16.hpp" - #include "openvino/frontend/manager.hpp" - - #ifdef IO_BUFFER_ENABLED --#include --#include - #include - #endif - -diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.cc b/onnxruntime/core/providers/openvino/ov_versions/capability.cc -index 11c8a1629b073c34f3857abf5bc258bb534ac4fe..3970bf6ff68a7bfd4427c8502693d754f1daa972 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/capability.cc -+++ b/onnxruntime/core/providers/openvino/ov_versions/capability.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) 2019- Intel Corporation - // Licensed under the MIT License - - #include "core/providers/shared_library/provider_api.h" -@@ -6,6 +6,7 @@ - #include "../backend_manager.h" - #include "capability.h" - #include "utils.h" -+#include "openvino/core/version.hpp" - - #if defined(_MSC_VER) - #pragma warning(disable : 4244 4245 5208) -@@ -25,20 +26,22 @@ namespace openvino_ep { - // Constructor - GetCapability::GetCapability(const GraphViewer& graph_viewer_param, - const std::string device_type_param, -- const std::string device_precision, -- const std::string version_param) -+ const std::string device_precision) - : graph_viewer_(graph_viewer_param), device_type_(device_type_param), device_precision_(device_precision) { -- if (version_param == "V_2023_0") { -- data_ops_ = new DataOps(graph_viewer_, V_2023_0, device_type_, device_precision_); -- } else if (version_param == "V_2023_1") { -- data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_, device_precision_); -- } else if (version_param == "V_2023_2") { -- data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_, device_precision_); -- } else if (version_param == "V_2023_3") { -- data_ops_ = new DataOps(graph_viewer_, V_2023_3, device_type_, device_precision_); -- } else { -- data_ops_ = new DataOps(graph_viewer_, V_2023_3, device_type_, device_precision_); -+ if (device_type_.find("NPU") != std::string::npos) { -+ device_type_ = "CPU_FP32"; - } -+#if OPENVINO_VERSION_MAJOR == 2023 && OPENVINO_VERSION_MINOR == 1 -+ data_ops_ = new DataOps(graph_viewer_, V_2023_1, device_type_, device_precision_); -+#elif OPENVINO_VERSION_MAJOR == 2023 && OPENVINO_VERSION_MINOR == 2 -+ data_ops_ = new DataOps(graph_viewer_, V_2023_2, device_type_, device_precision_); -+#elif OPENVINO_VERSION_MAJOR == 2023 && OPENVINO_VERSION_MINOR == 3 -+ data_ops_ = new DataOps(graph_viewer_, V_2023_3, device_type_, device_precision_); -+#elif OPENVINO_VERSION_MAJOR == 2024 && OPENVINO_VERSION_MINOR == 0 -+ data_ops_ = new DataOps(graph_viewer_, V_2024_0, device_type_, device_precision_); -+#else -+ data_ops_ = new DataOps(graph_viewer_, V_2024_0, device_type_, device_precision_); -+#endif - } - - std::vector> GetCapability::Execute() { -diff --git a/onnxruntime/core/providers/openvino/ov_versions/capability.h b/onnxruntime/core/providers/openvino/ov_versions/capability.h -index 2040634cc45d9712682efd6cbabbc646c796c796..d9fe5a95ef8333c137212f75976c4737101d978e 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/capability.h -+++ b/onnxruntime/core/providers/openvino/ov_versions/capability.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -21,8 +21,7 @@ class GetCapability { - public: - GetCapability(const GraphViewer& graph_viewer_param, - const std::string device_type_param, -- const std::string precision, -- const std::string version_param); -+ const std::string precision); - virtual std::vector> Execute(); - bool IsWhollySupportedGraph() { - return is_wholly_supported_graph_; -diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc -index e829bf377b1954a99783beb36d29dddac88c6fd7..c7c3e93595719b003fffca854ba0d0af1826021f 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc -+++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include -@@ -14,6 +14,7 @@ - #include "data_ops.h" - #include "capability.h" - #include "utils.h" -+#include "../ov_interface.h" - - #if defined(_MSC_VER) - #pragma warning(disable : 4244 4245 5208) -@@ -36,6 +37,7 @@ namespace openvino_ep { - std::set ops_supported_only_in_model = { - "Add", - "Cast", -+ "Celu", - "Concat", - "ConstantOfShape", - "DequantizeLinear", -@@ -46,6 +48,7 @@ std::set ops_supported_only_in_model = { - "EyeLike", - "GatherElements", - "GatherND", -+ "GridSample", - "Identity", - "LayerNormalization", - "Loop", -@@ -72,293 +75,171 @@ std::set ops_supported_only_in_model = { - std::set ops_supported_as_function = { - "LessOrEqual", - "GreaterOrEqual", -- "LayerNormalization"}; -+ "LayerNormalization", -+ "Celu"}; - - std::vector supported_op_mode = { - {"Abs", V_2020_4, {"CPU", "GPU"}}, -- {"Abs", V_2023_0, {"NPU"}}, - {"Acos", V_2020_4, {"CPU"}}, - {"Acos", V_2022_1, {"GPU"}}, -- {"Acos", V_2023_1, {"NPU"}}, - {"Acosh", V_2020_4, {"CPU"}}, - {"Acosh", V_2022_1, {"GPU"}}, -- {"Acosh", V_2023_1, {"NPU"}}, - {"Add", V_2020_4, {"CPU", "GPU"}}, -- {"Add", V_2023_0, {"NPU"}}, - {"And", V_2020_4, {"CPU", "GPU"}}, -- {"And", V_2023_1, {"NPU"}}, - {"ArgMax", V_2020_4, {"CPU"}}, - {"ArgMax", V_2021_1, {"GPU"}}, - {"ArgMin", V_2020_4, {"CPU"}}, - {"ArgMin", V_2022_1, {"GPU"}}, - {"Asin", V_2020_4, {"CPU", "GPU"}}, -- {"Asin", V_2023_1, {"NPU"}}, - {"Asinh", V_2020_4, {"CPU", "GPU"}}, -- {"Asinh", V_2023_1, {"NPU"}}, - {"Atan", V_2020_4, {"CPU", "GPU"}}, -- {"Atan", V_2023_1, {"NPU"}}, - {"Atanh", V_2020_4, {"CPU"}}, - {"Atanh", V_2022_1, {"GPU"}}, -- {"Atanh", V_2023_1, {"NPU"}}, - {"AveragePool", V_2020_4, {"CPU", "GPU"}}, -- {"AveragePool", V_2023_0, {"NPU"}}, - {"BatchNormalization", V_2020_4, {"CPU", "GPU"}}, -- {"BatchNormalization", V_2023_0, {"NPU"}}, - {"BitShift", V_2022_1, {"CPU"}}, -- {"BitShift", V_2023_1, {"NPU"}}, - {"Cast", V_2020_4, {"CPU", "GPU"}}, -- {"Cast", V_2023_0, {"NPU"}}, -- {"CastLike", V_2023_1, {"CPU", "GPU", "NPU"}}, -+ {"CastLike", V_2023_1, {"CPU", "GPU"}}, - {"Ceil", V_2020_4, {"GPU"}}, - {"Ceil", V_2021_4, {"CPU"}}, -- {"Ceil", V_2023_1, {"NPU"}}, - {"Celu", V_2022_1, {"CPU", "GPU"}}, - {"Clip", V_2020_4, {"CPU", "GPU"}}, -- {"Clip", V_2023_0, {"NPU"}}, - {"Compress", V_2023_1, {"CPU", "GPU"}}, - {"Concat", V_2020_4, {"CPU", "GPU"}}, -- {"Concat", V_2023_0, {"NPU"}}, - {"Constant", V_2020_4, {"CPU", "GPU"}}, -- {"Constant", V_2023_0, {"NPU"}}, - {"ConstantOfShape", V_2020_4, {"CPU", "GPU"}}, -- {"ConstantOfShape", V_2023_0, {"NPU"}}, // Gets mapped to broadcast op in the plugin. - {"Conv", V_2020_4, {"CPU", "GPU"}}, -- {"Conv", V_2023_0, {"NPU"}}, - {"ConvInteger", V_2022_1, {"CPU", "GPU"}}, -- {"ConvInteger", V_2023_1, {"NPU"}}, - {"ConvTranspose", V_2020_4, {"CPU", "GPU"}}, -- {"ConvTranspose", V_2023_1, {"NPU"}}, - {"Cos", V_2020_4, {"CPU"}}, - {"Cos", V_2022_1, {"GPU"}}, -- {"Cos", V_2023_0, {"NPU"}}, - {"Cosh", V_2020_4, {"CPU"}}, - {"Cosh", V_2022_1, {"GPU"}}, -- {"Cosh", V_2023_1, {"NPU"}}, - {"CumSum", V_2022_1, {"CPU", "GPU"}}, -- {"CumSum", V_2023_0, {"NPU"}}, - {"DepthToSpace", V_2020_4, {"CPU", "GPU"}}, -- {"DepthToSpace", V_2023_0, {"NPU"}}, - {"DequantizeLinear", V_2021_4, {"CPU", "GPU"}}, -- {"DequantizeLinear", V_2023_0, {"NPU"}}, - {"Div", V_2020_4, {"CPU", "GPU"}}, -- {"Div", V_2023_0, {"NPU"}}, - {"Dropout", V_2020_4, {"CPU", "GPU"}}, -- {"Dropout", V_2023_0, {"NPU"}}, - {"Elu", V_2020_4, {"CPU", "GPU"}}, -- {"Elu", V_2023_0, {"NPU"}}, - {"Einsum", V_2023_1, {"CPU", "GPU"}}, - {"Equal", V_2020_4, {"CPU", "GPU"}}, -- {"Equal", V_2023_0, {"NPU"}}, // Added for whisper decoder model. - {"Erf", V_2020_4, {"CPU", "GPU"}}, -- {"Erf", V_2023_0, {"NPU"}}, - {"Exp", V_2020_4, {"CPU", "GPU"}}, -- {"Exp", V_2023_0, {"NPU"}}, - {"Expand", V_2022_1, {"CPU", "GPU"}}, -- {"Expand", V_2023_0, {"NPU"}}, // Gets mapped to broadcast op and multiply op in the plugin. - {"EyeLike", V_2022_1, {"CPU"}}, -- {"EyeLike", V_2023_0, {"NPU"}}, // NoOP - {"Flatten", V_2020_4, {"CPU", "GPU"}}, -- {"Flatten", V_2023_0, {"NPU"}}, - {"Floor", V_2020_4, {"CPU", "GPU"}}, -- {"Floor", V_2023_1, {"NPU"}}, - {"Gather", V_2020_4, {"CPU", "GPU"}}, -- {"Gather", V_2023_0, {"NPU"}}, - {"GatherElements", V_2022_2, {"CPU", "GPU"}}, -- {"GatherElements", V_2023_1, {"NPU"}}, - {"GatherND", V_2021_4, {"CPU", "GPU"}}, -- {"GatherND", V_2023_1, {"NPU"}}, -+ {"Gelu", V_2023_1, {"CPU", "GPU"}}, - {"Gemm", V_2020_4, {"CPU", "GPU"}}, -- {"Gemm", V_2023_0, {"NPU"}}, - {"GlobalAveragePool", V_2020_4, {"CPU", "GPU"}}, -- {"GlobalAveragePool", V_2023_0, {"NPU"}}, - {"GlobalLpPool", V_2020_4, {"CPU", "GPU"}}, -- {"GlobalLpPool", V_2023_1, {"NPU"}}, - {"GlobalMaxPool", V_2022_1, {"CPU", "GPU"}}, -- {"GlobalMaxPool", V_2023_1, {"NPU"}}, - {"Greater", V_2020_4, {"CPU", "GPU"}}, -- {"Greater", V_2023_0, {"NPU"}}, - {"GreaterOrEqual", V_2022_1, {"CPU", "GPU"}}, -- {"GreaterOrEqual", V_2023_0, {"NPU"}}, - {"GridSample", V_2022_3, {"CPU"}}, - {"GridSample", V_2023_0, {"GPU"}}, -- {"GridSample", V_2023_1, {"NPU"}}, -- {"HardMax", V_2023_1, {"CPU", "GPU", "NPU"}}, -+ {"HardMax", V_2023_1, {"CPU", "GPU"}}, - {"Identity", V_2020_4, {"CPU", "GPU"}}, -- {"Identity", V_2023_0, {"NPU"}}, // NoOP - {"If", V_2022_3, {"CPU", "GPU"}}, -- {"If", V_2023_1, {"NPU"}}, - {"ImageScaler", V_2022_1, {"CPU", "GPU"}}, -- {"ImageScaler", V_2023_0, {"NPU"}}, - {"InstanceNormalization", V_2020_4, {"CPU", "GPU"}}, -- {"InstanceNormalization", V_2023_0, {"NPU"}}, - {"HardSigmoid", V_2020_4, {"CPU", "GPU"}}, -- {"HardSigmoid", V_2023_1, {"NPU"}}, - {"HardMax", V_2022_1, {"CPU", "GPU"}}, -+ {"LayerNormalization", V_2023_0, {"CPU", "GPU"}}, - {"LeakyRelu", V_2020_4, {"CPU", "GPU"}}, -- {"LeakyRelu", V_2023_0, {"NPU"}}, - {"Less", V_2020_4, {"CPU", "GPU"}}, -- {"Less", V_2023_0, {"NPU"}}, // Added for whisper decoder model. - {"LessOrEqual", V_2022_1, {"CPU", "GPU"}}, -- {"LessOrEqual", V_2023_0, {"NPU"}}, - {"Log", V_2020_4, {"CPU", "GPU"}}, -- {"Log", V_2023_0, {"NPU"}}, - {"LogSoftMax", V_2022_1, {"CPU", "GPU"}}, - {"Loop", V_2021_4, {"CPU", "GPU"}}, -- {"LpNormalization", V_2023_1, {"CPU", "GPU", "NPU"}}, -- {"LpPool", V_2023_1, {"CPU", "GPU", "NPU"}}, -+ {"LpNormalization", V_2023_1, {"CPU", "GPU"}}, - {"LRN", V_2020_4, {"CPU", "GPU"}}, -- {"LRN", V_2023_0, {"NPU"}}, - {"LSTM", V_2020_4, {"CPU", "GPU"}}, -- {"LSTM", V_2023_1, {"NPU"}}, - {"MatMul", V_2020_4, {"CPU", "GPU"}}, -- {"MatMul", V_2023_0, {"NPU"}}, - {"MatMulInteger", V_2022_1, {"CPU"}}, -- {"MatMulInteger", V_2023_1, {"NPU"}}, - {"Max", V_2020_4, {"CPU", "GPU"}}, -- {"Max", V_2023_0, {"NPU"}}, - {"MaxPool", V_2020_4, {"CPU", "GPU"}}, -- {"MaxPool", V_2023_0, {"NPU"}}, - {"Mean", V_2020_4, {"CPU", "GPU"}}, -- {"Mean", V_2023_0, {"NPU"}}, - {"MeanVarianceNormalization", V_2022_1, {"CPU", "GPU"}}, -- {"MeanVarianceNormalization", V_2023_1, {"NPU"}}, - {"Min", V_2020_4, {"CPU", "GPU"}}, -- {"Min", V_2023_0, {"NPU"}}, - {"Mod", V_2022_1, {"CPU", "GPU"}}, - {"Mul", V_2020_4, {"CPU", "GPU"}}, -- {"Mul", V_2023_0, {"NPU"}}, - {"Neg", V_2020_4, {"CPU", "GPU"}}, -- {"Neg", V_2023_0, {"NPU"}}, - {"NonMaxSuppression", V_2021_1, {"CPU", "GPU"}}, -- {"NonMaxSuppression", V_2023_1, {"NPU"}}, - {"NonZero", V_2021_1, {"CPU"}}, - {"NonZero", V_2023_0, {"GPU"}}, - {"Not", V_2021_1, {"CPU", "GPU"}}, - {"Not", V_2020_4, {"CPU", "GPU"}}, -- {"Not", V_2023_1, {"NPU"}}, - {"OneHot", V_2020_4, {"CPU", "GPU"}}, -- {"OneHot", V_2023_1, {"NPU"}}, - {"Or", V_2022_1, {"CPU", "GPU"}}, -- {"Or", V_2023_1, {"NPU"}}, - {"Pad", V_2020_4, {"CPU", "GPU"}}, -- {"Pad", V_2023_0, {"NPU"}}, - {"Pow", V_2020_4, {"CPU", "GPU"}}, -- {"Pow", V_2023_0, {"NPU"}}, - {"PRelu", V_2020_4, {"CPU", "GPU"}}, -- {"PRelu", V_2023_0, {"NPU"}}, - {"QLinearMatMul", V_2022_3, {"CPU"}}, -- // {"QLinearMatMul", V_2023_1, {"NPU"}}, - {"QuantizeLinear", V_2021_4, {"CPU", "GPU"}}, -- {"QuantizeLinear", V_2023_0, {"NPU"}}, - {"RNN", V_2023_1, {"CPU", "GPU"}}, - {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, - {"RandomNormalLike", V_2023_0, {"CPU", "GPU"}}, -- {"RandomNormalLike", V_2023_1, {"NPU"}}, - {"RandomNormal", V_2023_0, {"CPU", "GPU"}}, -- {"RandomNormal", V_2023_1, {"NPU"}}, - {"Range", V_2022_1, {"CPU", "GPU"}}, -- {"Range", V_2023_0, {"NPU"}}, - {"Reciprocal", V_2020_4, {"CPU", "GPU"}}, -- {"Reciprocal", V_2023_0, {"NPU"}}, - {"ReduceL1", V_2022_1, {"CPU", "GPU"}}, -- {"ReduceL1", V_2023_1, {"NPU"}}, - {"ReduceL2", V_2022_1, {"CPU", "GPU"}}, -- {"ReduceL2", V_2023_1, {"NPU"}}, - {"ReduceLogSum", V_2020_4, {"CPU"}}, - {"ReduceLogSum", V_2022_1, {"CPU", "GPU"}}, -- {"ReduceLogSum", V_2023_1, {"NPU"}}, - {"ReduceLogSumExp", V_2022_1, {"CPU", "GPU"}}, -- {"ReduceLogSumExp", V_2023_1, {"NPU"}}, - {"ReduceMax", V_2020_4, {"CPU", "GPU"}}, -- {"ReduceMax", V_2023_1, {"NPU"}}, - {"ReduceMean", V_2020_4, {"CPU", "GPU"}}, -- {"ReduceMean", V_2023_0, {"NPU"}}, - {"ReduceMin", V_2020_4, {"CPU", "GPU"}}, -- {"ReduceMin", V_2023_1, {"NPU"}}, - {"ReduceProd", V_2020_4, {"CPU"}}, - {"ReduceProd", V_2022_1, {"GPU"}}, -- {"ReduceProd", V_2023_1, {"NPU"}}, - {"ReduceSum", V_2020_4, {"CPU", "GPU"}}, -- // {"ReduceSum", V_2023_1, {"NPU"}}, - {"ReduceSumSquare", V_2020_4, {"CPU"}}, - {"ReduceSumSquare", V_2022_1, {"CPU", "GPU"}}, -- {"ReduceSumSquare", V_2023_1, {"NPU"}}, - {"Relu", V_2020_4, {"CPU", "GPU"}}, -- {"Relu", V_2023_0, {"NPU"}}, - {"Resize", V_2020_4, {"CPU"}}, - {"Resize", V_2022_1, {"GPU"}}, -- {"Resize", V_2023_1, {"NPU"}}, - {"Reshape", V_2020_4, {"CPU", "GPU"}}, -- {"Reshape", V_2023_0, {"NPU"}}, - {"ReverseSequence", V_2022_1, {"CPU", "GPU"}}, - {"RoiAlign", V_2021_1, {"CPU", "GPU"}}, -- {"RoiAlign", V_2023_1, {"NPU"}}, - {"Round", V_2021_4, {"CPU", "GPU"}}, -- {"Round", V_2023_1, {"NPU"}}, - {"Scatter", V_2022_1, {"CPU", "GPU"}}, -- {"Scatter", V_2023_1, {"NPU"}}, - {"ScatterElements", V_2022_1, {"CPU", "GPU"}}, -- {"ScatterElements", V_2023_1, {"NPU"}}, - {"ScatterND", V_2022_1, {"CPU", "GPU"}}, -- {"ScatterND", V_2023_1, {"NPU"}}, - {"Selu", V_2020_4, {"CPU", "GPU"}}, -- {"Selu", V_2023_1, {"NPU"}}, - {"Shape", V_2020_4, {"CPU", "GPU"}}, -- {"Shape", V_2023_0, {"NPU"}}, - {"Shrink", V_2022_1, {"CPU", "GPU"}}, -- {"Shrink", V_2023_0, {"NPU"}}, - {"Sigmoid", V_2020_4, {"CPU", "GPU"}}, -- {"Sigmoid", V_2023_0, {"NPU"}}, - {"Sign", V_2020_4, {"CPU"}}, - {"Sign", V_2022_1, {"GPU"}}, -- {"Sign", V_2023_0, {"NPU"}}, - {"Sin", V_2022_1, {"CPU", "GPU"}}, -- {"Sin", V_2023_0, {"NPU"}}, - {"Sinh", V_2020_4, {"CPU"}}, -- {"Sinh", V_2023_1, {"NPU"}}, - {"Size", V_2022_1, {"CPU", "GPU"}}, -- {"Size", V_2023_1, {"NPU"}}, - {"Slice", V_2020_4, {"CPU", "GPU"}}, -- {"Slice", V_2023_0, {"NPU"}}, - {"Softmax", V_2020_4, {"CPU", "GPU"}}, -- {"Softmax", V_2023_0, {"NPU"}}, - {"Softplus", V_2022_1, {"CPU", "GPU"}}, -- {"Softplus", V_2023_0, {"NPU"}}, - {"Softsign", V_2022_1, {"CPU", "GPU"}}, - {"SpaceToDepth", V_2020_4, {"CPU", "GPU"}}, -- {"SpaceToDepth", V_2023_0, {"NPU"}}, - {"Split", V_2020_4, {"CPU", "GPU"}}, -- {"Split", V_2023_0, {"NPU"}}, - {"Sqrt", V_2020_4, {"CPU", "GPU"}}, -- {"Sqrt", V_2023_0, {"NPU"}}, - {"Squeeze", V_2020_4, {"CPU", "GPU"}}, -- {"Squeeze", V_2023_0, {"NPU"}}, - {"Softsign", V_2020_4, {"CPU"}}, - {"Sub", V_2020_4, {"CPU", "GPU"}}, -- {"Sub", V_2023_0, {"NPU"}}, - {"Sum", V_2020_4, {"CPU", "GPU"}}, -- {"Sum", V_2023_0, {"NPU"}}, - {"Tan", V_2020_4, {"CPU", "GPU"}}, -- {"Tan", V_2023_1, {"NPU"}}, - {"Tanh", V_2020_4, {"CPU", "GPU"}}, -- {"Tanh", V_2023_0, {"NPU"}}, - {"ThresholdedRelu", V_2022_1, {"CPU", "GPU"}}, -- {"ThresholdedRelu", V_2023_0, {"NPU"}}, - {"Tile", V_2021_3, {"CPU", "GPU"}}, -- {"Tile", V_2023_0, {"NPU"}}, - {"Transpose", V_2020_4, {"CPU", "GPU"}}, -- {"Transpose", V_2023_0, {"NPU"}}, - {"Trilu", V_2023_0, {"CPU", "GPU"}}, -- {"Trilu", V_2023_1, {"NPU"}}, - {"TopK", V_2020_4, {"CPU", "GPU"}}, -- {"TopK", V_2023_0, {"NPU"}}, - {"Upsample", V_2020_4, {"CPU", "GPU"}}, - {"Unsqueeze", V_2020_4, {"CPU", "GPU"}}, -- {"Unsqueeze", V_2023_0, {"NPU"}}, - {"Where", V_2022_1, {"CPU", "GPU"}}, -- {"Where", V_2023_0, {"NPU"}}, // Added for whisper decoder model. - {"Xor", V_2022_1, {"CPU", "GPU"}}, -- {"Xor", V_2023_1, {"NPU"}}, - }; - - void DataOps::populate_types_supported() { -@@ -370,6 +251,8 @@ void DataOps::populate_types_supported() { - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_initializer_.insert( - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT64)); -+ supported_types_initializer_.insert( -+ std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT16)); - supported_types_initializer_.insert( - std::make_pair(V_2021_1, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT16)); - supported_types_initializer_.insert( -@@ -387,6 +270,8 @@ void DataOps::populate_types_supported() { - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); - supported_types_npu_.insert( - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); -+ supported_types_npu_.insert( -+ std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT16)); - supported_types_npu_.insert( - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_npu_.insert( -@@ -402,6 +287,8 @@ void DataOps::populate_types_supported() { - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT32)); - supported_types_cpu_.insert( - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16)); -+ supported_types_cpu_.insert( -+ std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT16)); - supported_types_cpu_.insert( - std::make_pair(V_2020_4, ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8)); - supported_types_cpu_.insert( -@@ -437,13 +324,12 @@ void DataOps::populate_op_mode_supported() { - no_dimension_supported_.push_back({"DequantizeLinear", V_2021_4, {"All"}}); - no_dimension_supported_.push_back({"Equal", V_2022_1, {"CPU"}}); - no_dimension_supported_.push_back({"Equal", V_2023_0, {"GPU"}}); -+ no_dimension_supported_.push_back({"Expand", V_2023_3, {"CPU"}}); - no_dimension_supported_.push_back({"Floor", V_2020_4, {"All"}}); - no_dimension_supported_.push_back({"Gather", V_2020_4, {"All"}}); -- no_dimension_supported_.push_back({"Greater", V_2023_0, {"NPU"}}); - no_dimension_supported_.push_back({"Identity", V_2023_0, {"All"}}); - no_dimension_supported_.push_back({"Less", V_2022_1, {"CPU"}}); - no_dimension_supported_.push_back({"Loop", V_2021_4, {"All"}}); -- no_dimension_supported_.push_back({"Max", V_2023_0, {"NPU"}}); - no_dimension_supported_.push_back({"Min", V_2020_4, {"All"}}); - no_dimension_supported_.push_back({"Mul", V_2020_4, {"All"}}); - no_dimension_supported_.push_back({"Neg", V_2023_0, {"CPU", "GPU"}}); -@@ -476,9 +362,8 @@ void DataOps::populate_op_mode_supported() { - { - UnsupportedOpMode obj = {{V_2022_1, V_2022_2, V_2022_3}, - [this](const Node* node, const InitializedTensorSet&) { -- // Abs is not supproted with INT8 or INT32 as input data type on GPU and NPU -- if ((device_id_.find("GPU") != std::string::npos) || -- (device_id_.find("NPU") != std::string::npos)) { -+ // Abs is not supproted with INT8 or INT32 as input data type on GPU -+ if ((device_id_.find("GPU") != std::string::npos)) { - for (size_t i = 0; i < node->InputDefs().size(); i++) { - if (node->InputDefs()[i]->TypeAsProto()->tensor_type().elem_type() == - ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 || -@@ -706,7 +591,7 @@ void DataOps::populate_op_mode_supported() { - op_list_.insert({"PRelu", obj}); - } - { -- UnsupportedOpMode obj = {{V_2023_0, V_2023_1, V_2023_2, V_2023_3}, -+ UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0}, - [this](const Node* node, const InitializedTensorSet&) { - const auto& input_arg = node->InputDefs()[1]; - auto shape = input_arg->Shape(); -@@ -821,7 +706,7 @@ void DataOps::populate_op_mode_supported() { - op_list_.insert({"Squeeze", obj}); - } - { -- UnsupportedOpMode obj = {{V_2023_0, V_2023_1, V_2023_2, V_2023_3}, -+ UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0}, - [this](const Node* node, const InitializedTensorSet&) { - // If the operator is unsqueeze - // If axes is an input, then we cannot produce a static graph. -@@ -836,7 +721,7 @@ void DataOps::populate_op_mode_supported() { - op_list_.insert({"Unsqueeze", obj}); - } - { -- UnsupportedOpMode obj = {{V_2023_0, V_2023_1, V_2023_2, V_2023_3}, -+ UnsupportedOpMode obj = {{V_2023_1, V_2023_2, V_2023_3, V_2024_0}, - [this](const Node* node, const InitializedTensorSet&) { - // check for attributes - auto& upsample_attr = node->GetAttributes(); -@@ -961,7 +846,7 @@ bool DataOps::type_is_supported(const NodeArg* node_arg, bool is_initializer) { - } else { - auto dtype = type_proto->tensor_type().elem_type(); - -- if (device_id_.find("NPU") != std::string::npos || device_id_.find("HETERO") != std::string::npos || -+ if (device_id_.find("HETERO") != std::string::npos || - device_id_.find("MULTI") != std::string::npos || device_id_.find("AUTO") != std::string::npos) { - for (auto const& var : supported_types_npu_) { - if ((var.first <= version_id_) && -@@ -1063,8 +948,7 @@ bool DataOps::dimension_unsupported(const Node* node) { - return true; - } - --bool DataOps::node_is_supported(const std::map>& op_map, -- const NodeIndex node_idx) { -+bool DataOps::node_is_supported(const NodeIndex node_idx) { - const auto& node = graph_viewer_.GetNode(node_idx); - const auto& optype = node->OpType(); - -@@ -1174,37 +1058,14 @@ bool DataOps::node_is_supported(const std::mapOpType()); -- if (opset == op_map.end()) { --#ifndef NDEBUG -- if (openvino_ep::backend_utils::IsDebugEnabled()) { -- std::cout << "Failed in Unsupported onnx model domain" << std::endl; -- } --#endif -- return false; -- } -- if (opset->second.find(optype) == opset->second.end() && op_fun == ops_supported_as_function.end()) { --#ifndef NDEBUG -- if (openvino_ep::backend_utils::IsDebugEnabled()) { -- std::cout << "The operator is not available in OpenVINO ngraph operators list" -- << "nor the operator is a special ONNX function" -- << std::endl; -- } --#endif -- return false; -- } - return true; - } - - std::vector DataOps::GetUnsupportedNodeIndices(std::unordered_set& ng_required_initializers) { -- const auto ng_supported_ops = GetNgSupportedOps(GetOnnxOpSet(graph_viewer_)); -- - std::vector unsupported_nodes_idx; - - for (const auto& node_idx : graph_viewer_.GetNodesInTopologicalOrder()) { -- if (node_is_supported(ng_supported_ops, node_idx)) { -+ if (node_is_supported(node_idx)) { - // Collect inputs that are initializers - graph_viewer_.GetNode(node_idx)->ForEachDef([&ng_required_initializers, this](const NodeArg& node_arg, - bool is_input) { -diff --git a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h -index 87688601ad6925f46de0b68a55cdcff71cbd3203..0990904908111e7b2cbac857a80115dc669fd5b4 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/data_ops.h -+++ b/onnxruntime/core/providers/openvino/ov_versions/data_ops.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #pragma once -@@ -26,7 +26,8 @@ enum versionNum { - V_2023_0, - V_2023_1, - V_2023_2, -- V_2023_3 -+ V_2023_3, -+ V_2024_0 - }; - - using VersionNum = enum versionNum; -@@ -67,9 +68,7 @@ class DataOps { - bool dimension_unsupported(const Node* node); - bool unsupported_op_mode(const Node* node); - bool type_is_supported(const NodeArg* node_arg, bool is_initializer); -- bool node_is_supported(const std::map>& op_map, -- const NodeIndex node_idx); -+ bool node_is_supported(const NodeIndex node_idx); - - public: - DataOps(const GraphViewer& graph_viewer_param, VersionNum ver, const std::string dev_id, const std::string device_precision) -diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.cc b/onnxruntime/core/providers/openvino/ov_versions/utils.cc -index ee0bfddb7dc83d8403c9e86030694fedc861d2f9..c5ed29df487b45110d3362866d4fd563d3fd0d79 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/utils.cc -+++ b/onnxruntime/core/providers/openvino/ov_versions/utils.cc -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - - #include "core/providers/shared_library/provider_api.h" -@@ -11,14 +11,6 @@ - #pragma GCC diagnostic ignored "-Wunused-parameter" - #endif - --#include "openvino/core/deprecated.hpp" --#define IN_OV_COMPONENT --#define NGRAPH_LEGACY_HEADER_INCLUDED --#include -- --#undef NGRAPH_LEGACY_HEADER_INCLUDED --#undef IN_OV_COMPONENT -- - #if defined(_MSC_VER) - #pragma warning(default : 4244 4245) - #elif __GNUC__ -@@ -95,20 +87,6 @@ int GetOnnxOpSet(const GraphViewer& graph_viewer) { - return dm_to_ver.at(kOnnxDomain); - } - --std::map> GetNgSupportedOps(const int onnx_opset) { -- std::map> ng_supported_ops; -- OPENVINO_SUPPRESS_DEPRECATED_START -- ng_supported_ops.emplace(kOnnxDomain, ngraph::onnx_import::get_supported_operators(onnx_opset, kOnnxDomain)); -- -- const std::set ng_disabled_ops = {"LSTM"}; // Place-holder for ops not supported. -- -- for (const auto& disabled_op : ng_disabled_ops) { -- ng_supported_ops.at(kOnnxDomain).erase(disabled_op); -- } -- OPENVINO_SUPPRESS_DEPRECATED_END -- return ng_supported_ops; --} -- - /** - * Returns a vector clusters(or node_idx). For each unsupported node, the graph is split into 3 parts. - * supported_cluster + (UNsupported_node + rest_of_the_graph). This functions returns vector of all supported_clusters by nGraph -diff --git a/onnxruntime/core/providers/openvino/ov_versions/utils.h b/onnxruntime/core/providers/openvino/ov_versions/utils.h -index b3edeef88dfec1951124dc17d2aab13ddd105967..34aa762ba9b6782067287417eef7ea55ccfd54a1 100644 ---- a/onnxruntime/core/providers/openvino/ov_versions/utils.h -+++ b/onnxruntime/core/providers/openvino/ov_versions/utils.h -@@ -1,4 +1,4 @@ --// Copyright (C) 2019-2022 Intel Corporation -+// Copyright (C) Intel Corporation - // Licensed under the MIT License - #pragma once - -diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc -index e5e0e81cb7da890cd970d64e4c6310cf02320210..7b56f0c68427a42ef305be933a9ef169b1b07210 100644 ---- a/onnxruntime/python/onnxruntime_pybind_state.cc -+++ b/onnxruntime/python/onnxruntime_pybind_state.cc -@@ -937,6 +937,20 @@ std::unique_ptr CreateExecutionProviderInstance( - ORT_THROW("Invalid value passed for disable_dynamic_shapes: ", option.second); - } - OV_provider_options_map[option.first] = option.second; -+ } else if (option.first == "enable_dynamic_shapes") { -+ LOGS_DEFAULT(WARNING) << " Deprecation notice - 'enable_dynamic_shapes' is Deprected. Upgrade the API to disable_dynamic_shapes parameter." -+ "Please refer https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#requirements to ensure all dependencies are met."; -+ std::string value; -+ if (!(option.second == "True" || option.second == "true" || -+ option.second == "False" || option.second == "false")) { -+ ORT_THROW("Invalid value passed for enable_dynamic_shapes: ", option.second); -+ } -+ if (option.second == "True" || option.second == "true") { -+ value = "false"; -+ } else { -+ value = "true"; -+ } -+ OV_provider_options_map["disable_dynamic_shapes"] = value; - } else if (option.first == "device_id") { - OV_provider_options_map[option.first] = option.second; - continue; -@@ -967,7 +981,7 @@ std::unique_ptr CreateExecutionProviderInstance( - if (!Env::Default().GetEnvironmentVar("INTEL_OPENVINO_DIR").empty()) { - ORT_THROW("INTEL_OPENVINO_DIR is set but OpenVINO library wasn't able to be loaded. Please install a supported version of OpenVINO as mentioned in the requirements page (https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#requirements), ensure dependency libraries are in the PATH and your hardware is supported."); - } else { -- LOGS_DEFAULT(WARNING) << "Failed to create " << type << ". Please reference https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#requirements to ensure all dependencies are met."; -+ LOGS_DEFAULT(WARNING) << "Failed to create " << type << ". Please refer https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#requirements to ensure all dependencies are met."; - } - } - #endif -diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h -index 6827f2c9dfd91ff5b7b1766acacec7a82838c650..22314610dbee96a695196a9b4e75bd74b110d291 100644 ---- a/onnxruntime/python/onnxruntime_pybind_state_common.h -+++ b/onnxruntime/python/onnxruntime_pybind_state_common.h -@@ -60,11 +60,8 @@ struct OrtStatus { - #elif OPENVINO_CONFIG_GPU_FP16 - #define BACKEND_OPENVINO "-OPENVINO_GPU_FP16" - --#elif OPENVINO_CONFIG_NPU_FP16 --#define BACKEND_OPENVINO "-OPENVINO_NPU_FP16" -- --#elif OPENVINO_CONFIG_NPU_U8 --#define BACKEND_OPENVINO "-OPENVINO_NPU_U8" -+#elif OPENVINO_CONFIG_NPU -+#define BACKEND_OPENVINO "-OPENVINO_NPU" - - #elif OPENVINO_CONFIG_MULTI - #define BACKEND_OPENVINO "-OPENVINO_MULTI" -diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py -index 1056c4ed8451011928e830fe26e15c90c0b7933a..fab23e360405263f201b015a6117dc316519c607 100644 ---- a/tools/ci_build/build.py -+++ b/tools/ci_build/build.py -@@ -75,13 +75,14 @@ def _str_to_bool(s): - - - def _openvino_verify_device_type(device_read): -- choices = ["CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16"] -+ choices = ["CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16", "NPU"] - - choices1 = [ - "CPU_FP32_NO_PARTITION", - "CPU_FP16_NO_PARTITION", - "GPU_FP32_NO_PARTITION", - "GPU_FP16_NO_PARTITION", -+ "NPU_NO_PARTITION", - ] - status_hetero = True - res = False -@@ -96,7 +97,7 @@ def _openvino_verify_device_type(device_read): - if len(comma_separated_devices) < 2: - print("At least two devices required in Hetero/Multi/Auto Mode") - status_hetero = False -- dev_options = ["CPU", "GPU"] -+ dev_options = ["CPU", "GPU", "NPU"] - for dev in comma_separated_devices: - if dev not in dev_options: - status_hetero = False -@@ -107,7 +108,7 @@ def invalid_hetero_build(): - print("specify the keyword HETERO or MULTI or AUTO followed by the devices ") - print("in the order of priority you want to build\n") - print("The different hardware devices that can be added in HETERO or MULTI or AUTO") -- print("are ['CPU','GPU'] \n") -+ print("are ['CPU','GPU','NPU'] \n") - print("An example of how to specify the hetero build type. Ex: HETERO:GPU,CPU \n") - print("An example of how to specify the MULTI build type. Ex: MULTI:GPU,CPU \n") - print("An example of how to specify the AUTO build type. Ex: AUTO:GPU,CPU \n") -@@ -1222,6 +1223,7 @@ def generate_build_tree( - "-Donnxruntime_USE_OPENVINO_GPU_FP16=" + ("ON" if args.use_openvino == "GPU_FP16" else "OFF"), - "-Donnxruntime_USE_OPENVINO_CPU_FP32=" + ("ON" if args.use_openvino == "CPU_FP32" else "OFF"), - "-Donnxruntime_USE_OPENVINO_CPU_FP16=" + ("ON" if args.use_openvino == "CPU_FP16" else "OFF"), -+ "-Donnxruntime_USE_OPENVINO_NPU=" + ("ON" if args.use_openvino == "NPU" else "OFF"), - "-Donnxruntime_USE_OPENVINO_GPU_FP32_NP=" - + ("ON" if args.use_openvino == "GPU_FP32_NO_PARTITION" else "OFF"), - "-Donnxruntime_USE_OPENVINO_GPU_FP16_NP=" -@@ -1230,6 +1232,7 @@ def generate_build_tree( - + ("ON" if args.use_openvino == "CPU_FP32_NO_PARTITION" else "OFF"), - "-Donnxruntime_USE_OPENVINO_CPU_FP16_NP=" - + ("ON" if args.use_openvino == "CPU_FP16_NO_PARTITION" else "OFF"), -+ "-Donnxruntime_USE_OPENVINO_NPU_NP=" + ("ON" if args.use_openvino == "NPU_NO_PARTITION" else "OFF"), - "-Donnxruntime_USE_OPENVINO_HETERO=" + ("ON" if args.use_openvino.startswith("HETERO") else "OFF"), - "-Donnxruntime_USE_OPENVINO_DEVICE=" + (args.use_openvino), - "-Donnxruntime_USE_OPENVINO_MULTI=" + ("ON" if args.use_openvino.startswith("MULTI") else "OFF"), -diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml -index c92fc93abba37d3f8c4414da767a85bd7118166b..03e0274fc198ac2b28f6eee2fe0915800d27b56a 100644 ---- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml -+++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml -@@ -32,5 +32,5 @@ jobs: - parameters: - AgentPool : 'Linux-CPU-2019' - JobName: 'Linux_CI_Dev' -- RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2023.0.0 -x "--use_openvino CPU_FP32 --build_wheel"' -+ RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2024.0.0 -x "--use_openvino CPU_FP32 --build_wheel"' - TimeoutInMinutes: 120 -diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino -index a0ba5ea232ca3d5061fdfc565a3f7b703d0ceb48..45682c797bbb8389cb7d90dc31bdd60b682f318f 100644 ---- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino -+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino -@@ -1,8 +1,8 @@ - ARG UBUNTU_VERSION=20.04 - FROM ubuntu:${UBUNTU_VERSION} - --ARG OPENVINO_VERSION=2023.0.0 --ARG PYTHON_VERSION=3.8 -+ARG OPENVINO_VERSION=2024.0.0 -+ARG PYTHON_VERSION=3.9 - - ADD scripts /tmp/scripts - RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} -d EdgeDevice && \ -@@ -14,15 +14,14 @@ RUN apt update && apt install -y libnuma1 ocl-icd-libopencl1 && \ - - ENV INTEL_OPENVINO_DIR /opt/intel/openvino_${OPENVINO_VERSION} - ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64:$INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH --ENV InferenceEngine_DIR $INTEL_OPENVINO_DIR/runtime/cmake --ENV ngraph_DIR $INTEL_OPENVINO_DIR/runtime/cmake -+ENV OpenVINO_DIR $INTEL_OPENVINO_DIR/runtime/cmake - ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64 - ENV DEBIAN_FRONTEND=noninteractive - - RUN cd /opt && mkdir -p intel && cd intel && \ -- wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0/linux/l_openvino_toolkit_ubuntu20_2023.0.0.10926.b4452d56304_x86_64.tgz && \ -- tar xzf l_openvino_toolkit_ubuntu20_2023.0.0.10926.b4452d56304_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2023.0.0.10926.b4452d56304_x86_64.tgz && \ -- mv l_openvino_toolkit_ubuntu20_2023.0.0.10926.b4452d56304_x86_64 openvino_2023.0.0 && \ -+ wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/linux/l_openvino_toolkit_ubuntu20_2024.0.0.14509.34caeefd078_x86_64.tgz && \ -+ tar xzf l_openvino_toolkit_ubuntu20_2024.0.0.14509.34caeefd078_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu20_2024.0.0.14509.34caeefd078_x86_64.tgz && \ -+ mv l_openvino_toolkit_ubuntu20_2024.0.0.14509.34caeefd078_x86_64 openvino_2024.0.0 && \ - cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y - - WORKDIR /root -diff --git a/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython b/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython -deleted file mode 100644 -index bc0b41277328622995a9b865ef03ff503d2ebd39..0000000000000000000000000000000000000000 ---- a/tools/ci_build/github/linux/docker/Dockerfile_manylinux2014_openvino_multipython -+++ /dev/null -@@ -1,83 +0,0 @@ --FROM quay.io/pypa/manylinux2014_x86_64:latest -- --ENV PATH /opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin --ADD scripts /tmp/scripts --RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts -- --ARG PYTHON_VER_PATH="cp38-cp38" --ARG PYTHON_VERSION="3.8" --ARG BUILD_UID=1001 --ARG BUILD_USER=onnxruntimedev --ARG OV_DEVICE_PRECISION="CPU_FP32" --ARG ENABLE_TRAINING=true --ARG ORT_BRANCH="rel-1.13.1" --ARG OV_VERSION="2022.2.0" --RUN adduser --uid $BUILD_UID $BUILD_USER --WORKDIR /home/$BUILD_USER --ENV PYTHON_EXE="/opt/python/$PYTHON_VER_PATH/bin/python$PYTHON_VERSION" -- --RUN yum -y install wget git -- --# libusb1.0.22 --RUN cd /home/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \ -- unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /home/libusb-1.0.22 && \ --# bootstrap steps -- ./bootstrap.sh && \ -- ./configure --disable-udev --enable-shared && \ -- make -j4 && \ --# configure libusb1.0.22 -- cd /home/libusb-1.0.22/libusb && \ -- /bin/mkdir -p '/usr/local/lib' && \ -- /bin/bash ../libtool --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \ -- /bin/mkdir -p '/usr/local/include/libusb-1.0' && \ -- /usr/bin/install -c -m 644 libusb.h '/usr/local/include/libusb-1.0' && \ -- /bin/mkdir -p '/usr/local/lib/pkgconfig' -- --RUN ${PYTHON_EXE} -m pip install onnx numpy wheel --USER $BUILD_USER --RUN cd $WORKDIR && git clone https://github.com/openvinotoolkit/openvino.git && \ -- cd openvino && \ -- git checkout $OV_VERSION && \ -- git submodule init && \ -- git submodule update --recursive -- --RUN cd $WORKDIR && cd openvino && mkdir build && cd build && \ -- cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS=-D_GLIBCXX_USE_CXX11_ABI=0 -DENABLE_PYTHON=ON -DPYTHON_EXECUTABLE=$PYTHON_EXE -DCMAKE_INSTALL_PREFIX=/home/onnxruntimedev/openvino_$OV_VERSION && \ -- make -j8 && make install -- --ENV INTEL_OPENVINO_DIR /home/onnxruntimedev/openvino_$OV_VERSION --ENV LD_LIBRARY_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64:$INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib:/usr/local/openblas/lib:$LD_LIBRARY_PATH --ENV TBB_LIBS $INTEL_OPENVINO_DIR/runtime/3rdparty/tbb/lib --ENV InferenceEngine_DIR $INTEL_OPENVINO_DIR/runtime/cmake --ENV ngraph_DIR $INTEL_OPENVINO_DIR/runtime/cmake --ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64 --ENV OPENVINO_MANYLINUX 1 -- --RUN cd $WORKDIR && \ -- git clone --recursive -b $ORT_BRANCH https://github.com/intel/onnxruntime.git --RUN cd onnxruntime/onnxruntime/core/providers/openvino && mkdir scripts -- --RUN cp ${IE_PLUGINS_PATH}/libopenvino.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_c.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_onnx_frontend.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_intel_cpu_plugin.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_intel_gpu_plugin.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_intel_myriad_plugin.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_hetero_plugin.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/libopenvino_auto_plugin.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/plugins.xml /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${IE_PLUGINS_PATH}/usb-ma2x8x.mvcmd /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${TBB_LIBS}/libtbb.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${TBB_LIBS}/libtbb.so.2 /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${TBB_LIBS}/libtbbmalloc.so /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cp ${TBB_LIBS}/libtbbmalloc.so.2 /home/onnxruntimedev/onnxruntime/onnxruntime/core/providers/openvino/scripts/ --RUN cd /home/onnxruntimedev/onnxruntime && git pull --RUN if $ENABLE_TRAINING; then \ -- ${PYTHON_EXE} ./onnxruntime/tools/ci_build/build.py \ -- --build_dir ./onnxruntime/build --use_openvino $(OV_DEVICE_PRECISION) --build_shared_lib \ -- --config Release --build_wheel --skip_tests --enable_training ; \ -- else \ -- ${PYTHON_EXE} ./onnxruntime/tools/ci_build/build.py \ -- --build_dir ./onnxruntime/build --use_openvino $(OV_DEVICE_PRECISION) --build_shared_lib \ -- --config Release --build_wheel --skip_tests ;\ -- fi -diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py -index 09fe99d36cc340fb145a52f319df75cdc45aa709..7180377bd744f86d9d5407aefd57cbb159024d30 100644 ---- a/tools/nuget/generate_nuspec_for_native_nuget.py -+++ b/tools/nuget/generate_nuspec_for_native_nuget.py -@@ -732,7 +732,7 @@ def generate_files(line_list, args): - ) - - if args.execution_provider == "openvino": -- openvino_path = get_env_var("INTEL_OPENVINO_DIR") -+ get_env_var("INTEL_OPENVINO_DIR") - files_list.append( - "' - ) - -- if is_windows(): -- dll_list_path = os.path.join(openvino_path, "runtime\\bin\\intel64\\Release\\") -- tbb_list_path = os.path.join(openvino_path, "runtime\\3rdparty\\tbb\\bin\\") -- -- for dll_element in os.listdir(dll_list_path): -- if dll_element.endswith("dll"): -- files_list.append( -- "' -- ) -- -- for tbb_element in os.listdir(tbb_list_path): -- if tbb_element.endswith("dll"): -- files_list.append( -- "' -- ) -- - if args.execution_provider == "cuda" or is_cuda_gpu_win_sub_package and not is_ado_packaging_build: - files_list.append( - " - - namespace onnxruntime::kernel_type_str_resolver_utils { - -diff --git a/onnxruntime/core/optimizer/compute_optimizer/upstream_gather_actors.h b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather_actors.h -index 0c21be1..3ee6061 100644 ---- a/onnxruntime/core/optimizer/compute_optimizer/upstream_gather_actors.h -+++ b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather_actors.h -@@ -5,6 +5,8 @@ - // while so far we mainly validate training during cooking the optimization. - #ifdef ENABLE_TRAINING - #pragma once -+#include -+#include - - #include "core/optimizer/compute_optimizer/shared_utils.h" - -diff --git a/onnxruntime/core/optimizer/compute_optimizer/upstream_transformer_base.h b/onnxruntime/core/optimizer/compute_optimizer/upstream_transformer_base.h -index 6e22fc7..79c8ac1 100644 ---- a/onnxruntime/core/optimizer/compute_optimizer/upstream_transformer_base.h -+++ b/onnxruntime/core/optimizer/compute_optimizer/upstream_transformer_base.h -@@ -15,6 +15,7 @@ - #include - #include - #include -+#include - - namespace onnxruntime::optimizer::compute_optimizer { - -diff --git a/orttraining/orttraining/core/optimizer/memory_optimizer/transformer_specific.cc b/orttraining/orttraining/core/optimizer/memory_optimizer/transformer_specific.cc -index 04f2679..9c4de02 100644 ---- a/orttraining/orttraining/core/optimizer/memory_optimizer/transformer_specific.cc -+++ b/orttraining/orttraining/core/optimizer/memory_optimizer/transformer_specific.cc -@@ -1,6 +1,7 @@ - // Copyright (c) Microsoft Corporation. All rights reserved. - // Licensed under the MIT License. - -+#include - #include - #include - #include +diff --git a/onnxruntime/core/framework/kernel_type_str_resolver.cc b/onnxruntime/core/framework/kernel_type_str_resolver.cc +index d05e02e..6f8a47c 100644 +--- a/onnxruntime/core/framework/kernel_type_str_resolver.cc ++++ b/onnxruntime/core/framework/kernel_type_str_resolver.cc +@@ -50,7 +50,7 @@ Status KernelTypeStrResolver::ResolveKernelTypeStr(const Node& node, std::string + // TODO(edgchen1) maybe we can use transparent hash/eq to enable lookup with string_view + const auto type_str_it = type_str_map.find(std::string(kernel_type_str)); + #else +- const auto type_str_it = type_str_map.find(kernel_type_str); ++ const auto type_str_it = type_str_map.find(std::string(kernel_type_str)); + #endif + + ORT_RETURN_IF(type_str_it == type_str_map.end(), +diff --git a/onnxruntime/core/framework/ort_value_name_idx_map.h b/onnxruntime/core/framework/ort_value_name_idx_map.h +index 1b5f6bc..b721205 100644 +--- a/onnxruntime/core/framework/ort_value_name_idx_map.h ++++ b/onnxruntime/core/framework/ort_value_name_idx_map.h +@@ -36,7 +36,7 @@ class OrtValueNameIdxMap { + #ifdef DISABLE_ABSEIL + auto it = map_.find(std::string(name)); + #else +- auto it = map_.find(name); ++ auto it = map_.find(std::string(name)); + #endif + if (it == map_.end()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Could not find OrtValue with name '", name, "'"); diff --git a/ports/onnxruntime/onnxruntime_vcpkg_deps.cmake b/ports/onnxruntime/onnxruntime_vcpkg_deps.cmake index 224e111d3b76e8..f7fcd83825f8c6 100644 --- a/ports/onnxruntime/onnxruntime_vcpkg_deps.cmake +++ b/ports/onnxruntime/onnxruntime_vcpkg_deps.cmake @@ -2,7 +2,30 @@ message(STATUS "Using Dependencies from vcpkg...") # ABSL should be included before protobuf because protobuf may use absl find_package(absl CONFIG REQUIRED) -list(APPEND onnxruntime_EXTERNAL_LIBRARIES absl::base) +list(APPEND ABSEIL_LIBS + absl::base + absl::city + absl::core_headers + absl::fixed_array + absl::flags + absl::flat_hash_map + absl::flat_hash_set + absl::hash + absl::inlined_vector + absl::low_level_hash + absl::node_hash_map + absl::node_hash_set + absl::optional + absl::raw_hash_set + absl::raw_logging_internal + absl::span + absl::str_format + absl::strings + absl::synchronization + absl::throw_delegate + absl::time +) +list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${ABSEIL_LIBS}) find_package(re2 CONFIG REQUIRED) # re2::re2 list(APPEND onnxruntime_EXTERNAL_LIBRARIES re2::re2) @@ -53,20 +76,30 @@ if (onnxruntime_ENABLE_CPUINFO) endif() if (NOT WIN32) - find_package(unofficial-nsync CONFIG REQUIRED) # unofficial::nsync::nsync_cpp - add_library(nsync::nsync_cpp ALIAS unofficial::nsync::nsync_cpp) + find_library(NSYNC_CPP_LIBRARY NAMES nsync_cpp REQUIRED) + add_library(nsync_cpp INTERFACE IMPORTED GLOBAL) + set_target_properties(nsync_cpp PROPERTIES INTERFACE_LINK_LIBRARIES "${NSYNC_CPP_LIBRARY}") + add_library(nsync::nsync_cpp ALIAS nsync_cpp) list(APPEND onnxruntime_EXTERNAL_LIBRARIES nsync::nsync_cpp) endif() find_package(Microsoft.GSL CONFIG REQUIRED) list(APPEND onnxruntime_EXTERNAL_LIBRARIES Microsoft.GSL::GSL) +set(GSL_TARGET Microsoft.GSL::GSL) # see onnxruntime_mlas # ONNX find_package(ONNX CONFIG REQUIRED) +if(TARGET ONNX::onnx AND NOT TARGET onnx) + add_library(onnx ALIAS ONNX::onnx) +endif() +if(TARGET ONNX::onnx_proto AND NOT TARGET onnx_proto) + add_library(onnx_proto ALIAS ONNX::onnx_proto) +endif() list(APPEND onnxruntime_EXTERNAL_LIBRARIES onnx onnx_proto) find_package(Eigen3 CONFIG REQUIRED) list(APPEND onnxruntime_EXTERNAL_LIBRARIES Eigen3::Eigen) +get_property(eigen_INCLUDE_DIRS TARGET Eigen3::Eigen PROPERTY INTERFACE_INCLUDE_DIRECTORIES) find_package(wil CONFIG REQUIRED) list(APPEND onnxruntime_EXTERNAL_LIBRARIES WIL::WIL) diff --git a/ports/onnxruntime/portfile.cmake b/ports/onnxruntime/portfile.cmake index 3df16f3e618ea9..249bb0848a484a 100644 --- a/ports/onnxruntime/portfile.cmake +++ b/ports/onnxruntime/portfile.cmake @@ -3,13 +3,11 @@ string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" BUILD_SHARED) vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO microsoft/onnxruntime - REF "v${VERSION}" - SHA512 f24e333ad113e15733867fae237c3495f93e373b2998508deeebb061ce9a56c444bf68fc49ae251bcc45539d0695f3ae758d73dc3c42bc01bbd7cfaa8561c793 + REF v1.18.0 + SHA512 2e1d724eda5635fc24f93966412c197c82ee933aaea4f4ce907b5f2ee7730c1e741f2ef4d50a2d54284fc7bd05bf104bd3c56fd4466525fcd70e63c07fbb2b16 PATCHES - fix-onnxruntime-pr-19966.patch # https://github.com/microsoft/onnxruntime/pull/19966 for OpenVINO 2024.0+ fix-cmake.patch fix-sources.patch - fix-clang-cl-simd-compile.patch fix-llvm-rc-unicode.patch ) file(COPY "${CMAKE_CURRENT_LIST_DIR}/onnxruntime_vcpkg_deps.cmake" DESTINATION "${SOURCE_PATH}/cmake/external") @@ -21,11 +19,10 @@ message(STATUS "Using protoc: ${PROTOC}") find_program(FLATC NAMES flatc PATHS "${CURRENT_HOST_INSTALLED_DIR}/tools/flatbuffers" REQUIRED NO_DEFAULT_PATH NO_CMAKE_PATH) message(STATUS "Using flatc: ${FLATC}") -set(SCHEMA_DIR "${SOURCE_PATH}/onnxruntime/core/flatbuffers/schema") vcpkg_execute_required_process( - COMMAND ${FLATC} --cpp --scoped-enums --filename-suffix ".fbs" ort.fbs ort_training_checkpoint.fbs + COMMAND "${FLATC}" --cpp --scoped-enums --filename-suffix ".fbs" ort.fbs ort_training_checkpoint.fbs LOGNAME codegen-flatc-cpp - WORKING_DIRECTORY "${SCHEMA_DIR}" + WORKING_DIRECTORY "${SOURCE_PATH}/onnxruntime/core/flatbuffers/schema" ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS @@ -58,21 +55,11 @@ vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS framework onnxruntime_BUILD_OBJC nccl onnxruntime_USE_NCCL mpi onnxruntime_USE_MPI - minimal onnxruntime_ORT_MINIMAL_BUILD INVERTED_FEATURES - abseil onnxruntime_DISABLE_ABSEIL cuda onnxruntime_USE_MEMORY_EFFICIENT_ATTENTION ) -if("python" IN_LIST FEATURES) - x_vcpkg_get_python_packages( - PYTHON_VERSION 3 - PACKAGES numpy sympy - OUT_PYTHON_VAR PYTHON3 - ) -else() - vcpkg_find_acquire_program(PYTHON3) -endif() +vcpkg_find_acquire_program(PYTHON3) get_filename_component(PYTHON_PATH "${PYTHON3}" PATH) message(STATUS "Using python3: ${PYTHON3}") vcpkg_add_to_path(PREPEND "${PYTHON_PATH}") @@ -82,15 +69,13 @@ vcpkg_cmake_configure( SOURCE_PATH "${SOURCE_PATH}/cmake" OPTIONS ${FEATURE_OPTIONS} - "-DPython_EXECUTABLE:FILEPATH=${PYTHON3}" - "-DProtobuf_PROTOC_EXECUTABLE:FILEPATH=${PROTOC}" - -DBUILD_PKGCONFIG_FILES=OFF # https://github.com/microsoft/onnxruntime/blob/56b660f36940a919295e6f1e18ad3a9a93a10bf7/cmake/CMakeLists.txt#L1746-L1756 - -DCMAKE_DISABLE_FIND_PACKAGE_Git=ON + -DPython_EXECUTABLE:FILEPATH=${PYTHON3} + -DProtobuf_PROTOC_EXECUTABLE:FILEPATH=${PROTOC} + -DBUILD_PKGCONFIG_FILES=${BUILD_SHARED} -Donnxruntime_BUILD_SHARED_LIB=${BUILD_SHARED} -Donnxruntime_BUILD_WEBASSEMBLY=OFF -Donnxruntime_CROSS_COMPILING=${VCPKG_CROSSCOMPILING} - -Donnxruntime_USE_FULL_PROTOBUF=ON # minimalize protoc execution? - -Donnxruntime_USE_PREINSTALLED_EIGEN=ON + -Donnxruntime_USE_FULL_PROTOBUF=ON -Donnxruntime_USE_EXTENSIONS=OFF -Donnxruntime_USE_NNAPI_BUILTIN=${VCPKG_TARGET_IS_ANDROID} -Donnxruntime_ENABLE_CPUINFO=ON @@ -101,10 +86,11 @@ vcpkg_cmake_configure( -Donnxruntime_ENABLE_LAZY_TENSOR=OFF -Donnxruntime_NVCC_THREADS=1 # parallel compilation -Donnxruntime_DISABLE_RTTI=OFF + -Donnxruntime_DISABLE_ABSEIL=OFF -Donnxruntime_USE_NEURAL_SPEED=OFF -DUSE_NEURAL_SPEED=OFF # for ORT_BUILD_INFO - "-DORT_GIT_COMMIT:STRING=v${VERSION}" + "-DORT_GIT_COMMIT:STRING=45737400a2f3015c11f005ed7603611eaed306a6" "-DORT_GIT_BRANCH:STRING=v${VERSION}" OPTIONS_DEBUG -Donnxruntime_ENABLE_MEMLEAK_CHECKER=OFF @@ -116,17 +102,13 @@ vcpkg_cmake_configure( onnxruntime_USE_CUSTOM_DIRECTML onnxruntime_NVCC_THREADS Python_EXECUTABLE - ORT_GIT_COMMIT - ORT_GIT_BRANCH ) vcpkg_cmake_install() vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/onnxruntime) - -if(("openvino" IN_LIST FEATURES) AND VCPKG_TARGET_IS_WINDOWS) - file(RENAME "${CURRENT_PACKAGES_DIR}/debug/lib/onnxruntime_providers_openvino.dll" "${CURRENT_PACKAGES_DIR}/debug/bin/onnxruntime_providers_openvino.dll") - file(RENAME "${CURRENT_PACKAGES_DIR}/lib/onnxruntime_providers_openvino.dll" "${CURRENT_PACKAGES_DIR}/bin/onnxruntime_providers_openvino.dll") -endif() vcpkg_copy_pdbs() +if(BUILD_SHARED) + vcpkg_fixup_pkgconfig() # pkg_check_modules(libonnxruntime) +endif() file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") diff --git a/ports/onnxruntime/vcpkg.json b/ports/onnxruntime/vcpkg.json index f0a982fc191580..0de6af50f7f15d 100644 --- a/ports/onnxruntime/vcpkg.json +++ b/ports/onnxruntime/vcpkg.json @@ -1,20 +1,33 @@ { "name": "onnxruntime", - "version": "1.17.3", + "version-semver": "1.18.0", "description": "ONNX Runtime: cross-platform, high performance ML inferencing and training accelerator", "homepage": "https://onnxruntime.ai/", "license": "MIT", + "supports": "windows | linux | osx", "dependencies": [ - "boost-config", - "boost-mp11", + "abseil", + { + "name": "boost-config", + "version>=": "1.82.0" + }, + { + "name": "boost-mp11", + "version>=": "1.82.0" + }, "cpuinfo", "cxxopts", "date", + "dlpack", "eigen3", - "flatbuffers", { "name": "flatbuffers", - "host": true + "host": true, + "version>=": "23.5.26" + }, + { + "name": "flatbuffers", + "version>=": "23.5.26" }, "ms-gsl", "nlohmann-json", @@ -22,12 +35,24 @@ "name": "nsync", "platform": "!windows" }, - "onnx", + { + "name": "nsync", + "platform": "!windows", + "version>=": "1.26.0" + }, + { + "name": "onnx", + "version>=": "1.16.0" + }, "optional-lite", - "protobuf", { "name": "protobuf", - "host": true + "version>=": "3.21.12" + }, + { + "name": "protobuf", + "host": true, + "version>=": "3.21.12" }, "re2", "safeint", @@ -39,37 +64,6 @@ "name": "vcpkg-cmake-config", "host": true }, - "wil", - "zlib" - ], - "features": { - "coreml": { - "description": "Build with CoreML support", - "supports": "osx | ios" - }, - "openvino": { - "description": "Build with OpenVINO support", - "supports": "!(osx | ios | android | emscripten) & !static", - "dependencies": [ - { - "name": "openvino", - "default-features": false, - "features": [ - "cpu", - "gpu", - "onnx" - ] - } - ] - }, - "python": { - "description": "Enable python bindings", - "dependencies": [ - { - "name": "vcpkg-get-python-packages", - "host": true - } - ] - } - } + "wil" + ] }