From 5ce07b84847619d2c81d02e457ff4d13c73102d2 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Mon, 1 Jul 2024 23:42:38 +0530 Subject: [PATCH 1/2] feat: Enable OVEP build with Static OV Libraries --- cmake/CMakeLists.txt | 9 +++-- cmake/onnxruntime_providers_openvino.cmake | 45 +++++++++++++++++++++- tools/ci_build/build.py | 7 +++- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index c4412e0934f17..e9c866b2f71e1 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -87,6 +87,7 @@ option(onnxruntime_USE_CUDA_NHWC_OPS "Build CUDA with NHWC op support" OFF) option(onnxruntime_CUDA_MINIMAL "Build CUDA without any operations apart from memcpy ops. Usefuel for a very minial TRT build" OFF) option(onnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO "When building with CUDA support, generate device code line number information." OFF) option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF) +option(onnxruntime_USE_OPENVINO_STATIC_LIBS "Build with OpenVINO support as Static Library" OFF) option(onnxruntime_USE_COREML "Build with CoreML support" OFF) option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF) option(onnxruntime_USE_QNN "Build with QNN support" OFF) @@ -140,7 +141,7 @@ option(onnxruntime_ARMNN_RELU_USE_CPU "Use the CPU implementation for the Relu o option(onnxruntime_ARMNN_BN_USE_CPU "Use the CPU implementation for the Batch Normalization operator for the ArmNN EP" ON) option(onnxruntime_ENABLE_INSTRUMENT "Enable Instrument with Event Tracing for Windows (ETW)" OFF) option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF) -cmake_dependent_option(onnxruntime_USE_MIMALLOC "Override new/delete and arena allocator with mimalloc" OFF "WIN32;NOT onnxruntime_USE_CUDA;NOT onnxruntime_USE_OPENVINO" OFF) +cmake_dependent_option(onnxruntime_USE_MIMALLOC "Override new/delete and arena allocator with mimalloc" OFF "WIN32;NOT onnxruntime_USE_CUDA;NOT onnxruntime_USE_OPENVINO; NOT onnxruntime_USE_OPENVINO_STATIC_LIBS" OFF) option(onnxruntime_USE_CANN "Build with CANN support" OFF) option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF) option(onnxruntime_USE_TVM "Build with TVM support" OFF) @@ -606,7 +607,7 @@ if (WIN32) # structure was padded due to __declspec(align()) list(APPEND ORT_WARNING_FLAGS "/wd4324") # warning C4800: Implicit conversion from 'X' to bool. Possible information loss - if (onnxruntime_USE_OPENVINO) + if (onnxruntime_USE_OPENVINO OR onnxruntime_USE_OPENVINO_STATIC_LIBS) list(APPEND ORT_WARNING_FLAGS "/wd4800") endif() # operator 'operator-name': deprecated between enumerations of different types @@ -782,7 +783,7 @@ if (onnxruntime_USE_DNNL) list(APPEND ONNXRUNTIME_PROVIDER_NAMES dnnl) list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_DNNL=1) endif() -if (onnxruntime_USE_OPENVINO) +if (onnxruntime_USE_OPENVINO OR onnxruntime_USE_OPENVINO_STATIC_LIBS) list(APPEND ORT_PROVIDER_FLAGS -DUSE_OPENVINO=1) list(APPEND ONNXRUNTIME_PROVIDER_NAMES openvino) list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_OPENVINO=1) @@ -1343,7 +1344,7 @@ if (onnxruntime_ENABLE_TRAINING_APIS) ) endif() -if (onnxruntime_USE_OPENVINO) +if (onnxruntime_USE_OPENVINO OR onnxruntime_USE_OPENVINO_STATIC_LIBS) add_definitions(-DUSE_OPENVINO=1) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 5876b2b5c448b..3069a75f958ca 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -18,11 +18,50 @@ # Header paths find_package(OpenVINO REQUIRED COMPONENTS Runtime ONNX) if(OpenVINO_VERSION VERSION_LESS 2023.0) - message(FATAL_ERROR "OpenVINO 2023.0 and newer are supported. Please, latest OpenVINO release") + message(FATAL_ERROR "OpenVINO 2023.0 and newer are supported. Please, latest OpenVINO release") endif() if (WIN32) - unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) + unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) + endif() + + if(onnxruntime_USE_OPENVINO_STATIC_LIBS) + # Get the INTEL_OPENVINO_DIR environment variable + file(TO_CMAKE_PATH "$ENV{INTEL_OPENVINO_DIR}" OpenVINO_BASE_DIR) + + # Define the suffix path + set(OPENVINO_SUFFIX_PATH "runtime/lib/intel64/Release") + + # Combine the base directory with the suffix path + file(TO_CMAKE_PATH "${OpenVINO_BASE_DIR}/${OPENVINO_SUFFIX_PATH}" OPENVINO_STATIC_LIB_DIR) + + # Check if the combined directory exists, If the directory exists, proceed with setting up the static libraries + if(IS_DIRECTORY "${OPENVINO_STATIC_LIB_DIR}") + # Initialize an empty list to hold the found static libraries + set(OPENVINO_FOUND_STATIC_LIBS) + + # Use the appropriate file extension for static libraries based on the host operating system + if(CMAKE_HOST_WIN32) + set(OPENVINO_STATIC_LIB_EXT "*.lib") + elseif(CMAKE_HOST_UNIX) + set(OPENVINO_STATIC_LIB_EXT "*.a") + endif() + + # Use GLOB_RECURSE to find all static library files in the specified directory based on the OS + file(GLOB_RECURSE OPENVINO_POSSIBLE_LIBS "${OPENVINO_STATIC_LIB_DIR}/${OPENVINO_STATIC_LIB_EXT}") + + # Iterate over each possible library and check if it exists before appending + foreach(lib ${OPENVINO_POSSIBLE_LIBS}) + if(EXISTS "${lib}") + list(APPEND OPENVINO_FOUND_STATIC_LIBS "${lib}") + endif() + endforeach() + + # Append the found static library files to the OPENVINO_LIB_LIST + list(APPEND OPENVINO_LIB_LIST ${OPENVINO_FOUND_STATIC_LIBS}) + else() + message(FATAL_ERROR "The specified OpenVINO static library directory does not exist: ${OPENVINO_STATIC_LIB_DIR}") + endif() endif() list(APPEND OPENVINO_LIB_LIST openvino::frontend::onnx openvino::runtime ${PYTHON_LIBRARIES}) @@ -38,6 +77,8 @@ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/) set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime") + set_target_properties(onnxruntime_providers_openvino PROPERTIES INTERPROCEDURAL_OPTIMIZATION_RELEASE ON INTERPROCEDURAL_OPTIMIZATION_DEBUG ON INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO ON) + if(NOT MSVC) target_compile_options(onnxruntime_providers_openvino PRIVATE "-Wno-parentheses") endif() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b73a17db3ce13..41f2264ad71fa 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -547,6 +547,8 @@ def convert_arg_line_to_args(self, arg_line): type=_openvino_verify_device_type, help="Build with OpenVINO for specific hardware.", ) + parser.add_argument("--use_openvino_static_libs", action="store_true", + help="Build with OpenVINO built as Static Library.") parser.add_argument( "--dnnl_aarch64_runtime", action="store", default="", type=str.lower, help="e.g. --dnnl_aarch64_runtime acl" ) @@ -1227,9 +1229,10 @@ def generate_build_tree( if args.winml_root_namespace_override: cmake_args += ["-Donnxruntime_WINML_NAMESPACE_OVERRIDE=" + args.winml_root_namespace_override] - if args.use_openvino: + if args.use_openvino or args.use_openvino_static_libs: cmake_args += [ "-Donnxruntime_USE_OPENVINO=ON", + "-Donnxruntime_USE_OPENVINO_STATIC_LIBS=" + ("ON" if args.use_openvino_static_libs else "OFF"), "-Donnxruntime_NPU_NO_FALLBACK=" + ("ON" if args.use_openvino == "NPU_NO_CPU_FALLBACK" else "OFF"), "-Donnxruntime_USE_OPENVINO_GPU=" + ("ON" if args.use_openvino == "GPU" else "OFF"), "-Donnxruntime_USE_OPENVINO_CPU=" + ("ON" if args.use_openvino == "CPU" else "OFF"), @@ -1244,7 +1247,7 @@ def generate_build_tree( ] # VitisAI and OpenVINO providers currently only support full_protobuf option. - if args.use_full_protobuf or args.use_openvino or args.use_vitisai or args.gen_doc: + if args.use_full_protobuf or args.use_openvino or args.use_openvino_static_libs or args.use_vitisai or args.gen_doc: cmake_args += ["-Donnxruntime_USE_FULL_PROTOBUF=ON", "-DProtobuf_USE_STATIC_LIBS=ON"] if args.use_tvm and args.llvm_path is not None: From abccf238b736b90612fac918016e682c22cbbc22 Mon Sep 17 00:00:00 2001 From: ankitm3k Date: Tue, 9 Jul 2024 10:52:44 +0530 Subject: [PATCH 2/2] update: Enable OVEP build with Static OV Libraries for different devices --- cmake/CMakeLists.txt | 3 ++ cmake/onnxruntime_providers_openvino.cmake | 36 +++++++++++++++++++--- tools/ci_build/build.py | 26 +++++++++++++--- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e9c866b2f71e1..7e5896ded50ba 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -88,6 +88,9 @@ option(onnxruntime_CUDA_MINIMAL "Build CUDA without any operations apart from me option(onnxruntime_ENABLE_CUDA_LINE_NUMBER_INFO "When building with CUDA support, generate device code line number information." OFF) option(onnxruntime_USE_OPENVINO "Build with OpenVINO support" OFF) option(onnxruntime_USE_OPENVINO_STATIC_LIBS "Build with OpenVINO support as Static Library" OFF) +option(onnxruntime_USE_OPENVINO_CPU_DEVICE "Build with OpenVINO support as Static Library for CPU Device" OFF) +option(onnxruntime_USE_OPENVINO_GPU_DEVICE "Build with OpenVINO support as Static Library for GPU Device" OFF) +option(onnxruntime_USE_OPENVINO_NPU_DEVICE "Build with OpenVINO support as Static Library for NPU Device" OFF) option(onnxruntime_USE_COREML "Build with CoreML support" OFF) option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF) option(onnxruntime_USE_QNN "Build with QNN support" OFF) diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 3069a75f958ca..70754beb27c57 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -18,11 +18,11 @@ # Header paths find_package(OpenVINO REQUIRED COMPONENTS Runtime ONNX) if(OpenVINO_VERSION VERSION_LESS 2023.0) - message(FATAL_ERROR "OpenVINO 2023.0 and newer are supported. Please, latest OpenVINO release") + message(FATAL_ERROR "OpenVINO 2023.0 and newer are supported. Please, latest OpenVINO release") endif() if (WIN32) - unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) + unset(CMAKE_MAP_IMPORTED_CONFIG_RELWITHDEBINFO) endif() if(onnxruntime_USE_OPENVINO_STATIC_LIBS) @@ -50,13 +50,41 @@ # Use GLOB_RECURSE to find all static library files in the specified directory based on the OS file(GLOB_RECURSE OPENVINO_POSSIBLE_LIBS "${OPENVINO_STATIC_LIB_DIR}/${OPENVINO_STATIC_LIB_EXT}") - # Iterate over each possible library and check if it exists before appending - foreach(lib ${OPENVINO_POSSIBLE_LIBS}) + # Copy all libraries to OPENVINO_COMMON_LIBS + set(OPENVINO_COMMON_LIBS ${OPENVINO_POSSIBLE_LIBS}) + + # Define the patterns for device-specific libraries + set(OPENVINO_DEVICE_PATTERNS "cpu" "gpu" "npu") + + # Filter out device-specific libraries from OPENVINO_COMMON_LIBS + foreach(device_pattern IN LISTS OPENVINO_DEVICE_PATTERNS) + foreach(lib IN LISTS OPENVINO_COMMON_LIBS) + string(FIND "${lib}" "${device_pattern}" device_pos) + if(NOT device_pos EQUAL -1) + list(REMOVE_ITEM OPENVINO_COMMON_LIBS "${lib}") + endif() + endforeach() + endforeach() + + # Iterate over each possible common library and check if it exists before appending + foreach(lib ${OPENVINO_COMMON_LIBS}) if(EXISTS "${lib}") list(APPEND OPENVINO_FOUND_STATIC_LIBS "${lib}") endif() endforeach() + # Iterate over each possible library for the specified device and check if it exists before appending + foreach(lib ${OPENVINO_POSSIBLE_LIBS}) + if(EXISTS "${lib}") + # Check device-specific variables and append only the required libraries + if((DEFINED onnxruntime_USE_OPENVINO_CPU_DEVICE AND onnxruntime_USE_OPENVINO_CPU_DEVICE AND lib MATCHES ".*cpu.*") OR + (DEFINED onnxruntime_USE_OPENVINO_GPU_DEVICE AND onnxruntime_USE_OPENVINO_GPU_DEVICE AND lib MATCHES ".*gpu.*") OR + (DEFINED onnxruntime_USE_OPENVINO_NPU_DEVICE AND onnxruntime_USE_OPENVINO_NPU_DEVICE AND lib MATCHES ".*npu.*")) + list(APPEND OPENVINO_FOUND_STATIC_LIBS "${lib}") + endif() + endif() + endforeach() + # Append the found static library files to the OPENVINO_LIB_LIST list(APPEND OPENVINO_LIB_LIST ${OPENVINO_FOUND_STATIC_LIBS}) else() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 41f2264ad71fa..f345b87fb44c1 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -547,8 +547,9 @@ def convert_arg_line_to_args(self, arg_line): type=_openvino_verify_device_type, help="Build with OpenVINO for specific hardware.", ) - parser.add_argument("--use_openvino_static_libs", action="store_true", - help="Build with OpenVINO built as Static Library.") + parser.add_argument("--use_openvino_static_libs", type=str, default="", + help="Build with OpenVINO built as Static Library." + "Specify the device(s) to use in the format [CPU,GPU,NPU]") parser.add_argument( "--dnnl_aarch64_runtime", action="store", default="", type=str.lower, help="e.g. --dnnl_aarch64_runtime acl" ) @@ -1229,10 +1230,9 @@ def generate_build_tree( if args.winml_root_namespace_override: cmake_args += ["-Donnxruntime_WINML_NAMESPACE_OVERRIDE=" + args.winml_root_namespace_override] - if args.use_openvino or args.use_openvino_static_libs: + if args.use_openvino: cmake_args += [ "-Donnxruntime_USE_OPENVINO=ON", - "-Donnxruntime_USE_OPENVINO_STATIC_LIBS=" + ("ON" if args.use_openvino_static_libs else "OFF"), "-Donnxruntime_NPU_NO_FALLBACK=" + ("ON" if args.use_openvino == "NPU_NO_CPU_FALLBACK" else "OFF"), "-Donnxruntime_USE_OPENVINO_GPU=" + ("ON" if args.use_openvino == "GPU" else "OFF"), "-Donnxruntime_USE_OPENVINO_CPU=" + ("ON" if args.use_openvino == "CPU" else "OFF"), @@ -1245,9 +1245,25 @@ def generate_build_tree( "-Donnxruntime_USE_OPENVINO_MULTI=" + ("ON" if args.use_openvino.startswith("MULTI") else "OFF"), "-Donnxruntime_USE_OPENVINO_AUTO=" + ("ON" if args.use_openvino.startswith("AUTO") else "OFF"), ] + if args.use_openvino_static_libs: + cmake_args += ["-Donnxruntime_USE_OPENVINO_STATIC_LIBS=ON"] + devices_str = args.use_openvino_static_libs.strip('[]') + devices = re.split(r',\s*', devices_str) + + valid_devices = {'CPU', 'GPU', 'NPU'} + if not all(device in valid_devices for device in devices): + raise ValueError("Invalid device specified. Valid devices are: CPU, GPU, NPU") + + for device in devices: + if device == 'CPU': + cmake_args.append("-Donnxruntime_USE_OPENVINO_CPU_DEVICE=ON") + if device == 'GPU': + cmake_args.append("-Donnxruntime_USE_OPENVINO_GPU_DEVICE=ON") + if device == 'NPU': + cmake_args.append("-Donnxruntime_USE_OPENVINO_NPU_DEVICE=ON") # VitisAI and OpenVINO providers currently only support full_protobuf option. - if args.use_full_protobuf or args.use_openvino or args.use_openvino_static_libs or args.use_vitisai or args.gen_doc: + if args.use_full_protobuf or args.use_openvino or args.use_vitisai or args.gen_doc: cmake_args += ["-Donnxruntime_USE_FULL_PROTOBUF=ON", "-DProtobuf_USE_STATIC_LIBS=ON"] if args.use_tvm and args.llvm_path is not None: