[C API] remote tensor support (openvinotoolkit#13719)

* [C API] remote tensor support

Provide C interface for remote context and remote tensor:
   1. OCL and VA context and buffer support
   2. unite test for remote context and remote tensor

Change-Id: I2c449aef21cbe928ca470b4e3bcf1e03a1d1ca43

* Fix clang issue

Change-Id: I83c9592d21ff9cb8aeb85148277d96db74b455c7

* [CAPI] Add ocl nv12 input inference test case

1. Add fully nv12(2 ocl remote tensor) as input plus preprocess do csc+resize, then do inference
2. Add get_device_name for remote tensor
3. Add test case for preprocess to set mem type

Change-Id: Ieaab50c8de20e5c7258697030672e0b010627a81

* Update documentation

Change-Id: Ia7dbaea48d38f5534aba60fbb25cd0a1f2f9eab0

* Remove debug code

Change-Id: Ic5c5a24d3c40bb258b7007dcea44594af2d92344

* Fix issues brought by rebase

Change-Id: I2520c5ccf3620349e202ea40c08bb1c437d5af88

* Resolve document issue

Change-Id: Ia14500f8623147f481dda286a0afaa8ecfffa7c9

* Resolve some comments
1. Add specific header file for gpu plugin
2. clang-format issue
3. interface compatible issue

Change-Id: Icc4723af071af30f0422ac9a107e57ddeec94aac

* fix clang issue

Change-Id: I46e1fed3dd9a4e51260b695dc3fb194b9571ed58

* Add gpu header file directory

Change-Id: I8c15d9da58a46c070dcc68530cb2beea8cd4bba9

* Remove HAVE_OCL_SUPPORT macro

Change-Id: I10093a99c1858649f1c5502248729704fcec34ef

* Address some comments

Change-Id: I72830288d063623641e8946c8470631e81fdeb34

* Printov:AnyMmap with the help of ov::Any

Change-Id: I8abd3a8d94ba8116974c59a489cda2af15f225d7

Author: riverlijunjie

src/bindings/c/include/openvino/c/gpu/gpu_plugin_properties.h

@@ -0,0 +1,61 @@
+// Copyright (C) 2018-2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief This is a specified header file for gpu plugin's properties
+ *
+ * @file gpu_plugin_properties.h
+ */
+
+#pragma once
+#include "openvino/c/ov_common.h"
+
+/**
+ * @brief gpu plugin properties key for remote context/tensor
+ */
+
+//!< Read-write property: shared device context type, can be either pure OpenCL (OCL) or
+//!< shared video decoder (VA_SHARED) context.
+//!< Value is string, it can be one of below strings:
+//!<    "OCL"       - Pure OpenCL context
+//!<    "VA_SHARED" - Context shared with a video decoding device
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_context_type;
+
+//!< Read-write property<void *>: identifies OpenCL context handle in a shared context or shared memory blob
+//!< parameter map.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_ocl_context;
+
+//!< Read-write property<int string>: ID of device in OpenCL context if multiple devices are present in the context.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_ocl_context_device_id;
+
+//!< Read-write property<int string>: In case of multi-tile system, this key identifies tile within given context.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_tile_id;
+
+//!< Read-write property<void *>: OpenCL queue handle in a shared context
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_ocl_queue;
+
+//!< Read-write property<void *>: video acceleration device/display handle in a shared context or shared
+//!< memory blob parameter map.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_va_device;
+
+//!< Read-write property: type of internal shared memory in a shared memory blob
+//!< parameter map.
+//!< Value is string, it can be one of below strings:
+//!<    "OCL_BUFFER"        - Shared OpenCL buffer blob
+//!<    "OCL_IMAGE2D"       - Shared OpenCL 2D image blob
+//!<    "USM_USER_BUFFER"   - Shared USM pointer allocated by user
+//!<    "USM_HOST_BUFFER"   - Shared USM pointer type with host allocation type allocated by plugin
+//!<    "USM_DEVICE_BUFFER" - Shared USM pointer type with device allocation type allocated by plugin
+//!<    "VA_SURFACE"        - Shared video decoder surface or D3D 2D texture blob
+//!<    "DX_BUFFER"         - Shared D3D buffer blob
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_shared_mem_type;
+
+//!< Read-write property<void *>: OpenCL memory handle in a shared memory blob parameter map.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_mem_handle;
+
+//!< Read-write property<uint32_t string>: video decoder surface handle in a shared memory blob parameter map.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_dev_object_handle;
+
+//!< Read-write property<uint32_t string>: video decoder surface plane in a shared memory blob parameter map.
+OPENVINO_C_VAR(const char*) ov_property_key_intel_gpu_va_plane;

src/bindings/c/include/openvino/c/openvino.h

@@ -29,3 +29,4 @@
 #include "openvino/c/ov_rank.h"
 #include "openvino/c/ov_shape.h"
 #include "openvino/c/ov_tensor.h"
+#include "openvino/c/ov_remote_context.h"

src/bindings/c/include/openvino/c/ov_common.h

@@ -120,6 +120,10 @@
  * @defgroup ov_tensor_c_api Tensor
  * @ingroup ov_c_api
  * @brief The definitions & operations about tensor
+ * 
+ * @defgroup ov_remote_context_c_api ov_remote_context
+ * @ingroup ov_c_api
+ * @brief Set of functions representing of RemoteContext
  */
 
 /**

src/bindings/c/include/openvino/c/ov_compiled_model.h

@@ -16,6 +16,7 @@
 #include "openvino/c/ov_model.h"
 #include "openvino/c/ov_node.h"
 #include "openvino/c/ov_property.h"
+#include "openvino/c/ov_remote_context.h"
 
 /**
  * @struct ov_compiled_model_t
@@ -183,3 +184,15 @@ ov_compiled_model_export_model(const ov_compiled_model_t* compiled_model,
  */
 OPENVINO_C_API(void)
 ov_compiled_model_free(ov_compiled_model_t* compiled_model);
+
+/**
+ * @brief Returns pointer to device-specific shared context
+ * on a remote accelerator device that was used to create this CompiledModel.
+ * @ingroup ov_compiled_model_c_api
+ * @param compiled_model A pointer to the ov_compiled_model_t.
+ * @param context Return context.
+ * @return Status code of the operation: OK(0) for success.
+ *
+ */
+OPENVINO_C_API(ov_status_e)
+ov_compiled_model_get_context(const ov_compiled_model_t* compiled_model, ov_remote_context_t** context);

src/bindings/c/include/openvino/c/ov_core.h

@@ -15,6 +15,7 @@
 #include "openvino/c/ov_model.h"
 #include "openvino/c/ov_node.h"
 #include "openvino/c/ov_property.h"
+#include "openvino/c/ov_remote_context.h"
 #include "openvino/c/ov_tensor.h"
 
 /**
@@ -341,7 +342,54 @@ ov_core_get_versions_by_device_name(const ov_core_t* core,
 /**
  * @brief Releases memory occupied by ov_core_version_list_t.
  * @ingroup ov_core_c_api
- * @param vers A pointer to the ie_core_versions to free memory.
+ * @param versions A pointer to the ie_core_versions to free memory.
  */
-OPENVINO_C_API(void)
-ov_core_versions_free(ov_core_version_list_t* versions);
+OPENVINO_C_API(void) ov_core_versions_free(ov_core_version_list_t* versions);
+
+/**
+ * @brief Creates a new remote shared context object on the specified accelerator device
+ * using specified plugin-specific low-level device API parameters (device handle, pointer, context, etc.).
+ * @ingroup ov_core_c_api
+ * @param core A pointer to the ov_core_t instance.
+ * @param device_name Device name to identify a plugin.
+ * @param context_args_size How many property args will be for this remote context creation.
+ * @param context A pointer to the newly created remote context.
+ * @param variadic parmameters Actual context property parameter for remote context
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_core_create_context(const ov_core_t* core,
+                       const char* device_name,
+                       const size_t context_args_size,
+                       ov_remote_context_t** context,
+                       ...);
+
+/**
+ * @brief Creates a compiled model from a source model within a specified remote context.
+ * @ingroup ov_core_c_api
+ * @param core A pointer to the ov_core_t instance.
+ * @param model Model object acquired from ov_core_read_model.
+ * @param context A pointer to the newly created remote context.
+ * @param property_args_size How many args will be for this compiled model.
+ * @param compiled_model A pointer to the newly created compiled_model.
+ * @param variadic parmameters Actual property parameter for remote context
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_core_compile_model_with_context(const ov_core_t* core,
+                                   const ov_model_t* model,
+                                   const ov_remote_context_t* context,
+                                   const size_t property_args_size,
+                                   ov_compiled_model_t** compiled_model,
+                                   ...);
+
+/**
+ * @brief Gets a pointer to default (plugin-supplied) shared context object for the specified accelerator device.
+ * @ingroup ov_core_c_api
+ * @param core A pointer to the ov_core_t instance.
+ * @param device_name Name of a device to get a default shared context from.
+ * @param context A pointer to the referenced remote context.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_core_get_default_context(const ov_core_t* core, const char* device_name, ov_remote_context_t** context);

src/bindings/c/include/openvino/c/ov_prepostprocess.h

@@ -274,6 +274,7 @@ ov_preprocess_preprocess_steps_reverse_channels(ov_preprocess_preprocess_steps_t
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
  * @param element_type A point to element_type
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_input_tensor_info_set_element_type(
@@ -285,30 +286,59 @@ ov_preprocess_input_tensor_info_set_element_type(
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
  * @param colorFormat The enumerate of colorFormat
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
-ov_preprocess_input_tensor_info_set_color_format(
-                                ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info,
-                                const ov_color_format_e colorFormat);
+ov_preprocess_input_tensor_info_set_color_format(ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info,
+                                                 const ov_color_format_e colorFormat);
+
+/**
+ * @brief Set ov_preprocess_input_tensor_info_t color format with subname.
+ * @ingroup ov_prepostprocess_c_api
+ * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
+ * @param colorFormat The enumerate of colorFormat
+ * @param sub_names_size The size of sub_names
+ * @param variadic params sub_names Optional list of sub-names assigned for each plane (e.g. "Y", "UV").
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_preprocess_input_tensor_info_set_color_format_with_subname(
+    ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info,
+    const ov_color_format_e colorFormat,
+    const size_t sub_names_size,
+    ...);
 
 /**
  * @brief Set ov_preprocess_input_tensor_info_t spatial_static_shape.
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
  * @param input_height The height of input
  * @param input_width The width of input
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_input_tensor_info_set_spatial_static_shape(
                                 ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info,
                                 const size_t input_height,
                                 const size_t input_width);
 
+/**
+ * @brief Set ov_preprocess_input_tensor_info_t memory type.
+ * @ingroup prepostprocess
+ * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
+ * @param mem_type Memory type. Refer to ov_remote_context.h to get memory type string info.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_preprocess_input_tensor_info_set_memory_type(ov_preprocess_input_tensor_info_t* preprocess_input_tensor_info,
+                                                const char* mem_type);
+
 /**
  * @brief Convert ov_preprocess_preprocess_steps_t element type.
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_steps A pointer to the ov_preprocess_preprocess_steps_t.
  * @param element_type preprocess input element type.
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_preprocess_steps_convert_element_type(
@@ -320,6 +350,7 @@ ov_preprocess_preprocess_steps_convert_element_type(
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_steps A pointer to the ov_preprocess_preprocess_steps_t.
  * @param colorFormat The enumerate of colorFormat.
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_preprocess_steps_convert_color(
@@ -331,6 +362,7 @@ ov_preprocess_preprocess_steps_convert_color(
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
  * @param tensor A point to ov_tensor_t
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_input_tensor_info_set_from(
@@ -342,6 +374,7 @@ ov_preprocess_input_tensor_info_set_from(
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_tensor_info A pointer to the ov_preprocess_input_tensor_info_t.
  * @param layout A point to ov_layout_t
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_input_tensor_info_set_layout(
@@ -422,6 +455,7 @@ ov_preprocess_output_tensor_info_free(
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_output_tensor_info A pointer to the ov_preprocess_output_tensor_info_t.
  * @param element_type A point to element_type
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_output_set_element_type(
@@ -453,6 +487,7 @@ ov_preprocess_input_model_info_free(ov_preprocess_input_model_info_t* preprocess
  * @ingroup ov_prepostprocess_c_api
  * @param preprocess_input_model_info A pointer to the ov_preprocess_input_model_info_t
  * @param layout A point to ov_layout_t
+ * @return Status code of the operation: OK(0) for success.
  */
 OPENVINO_C_API(ov_status_e)
 ov_preprocess_input_model_info_set_layout(

src/bindings/c/include/openvino/c/ov_remote_context.h

@@ -0,0 +1,114 @@
+// Copyright (C) 2018-2022 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+/**
+ * @brief This is a header file for ov_remote_context C API
+ *
+ * @file ov_remote_context.h
+ */
+#pragma once
+#include "openvino/c/ov_common.h"
+#include "openvino/c/ov_shape.h"
+#include "openvino/c/ov_tensor.h"
+#include "openvino/c/gpu/gpu_plugin_properties.h"
+
+typedef struct ov_remote_context ov_remote_context_t;
+
+/**
+ * @brief Allocates memory tensor in device memory or wraps user-supplied memory handle
+ * using the specified tensor description and low-level device-specific parameters.
+ * Returns a pointer to the object that implements the RemoteTensor interface.
+ * @ingroup ov_remote_context_c_api
+ * @param context A pointer to the ov_remote_context_t instance.
+ * @param type Defines the element type of the tensor.
+ * @param shape Defines the shape of the tensor.
+ * @param object_args_size Size of the low-level tensor object parameters.
+ * @param remote_tensor Pointer to returned ov_tensor_t that contains remote tensor instance.
+ * @param variadic params Contains low-level tensor object parameters.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_context_create_tensor(const ov_remote_context_t* context,
+                                const ov_element_type_e type,
+                                const ov_shape_t shape,
+                                const size_t object_args_size,
+                                ov_tensor_t** remote_tensor,
+                                ...);
+
+/**
+ * @brief Returns name of a device on which underlying object is allocated.
+ * @ingroup ov_remote_context_c_api
+ * @param context A pointer to the ov_remote_context_t instance.
+ * @param device_name Device name will be returned.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_context_get_device_name(const ov_remote_context_t* context, char** device_name);
+
+/**
+ * @brief Returns a string contains device-specific parameters required for low-level
+ * operations with the underlying object.
+ * Parameters include device/context handles, access flags,
+ * etc. Content of the returned map depends on a remote execution context that is
+ * currently set on the device (working scenario).
+ * One actaul example: "CONTEXT_TYPE OCL OCL_CONTEXT 0x5583b2ec7b40 OCL_QUEUE 0x5583b2e98ff0"
+ * @ingroup ov_remote_context_c_api
+ * @param context A pointer to the ov_remote_context_t instance.
+ * @param size The size of param pairs.
+ * @param params Param name:value list.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_context_get_params(const ov_remote_context_t* context, size_t* size, char** params);
+
+/**
+ * @brief This method is used to create a host tensor object friendly for the device in current context.
+ * For example, GPU context may allocate USM host memory (if corresponding extension is available),
+ * which could be more efficient than regular host memory.
+ * @ingroup ov_remote_context_c_api
+ * @param context A pointer to the ov_remote_context_t instance.
+ * @param type Defines the element type of the tensor.
+ * @param shape Defines the shape of the tensor.
+ * @param tensor Pointer to ov_tensor_t that contains host tensor.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_context_create_host_tensor(const ov_remote_context_t* context,
+                                     const ov_element_type_e type,
+                                     const ov_shape_t shape,
+                                     ov_tensor_t** tensor);
+
+/**
+ * @brief Release the memory allocated by ov_remote_context_t.
+ * @ingroup ov_remote_context_c_api
+ * @param context A pointer to the ov_remote_context_t to free memory.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(void) ov_remote_context_free(ov_remote_context_t* context);
+
+/**
+ * @brief Returns a string contains device-specific parameters required for low-level
+ * operations with underlying object.
+ * Parameters include device/context/surface/buffer handles, access flags,
+ * etc. Content of the returned map depends on remote execution context that is
+ * currently set on the device (working scenario).
+ * One example: "MEM_HANDLE:0x559ff6904b00;OCL_CONTEXT:0x559ff71d62f0;SHARED_MEM_TYPE:OCL_BUFFER;"
+ * @ingroup ov_remote_context_c_api
+ * @param tensor Pointer to ov_tensor_t that contains host tensor.
+ * @param size The size of param pairs.
+ * @param params Param name:value list.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_tensor_get_params(ov_tensor_t* tensor, size_t* size, char** params);
+
+/**
+ * @brief Returns name of a device on which underlying object is allocated.
+ * @ingroup ov_remote_context_c_api
+ * @param remote_tensor A pointer to the remote tensor instance.
+ * @param device_name Device name will be return.
+ * @return Status code of the operation: OK(0) for success.
+ */
+OPENVINO_C_API(ov_status_e)
+ov_remote_tensor_get_device_name(ov_tensor_t* remote_tensor, char** device_name);