Add new Mallocator

Author: MirceaDan99

src/plugins/intel_npu/src/al/include/intel_npu/al/icompiler.hpp

@@ -151,6 +151,85 @@ struct NetworkDescription final {
     NetworkMetadata metadata;
 };
 
+template<class T>
+struct Mallocator
+{
+    typedef T value_type;
+ 
+    Mallocator() : _allocatedByUs(true) {}
+
+    Mallocator(void* preAllocatedMem, size_t size) : _size(size), _allocatedByUs(false), _constPreAllocatedMem(nullptr) {
+        _preAllocatedMem = static_cast<void*>(new(preAllocatedMem)char[_size * sizeof(T)]);
+    }
+
+    Mallocator(const void* constPreAllocatedMem, size_t size) : _size(size), _allocatedByUs(false), _preAllocatedMem(nullptr), _constPreAllocatedMem(constPreAllocatedMem) {}
+ 
+    template<class U>
+    constexpr Mallocator(const Mallocator <U>& other) noexcept {
+        // Workaround for MSFT std::_Container_Proxy
+        _allocatedByUs = true;
+        _preAllocatedMem = other._preAllocatedMem;
+        _constPreAllocatedMem = other._constPreAllocatedMem;
+        _size = other._size;
+    }
+ 
+    [[nodiscard]] T* allocate(std::size_t n)
+    {
+        if (n > std::numeric_limits<std::size_t>::max() / sizeof(T)) {
+            throw std::bad_array_new_length();
+        }
+
+        if (!_allocatedByUs) {
+            return (T*)_preAllocatedMem;
+        }
+
+        T* p = static_cast<T*>(std::malloc(n * sizeof(T)));
+        _preAllocatedMem = (void*)p;
+        return p;
+    }
+ 
+    void deallocate(T* p, std::size_t n) noexcept
+    {
+        if (_allocatedByUs) {
+            delete p;
+        }
+    }
+
+    const T* data() {
+        if (_allocatedByUs) {
+            return (const T*)_preAllocatedMem;
+        }
+        if (_preAllocatedMem != nullptr) {
+            return (const T*)_preAllocatedMem;
+        }
+        if (_constPreAllocatedMem != nullptr) {
+            return (const T*)_constPreAllocatedMem;
+        }
+        throw std::bad_alloc();
+    }
+
+    size_t size() {
+        return _size;
+    }
+private:
+
+    void* _preAllocatedMem;
+    const void* _constPreAllocatedMem;
+    size_t _size;
+
+    bool _allocatedByUs;
+
+    template <typename>
+    friend struct Mallocator;
+};
+ 
+template<class T, class U>
+bool operator==(const Mallocator <T>&, const Mallocator <U>&) { return true; }
+ 
+template<class T, class U>
+bool operator!=(const Mallocator <T>&, const Mallocator <U>&) { return false; }
+
+
 /**
  * @interface ICompiler
  * @brief An interface to be implemented by a concrete compiler to provide
@@ -203,8 +282,10 @@ class ICompiler : public std::enable_shared_from_this<ICompiler> {
     // Driver compiler can use this to release graphHandle, if we do not have executor
     virtual void release([[maybe_unused]] std::shared_ptr<const NetworkDescription> networkDescription){};
 
-    virtual std::vector<uint8_t> getCompiledNetwork(const NetworkDescription& networkDescription) {
-        return networkDescription.compiledNetwork;
+    virtual std::vector<uint8_t, Mallocator<uint8_t>> getCompiledNetwork(const NetworkDescription& networkDescription) {
+        Mallocator<uint8_t> mal(networkDescription.compiledNetwork.data(),
+                                networkDescription.compiledNetwork.size());
+        return std::vector<uint8_t, Mallocator<uint8_t>>(mal);
     }
 
 protected:

src/plugins/intel_npu/src/compiler/include/driver_compiler_adapter.hpp

@@ -36,7 +36,7 @@ class LevelZeroCompilerAdapter final : public ICompiler {
 
     void release(std::shared_ptr<const NetworkDescription> networkDescription) override;
 
-    CompiledNetwork getCompiledNetwork(const NetworkDescription& networkDescription) override;
+    std::vector<uint8_t, Mallocator<uint8_t>> getCompiledNetwork(const NetworkDescription& networkDescription) override;
 
 private:
     /**

src/plugins/intel_npu/src/compiler/include/zero_compiler_in_driver.hpp

@@ -105,7 +105,7 @@ class LevelZeroCompilerInDriver final : public ICompiler {
 
     void release(std::shared_ptr<const NetworkDescription> networkDescription) override;
 
-    std::vector<uint8_t> getCompiledNetwork(const NetworkDescription& networkDescription) override;
+    std::vector<uint8_t, Mallocator<uint8_t>> getCompiledNetwork(const NetworkDescription& networkDescription) override;
 
 private:
     NetworkMetadata getNetworkMeta(ze_graph_handle_t graphHandle) const;
@@ -129,14 +129,12 @@ class LevelZeroCompilerInDriver final : public ICompiler {
                      std::vector<IODescriptor>& outputs) const;
 
     template <typename T = TableExtension, typename std::enable_if_t<UseCopyForNativeBinary(T), bool> = true>
-    void getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
-                         ze_graph_handle_t graphHandle,
-                         std::vector<uint8_t>& blob) const;
+    std::vector<uint8_t, Mallocator<uint8_t>> getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
+                         ze_graph_handle_t graphHandle) const;
 
     template <typename T = TableExtension, typename std::enable_if_t<!UseCopyForNativeBinary(T), bool> = true>
-    void getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
-                         ze_graph_handle_t graphHandle,
-                         std::vector<uint8_t>& blob) const;
+    std::vector<uint8_t, Mallocator<uint8_t>> getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
+                         ze_graph_handle_t graphHandle) const;
 
     template <typename T = TableExtension, typename std::enable_if_t<SupportAPIGraphQueryNetworkV2(T), bool> = true>
     ze_result_t seriazlideIRModelAndQueryNetworkCreateV2(const std::shared_ptr<const ov::Model>& model,

src/plugins/intel_npu/src/compiler/src/driver_compiler_adapter.cpp

@@ -115,7 +115,7 @@ void LevelZeroCompilerAdapter::release(std::shared_ptr<const NetworkDescription>
     apiAdapter->release(std::move(networkDescription));
 }
 
-CompiledNetwork LevelZeroCompilerAdapter::getCompiledNetwork(const NetworkDescription& networkDescription) {
+std::vector<uint8_t, Mallocator<uint8_t>> LevelZeroCompilerAdapter::getCompiledNetwork(const NetworkDescription& networkDescription) {
     _logger.info("getCompiledNetwork - using adapter to perform getCompiledNetwork(networkDescription)");
     return apiAdapter->getCompiledNetwork(networkDescription);
 }

src/plugins/intel_npu/src/compiler/src/zero_compiler_in_driver.cpp

@@ -362,61 +362,14 @@ void LevelZeroCompilerInDriver<TableExtension>::release(std::shared_ptr<const Ne
     _logger.debug("release completed");
 }
 
-/*
-FAILED: src/plugins/intel_npu/src/compiler/CMakeFiles/openvino_npu_driver_compiler_adapter.dir/src/zero_compiler_in_driver.cpp.obj
-C:\Work\mirceaau\ccache-4.8.2-windows-x86_64\ccache C:\PROGRA~2\MICROS~2\2019\BUILDT~1\VC\Tools\MSVC\1429~1.301\bin\Hostx64\x64\cl.exe  /nologo /TP -DIN_OV_COMPONENT -DNPU_PLUGIN_DEVELOPER_BUILD -DOV_BUILD_POSTFIX=\"d\" -DOV_NATIVE_PARENT_PROJECT_ROOT_DIR=\"openvino\" -DOV_THREAD=OV_THREAD_TBB -DSNIPPETS_DEBUG_CAPS -DTBB_USE_DEBUG -D__TBB_NO_IMPLICIT_LINKAGE=1 -IC:\Work\mirceaau\openvino\src\plugins\intel_npu\src\compiler\include -IC:\Work\mirceaau\openvino\src\plugins\intel_npu\src\al\include -IC:\Work\mirceaau\openvino\src\plugins\intel_npu\src\utils\include -IC:\Work\mirceaau\openvino\src\inference\dev_api -IC:\Work\mirceaau\openvino\src\core\include -IC:\Work\mirceaau\openvino\src\frontends\common\include -IC:\Work\mirceaau\openvino\src\inference\include -IC:\Work\mirceaau\openvino\src\core\dev_api -IC:\Work\mirceaau\openvino\src\common\transformations\include -IC:\Work\mirceaau\openvino\src\common\low_precision_transformations\include -IC:\Work\mirceaau\openvino\src\common\itt\include -IC:\Work\mirceaau\openvino\src\common\util\include -IC:\Work\mirceaau\openvino\thirdparty\pugixml\src -IC:\Work\mirceaau\openvino\thirdparty\level_zero\level-zero\include -IC:\Work\mirceaau\openvino\src\plugins\intel_npu\thirdparty\level-zero-ext -IC:\Work\mirceaau\openvino\src\plugins\intel_npu\src\backend\include -external:IC:\Work\mirceaau\openvino\temp\tbb\include -external:W0 /DWIN32 /D_WINDOWS /GR /EHsc /D_CRT_SECURE_NO_WARNINGS /D_SCL_SECURE_NO_WARNINGS /EHsc /Gy /W3 /bigobj /MP /wd4251 /wd4275  /Z7 /Ob0 /Od /RTC1 -std:c++17 -MDd /d1trimfile:C:\Work\mirceaau\openvino\ /d1trimfile:C:/Work/mirceaau/openvino/ -WX /showIncludes /Fosrc\plugins\intel_npu\src\compiler\CMakeFiles\openvino_npu_driver_compiler_adapter.dir\src\zero_compiler_in_driver.cpp.obj /FdC:\Work\mirceaau\openvino\bin\intel64\Debug\ /FS -c C:\Work\mirceaau\openvino\src\plugins\intel_npu\src\compiler\src\zero_compiler_in_driver.cpp
-C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.29.30133\include\vector(701): error C2440: 'static_cast': cannot convert from 'intel_npu::driverCompilerAdapter::PreAllocatedAllocator<uint8_t>' to 'intel_npu::driverCompilerAdapter::PreAllocatedAllocator<U>'
-        with
-        [
-            U=std::_Container_proxy
-        ]
-*/
-template <typename T>
-class PreAllocatedAllocator
-{
-public:
-        using value_type = T;
-
-        // Constructor accepts a pointer to the pre-allocated memory block
-        PreAllocatedAllocator(T* pre_allocated, std::size_t max_size)
-            : memory(pre_allocated), max_size(max_size), offset(0) {}
-
-        // Allocator constructor and copy constructor
-        PreAllocatedAllocator(const PreAllocatedAllocator<T>& other)
-            : memory(other.memory), max_size(other.max_size), offset(other.offset) {}
-
-        T* allocate(std::size_t n) {
-            if (offset + n > max_size) {
-                throw std::bad_alloc();
-            }
-            T* ptr = memory + offset;
-            offset += n;
-            return ptr;
-        }
-
-        void deallocate(T*, std::size_t) {
-            // Deallocate does nothing since memory is externally managed
-        }
-
-        template <typename U>
-        struct rebind {
-            using other = PreAllocatedAllocator<U>;
-        };
-
-private:
-        T* memory;
-        std::size_t max_size;
-        std::size_t offset;
-
-};
-
 template <typename TableExtension>
 template <typename T, std::enable_if_t<UseCopyForNativeBinary(T), bool>>
-void LevelZeroCompilerInDriver<TableExtension>::getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
-                                                                ze_graph_handle_t graphHandle,
-                                                                std::vector<uint8_t>& blob) const {
-    // Get blob size first
-    size_t blobSize = -1;
+std::vector<uint8_t, Mallocator<uint8_t>> LevelZeroCompilerInDriver<TableExtension>::getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
+                                                                ze_graph_handle_t graphHandle) const {
+    Mallocator<uint8_t> mal;
+    std::vector<uint8_t, Mallocator<uint8_t>> blob(mal);
+    size_t blobSize;
+    
     auto result = _graphDdiTableExt.pfnGetNativeBinary(graphHandle, &blobSize, nullptr);
     blob.resize(blobSize);
 
@@ -442,71 +395,48 @@ void LevelZeroCompilerInDriver<TableExtension>::getNativeBinary(ze_graph_dditabl
                     uint64_t(result),
                     ". ",
                     getLatestBuildError());
+    return std::move(blob);
 }
 
-// Allocators of the same type are always equal
-template <typename T1, typename T2>
-bool operator==(const PreAllocatedAllocator<T1>&, const PreAllocatedAllocator<T2>&) { return true; }
-
-template <typename T1, typename T2>
-bool operator!=(const PreAllocatedAllocator<T1>&, const PreAllocatedAllocator<T2>&) { return false; }
-
 template <typename TableExtension>
 template <typename T, std::enable_if_t<!UseCopyForNativeBinary(T), bool>>
-void LevelZeroCompilerInDriver<TableExtension>::getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
-                                                                ze_graph_handle_t graphHandle,
-                                                                std::vector<uint8_t>& blob) const {
-        // Get blob ptr and size
-        uint8_t* blobPtr = nullptr;
-        size_t blobSize = -1;
-
-        auto result = _graphDdiTableExt.pfnGetNativeBinary2(graphHandle, &blobSize, &blobPtr);
-
-        OPENVINO_ASSERT(result == ZE_RESULT_SUCCESS,
-                        "Failed to compile network. L0 pfnGetNativeBinary get blob size",
-                        " result: ",
-                        ze_result_to_string(result),
-                        ", code 0x",
-                        std::hex,
-                        uint64_t(result),
-                        ". ",
-                        getLatestBuildError());
-
-        // std::initializer_list<uint8_t> initializerListTmp(blobPtr, blobPtr + blobSize);
+std::vector<uint8_t, Mallocator<uint8_t>> LevelZeroCompilerInDriver<TableExtension>::getNativeBinary(ze_graph_dditable_ext_curr_t& graphDdiTableExt,
+                                                                ze_graph_handle_t graphHandle) const {
 
-        // std::vector<uint8_t> blobTmp = {initializerListTmp.begin(), initializerListTmp.end()};
+    uint8_t* blobPtr;
+    size_t blobSize;
 
-        // blob.swap(blobTmp);
+    // Get blob ptr and size
+    auto result = _graphDdiTableExt.pfnGetNativeBinary2(graphHandle, &blobSize, &blobPtr);
 
-        /*placement_memory_allocator<uint8_t> pl(blobPtr);
-        std::vector<uint8_t, placement_memory_allocator<uint8_t>> tmpBlob(pl);
-        tmpBlob.reserve(blobSize);
-        tmpBlob.push_back(0);*/
-        // blob.swap(tmpBlob);
-
-        PreAllocatedAllocator allocator(blobPtr, blobSize);
-
-        std::vector<uint8_t, PreAllocatedAllocator<uint8_t>> tmpBlob(allocator);
+    OPENVINO_ASSERT(result == ZE_RESULT_SUCCESS,
+                    "Failed to compile network. L0 pfnGetNativeBinary get blob size",
+                    " result: ",
+                    ze_result_to_string(result),
+                    ", code 0x",
+                    std::hex,
+                    uint64_t(result),
+                    ". ",
+                    getLatestBuildError());
 
-        std::cout << tmpBlob[0] << " " << tmpBlob[1] << std::endl;
+    Mallocator<uint8_t> mal(blobPtr, blobSize);
+    return std::vector<uint8_t, Mallocator<uint8_t>>(mal);
 }
 
 template <typename TableExtension>
-std::vector<uint8_t> LevelZeroCompilerInDriver<TableExtension>::getCompiledNetwork(
+std::vector<uint8_t, Mallocator<uint8_t>> LevelZeroCompilerInDriver<TableExtension>::getCompiledNetwork(
     const NetworkDescription& networkDescription) {
     if (networkDescription.metadata.graphHandle != nullptr && networkDescription.compiledNetwork.size() == 0) {
         _logger.info("LevelZeroCompilerInDriver getCompiledNetwork get blob from graphHandle");
         ze_graph_handle_t graphHandle = static_cast<ze_graph_handle_t>(networkDescription.metadata.graphHandle);
 
-        std::vector<uint8_t> blob;
-
-        getNativeBinary(_graphDdiTableExt, graphHandle, blob);
-
         _logger.info("LevelZeroCompilerInDriver getCompiledNetwork returning blob");
-        return std::move(blob);
+        return getNativeBinary(_graphDdiTableExt, graphHandle);
     }
     _logger.info("return the blob from network description");
-    return networkDescription.compiledNetwork;
+    
+    Mallocator<uint8_t> mal(networkDescription.compiledNetwork.data(), networkDescription.compiledNetwork.size());
+    return std::vector<uint8_t, Mallocator<uint8_t>>(mal);
 }
 
 template <typename TableExtension>

src/plugins/intel_npu/src/plugin/src/compiled_model.cpp

@@ -27,10 +27,11 @@ constexpr std::string_view NO_EXECUTOR_FOR_INFERENCE =
     "Can't create infer request!\n"
     "Please make sure that the device is available. Only exports can be made.";
 
-std::uint32_t hash(const std::vector<uint8_t>& blob) {
+std::uint32_t hash(const std::vector<uint8_t, intel_npu::Mallocator<uint8_t>>& blob) {
     std::uint32_t result = 1171117u;
-    for (auto c : blob)
-        result = ((result << 7) + result) + static_cast<uint32_t>(c);
+    for (const uint8_t* it = blob.get_allocator().data(); it != blob.data() + blob.get_allocator().size(); ++it) {
+        result = ((result << 7) + result) + static_cast<uint32_t>(*it);
+    }
     return result;
 }
 
@@ -140,14 +141,14 @@ std::shared_ptr<ov::ISyncInferRequest> CompiledModel::create_sync_infer_request(
 void CompiledModel::export_model(std::ostream& stream) const {
     _logger.debug("CompiledModel::export_model");
     const auto blob = _compiler->getCompiledNetwork(*_networkPtr);
-    stream.write(reinterpret_cast<const char*>(blob.data()), blob.size());
+    stream.write(reinterpret_cast<const char*>(blob.get_allocator().data()), blob.get_allocator().size());
 
     if (!stream) {
         _logger.error("Write blob to stream failed. Blob is broken!");
     } else {
         if (_logger.level() >= ov::log::Level::INFO) {
             std::stringstream str;
-            str << "Blob size: " << blob.size() << ", hash: " << std::hex << hash(blob);
+            str << "Blob size: " << blob.get_allocator().size() << ", hash: " << std::hex << hash(blob);
             _logger.info(str.str().c_str());
         }
         _logger.info("Write blob to stream successfully.");