Add support of GGUF as a input format for LLM

cmake/features.cmake

@@ -5,6 +5,7 @@
 option(ENABLE_PYTHON "Enable Python API build" ON)
 option(ENABLE_JS "Enable JS API build" OFF)
 option(ENABLE_SAMPLES "Enable samples build" ON)
+option(ENABLE_GGUF "Enable support for GGUF format" ON)
 
 # Disable building samples for NPM package
 if(CPACK_GENERATOR STREQUAL "NPM")

samples/cpp/text_generation/CMakeLists.txt

@@ -22,6 +22,7 @@ endfunction()
 
 set (SAMPLE_LIST
     greedy_causal_lm
+    gguf_example
     encrypted_model_causal_lm
     beam_search_causal_lm
     chat_sample
@@ -34,6 +35,7 @@ foreach(sample IN LISTS SAMPLE_LIST)
     add_sample_executable(${sample})
 endforeach()
 
+target_include_directories(gguf_example INTERFACE "$<BUILD_INTERFACE:${OpenVINOGenAI_SOURCE_DIR}/src/cpp/src/gguf_utils>")
 
 # benchmark_genai
 include(FetchContent)

samples/cpp/text_generation/gguf_example.cpp

@@ -0,0 +1,17 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include "openvino/genai/llm_pipeline.hpp"
+
+#include "gguf_modeling.hpp"
+
+#include "openvino/openvino.hpp"
+
+int main(int argc, char* argv[]) {
+    std::string models_path = argv[1];
+    std::string output_path = argv[2];
+
+    auto model = create_from_gguf(models_path);
+
+    ov::save_model(model, output_path + "/openvino_model.xml", false);
+}

src/cpp/CMakeLists.txt

@@ -76,7 +76,29 @@ target_include_directories(${TARGET_NAME_OBJ}
            "$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
 
-target_include_directories(${TARGET_NAME_OBJ} SYSTEM PRIVATE "${safetensors.h_SOURCE_DIR}")
+if(ENABLE_GGUF)
+    message(STATUS "Downloading gguflib")
+    FetchContent_Declare(
+      gguflib
+      URL https://github.com/antirez/gguf-tools/archive/af7d88d808a7608a33723fba067036202910acb3.zip
+      URL_HASH SHA256=d613559c7a398eb4a0919982e6a370055f8466497f0f866d331dc92b735927e7)
+    FetchContent_MakeAvailable(gguflib)
+    target_include_directories(${TARGET_NAME_OBJ}
+                               PRIVATE "${gguflib_SOURCE_DIR}")
+    target_include_directories(${TARGET_NAME_OBJ}
+                              PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/gguf_utils")
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+    add_library(gguflib STATIC ${gguflib_SOURCE_DIR}/fp16.c
+                               ${gguflib_SOURCE_DIR}/gguflib.c)
+    #target_compile_features(gguflib PRIVATE fPIC)
+    target_link_libraries(${TARGET_NAME_OBJ} PRIVATE gguflib) 
+    target_sources(${TARGET_NAME_OBJ} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/gguf_utils/gguf.cpp
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/gguf_utils/gguf_quants.cpp
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/gguf_utils/gguf_modeling.cpp
+                               ${CMAKE_CURRENT_SOURCE_DIR}/src/gguf_utils/building_blocks.cpp)
+endif()
+
+target_include_directories(${TARGET_NAME_OBJ} SYSTEM PRIVATE "${safetensors.h_SOURCE_DIR}" "${gguflib_SOURCE_DIR}")
 
 target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::runtime openvino::threading nlohmann_json::nlohmann_json jinja2cpp)
 
@@ -93,10 +115,15 @@ add_library(openvino::genai ALIAS ${TARGET_NAME})
 
 target_include_directories(${TARGET_NAME} INTERFACE "$<INSTALL_INTERFACE:runtime/include>"
                                                     "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
+                                                    "$<BUILD_INTERFACE:${OpenVINOGenAI_SOURCE_DIR}/src/cpp/src/gguf_utils>"
                                                     "$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>")
 
 target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime PRIVATE openvino::threading nlohmann_json::nlohmann_json jinja2cpp ${CMAKE_DL_LIBS})
 
+if(ENABLE_GGUF)
+  target_link_libraries(${TARGET_NAME} PRIVATE gguflib ${CMAKE_DL_LIBS})
+endif()
+
 target_compile_features(${TARGET_NAME} INTERFACE cxx_std_17)
 
 if(TARGET openvino_tokenizers)

src/cpp/src/continuous_batching_pipeline.cpp

@@ -58,17 +58,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline( const std::filesystem::p
 
     std::filesystem::path model_path = models_path;
     std::filesystem::path directory = models_path;
-    if (std::filesystem::exists(model_path / "openvino_model.xml")) {
-        model_path = model_path / "openvino_model.xml";
-    }
-    else if (std::filesystem::exists(model_path / "openvino_language_model.xml")) {
-        model_path = model_path / "openvino_language_model.xml";
-    }
-    else {
-        OPENVINO_THROW("Could not find a model in the directory.");
-    }
 
-    auto model = utils::singleton_core().read_model(model_path, {}, properties);
+    auto model = utils::read_model(model_path, properties);
     auto tokenizer = ov::genai::Tokenizer(directory, tokenizer_properties);
     auto generation_config = utils::from_config_json_if_exists(directory);