add set_chat_template

Author: Wovchena

src/cpp/include/openvino/genai/vlm_pipeline.hpp

@@ -54,6 +54,14 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
     std::string m_templated_chat_history;
     size_t image_id = 0;  // Used to insert <image_id>i</image_id> per image (not a slice).
 
+    /// @brief Construct a pipeline form a folder containing tokenizer
+    /// and model IRs.
+    /// @param model_dir A folder to read tokenizer and model IRs.
+    /// @param device Inference device. A tokenizer is always compiled
+    /// for CPU.
+    /// @param device_config A config to pass to ov::Core.set_property()
+    /// and ov::Core::compile_model().
+    /// @param core ov::Core instance to use.
     explicit VLMPipeline(
         const std::filesystem::path& model_dir,
         const std::string& device="CPU",
@@ -67,6 +75,14 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
         core
     } {}
 
+    /// @brief Construct a pipeline form a folder containing model IRs
+    /// and from a Tokenizer instance.
+    /// @param model_dir A folder to read model IRs.
+    /// @param tokenizer An instance of Tokenizer to use.
+    /// @param device Inference device.
+    /// @param device_config A config to pass to ov::Core.set_property()
+    /// and ov::Core::compile_model().
+    /// @param core ov::Core instance to use.
     VLMPipeline(
         const std::filesystem::path& model_dir,
         const ov::genai::Tokenizer& tokenizer,
@@ -78,16 +94,37 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
     /// @brief Default destructor.
     ~VLMPipeline();
 
+    /// @brief Generate a response given a prompt and any number of
+    /// uint8 RGB images.
+    /// @param prompt A prompt to respond to.
+    /// @param images Images to be prepended to a prompt.
+    /// @param generation_config A config to follow for text generation.
+    /// @param streamer A streamer to acquire intermidiate result.
+    /// @return A string generated by a model.
     DecodedResults generate(
         const std::string& prompt,
-        const std::vector<ov::Tensor>& images,
+        const std::vector<ov::Tensor>& rgbs,
         const GenerationConfig& generation_config,
         const StreamerVariant& streamer
     );
+    /// @brief Generate a response given a prompt and config.
+    /// @param prompt A prompt to respond to.
+    /// @param config_map A config may contain GenerationConfig, values
+    /// for its members, StreamerVariant a single image or multiple
+    /// images.
+    /// @return A string generated by a model.
     DecodedResults generate(
         const std::string& prompt,
         const ov::AnyMap& config_map
     );
+    /// @brief Generate a response given a prompt and arbitrary number
+    /// of ov::Property instances.
+    /// Example:
+    /// generate("text", image(std::move(rgb)), do_sample(true));
+    /// @param prompt A prompt to respond to.
+    /// @param ...properties ov::Property instances to be combined into
+    /// ov::AnyMap.
+    /// @return A string generated by a model.
     template <typename... Properties>
     util::EnableIfAllStringAny<DecodedResults, Properties...> generate(
         const std::string& prompt,
@@ -97,9 +134,30 @@ class OPENVINO_GENAI_EXPORTS VLMPipeline {
             prompt, AnyMap{std::forward<Properties>(properties)...}
         );
     }
+    /// @brief Activate chat mode. Chat preserves previous history and
+    /// applies chat_template to input prompts. Calling start_chat()
+    /// again or finish_chat() drops the memorized history.
+    /// It's possible to disable
+    /// chat_template application by calling
+    /// set_chat_template("{% for message in messages %}{{ message['content'] }}{% endfor %}")
+    /// @param system_message Some chat_templates contain system role
+    /// in addition to user and assistant roles. Set a message for that
+    /// role.
     void start_chat(const std::string& system_message="");
+    /// @brief Deactivate chat mode.
     void finish_chat() {m_is_chat_conversation = false;}
+    /// @brief Set a custom chat template. Can be used to deactivate
+    /// chat_template application for chat mode if called with
+    /// "{% for message in messages %}{{ message['content'] }}{% endfor %}"
+    /// or workaround unsupported chat_template entries in a default
+    /// model chat_template.
+    /// @param new_template A new template to override with.
+    void set_chat_template(const std::string& new_template);
+    /// @brief Extract GenerationConfig used to get default values.
+    /// @return Default values used.
     GenerationConfig get_generation_config() const;
+    /// @brief Override default values for GenerationConfig
+    /// @param new_config A config to override default values with.
     void set_generation_config(const GenerationConfig& new_config);
 private:
     class VLMPipelineImpl;

src/cpp/src/vlm_pipeline.cpp

@@ -333,15 +333,15 @@ ov::genai::VLMPipeline::~VLMPipeline() = default;
 
 DecodedResults VLMPipeline::generate(
     const std::string& prompt,
-    const std::vector<ov::Tensor>& images,
+    const std::vector<ov::Tensor>& rgbs,
     const GenerationConfig& generation_config,
     const StreamerVariant& streamer
 ) {
     std::string images_prompt;
     EncodedImage embeds;
-    if (!images.empty()) {
-        OPENVINO_ASSERT(1 == images.size(), "TODO: Only a single image allowed");
-        embeds = m_vision_encoder.encode(images.at(0));
+    if (!rgbs.empty()) {
+        OPENVINO_ASSERT(1 == rgbs.size(), "TODO: Only a single image allowed");
+        embeds = m_vision_encoder.encode(rgbs.at(0));
         if (m_vlm_config.use_image_id) {
             images_prompt = m_vlm_config.im_id_start + std::to_string(image_id) + m_vlm_config.im_id_end;
             ++image_id;
@@ -403,7 +403,7 @@ DecodedResults VLMPipeline::generate(
         m_vlm_config.hidden_size == inputs_embeds.get_shape().at(2),
         "Unexpected embedding size"
     );
-    if (!images.empty()) {
+    if (!rgbs.empty()) {
         int64_t* ids = input_ids.data<int64_t>();
         const ov::Tensor& resampled_source = resample(*this, embeds.resized_source, {embeds.resized_source_size});
         float* emb = resampled_source.data<float>();
@@ -575,6 +575,10 @@ void VLMPipeline::start_chat(const std::string& system_message) {
     m_templated_chat_history = m_tokenizer.apply_chat_template(m_history, add_generation_prompt);
 }
 
+void VLMPipeline::set_chat_template(const std::string& new_template) {
+    m_tokenizer.set_chat_template(new_template);
+}
+
 GenerationConfig VLMPipeline::get_generation_config() const {
     return m_generation_config;
 }