fix jump

Author: Wovchena

.github/workflows/causal_lm_cpp.yml

@@ -701,8 +701,13 @@ jobs:
         run: >
           curl ${{ env.m_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
           && brew install coreutils scons
-      - run: OpenVINO_DIR=./ov/runtime/cmake/ cmake -DCMAKE_BUILD_TYPE=Release -B ./build/ ./
-      - run: LD_LIBRARY_PATH=${{ github.workspace }}/ov/runtime/3rdparty/tbb/lib/:$LD_LIBRARY_PATH cmake --build ./build/ --config Release --target visual_language_chat -j
+      - run: >
+          OpenVINO_DIR=./ov/runtime/cmake/
+          TBB_DIR=./ov/runtime/3rdparty/tbb/lib/cmake
+          cmake -DCMAKE_BUILD_TYPE=Release -B ./build/ ./
+      - run: >
+          LD_LIBRARY_PATH=${{ github.workspace }}/ov/runtime/3rdparty/tbb/lib/:$LD_LIBRARY_PATH
+          cmake --build ./build/ --config Release --target visual_language_chat -j
       - run: >
           source ./ov/setupvars.sh
           && python -m pip install --upgrade-strategy eager ./thirdparty/openvino_tokenizers/[transformers] -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly

samples/cpp/visual_language_chat/visual_language_chat.cpp

@@ -9,7 +9,7 @@ bool print_subword(std::string&& subword) {
     return !(std::cout << subword << std::flush);
 }
 
-int main(int argc, char* argv[]) {
+int main(int argc, char* argv[]) try {
     if (3 != argc) {
         throw std::runtime_error(std::string{"Usage "} + argv[0] + " <MODEL_DIR> <IMAGE_FILE>");
     }
@@ -42,14 +42,14 @@ int main(int argc, char* argv[]) {
             "question:\n";
     }
     pipe.finish_chat();
-// } catch (const std::exception& error) {
-//     try {
-//         std::cerr << error.what() << '\n';
-//     } catch (const std::ios_base::failure&) {}
-//     return EXIT_FAILURE;
-// } catch (...) {
-//     try {
-//         std::cerr << "Non-exception object thrown\n";
-//     } catch (const std::ios_base::failure&) {}
-//     return EXIT_FAILURE;
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
 }

src/cpp/src/vision_encoder.cpp

@@ -285,7 +285,7 @@ EncodedImage llava_image_embed_make_with_bytes_slice(clip_ctx& ctx_clip, const o
         }
     }
     // Override prev output tensor that doesn't own memory.
-    encoder.set_output_tensor(resized_source);
+    encoder.set_output_tensor(ov::Tensor{ov::element::f32, {0, 0, old_hidden_size}});
     return {resized_source, resized_source_size, encoded_slices, sliced_sizes};
 }
 }

src/cpp/src/vlm_pipeline.cpp

@@ -380,6 +380,7 @@ DecodedResults VLMPipeline::generate(
         m_history.push_back({{"role", "user"}, {"content", images_prompt}});
         constexpr bool add_generation_prompt = true;
         std::string new_templated_chat_history = m_tokenizer.apply_chat_template(m_history, add_generation_prompt);
+        std::cout << new_templated_chat_history << '\n';
         ov::Tensor new_chat_tokens = m_tokenizer.encode(new_templated_chat_history).input_ids;
         if (0 == m_language.get_tensor("attention_mask").get_shape().at(1)) {
             encoded_input = new_chat_tokens;
@@ -418,6 +419,10 @@ DecodedResults VLMPipeline::generate(
     int64_t slice_end_id = special_tokens.data<int64_t>()[3];
     int64_t im_start_pos = 0, slice_start_pos = 0;
     int64_t* begin = encoded_input.data<int64_t>();
+    for (size_t cont = 0; cont < encoded_input.get_size(); ++cont) {
+        std::cout << begin[cont] << ", ";
+    }
+    std::cout << '\n';
     int64_t* ids = begin;
     size_t encoded_input_size = encoded_input.get_size();
     int64_t* end = ids + encoded_input_size;
@@ -426,9 +431,11 @@ DecodedResults VLMPipeline::generate(
         const ov::Tensor& resampled_source = resample(*this, encoded_image.resized_source, {encoded_image.resized_source_size});
         float* emb = resampled_source.data<float>();
         ids = std::find(ids, end, im_start_id);
+        std::cout << std::distance(begin, ids) << '\n';
         OPENVINO_ASSERT(end != ids);
         std::copy_n(emb, resampled_source.get_size(), inputs_embeds_data + std::distance(begin, ids) * m_vlm_config.hidden_size);
-        ids += m_vlm_config.hidden_size;
+        ids += m_vlm_config.query_num;
+        std::cout << std::distance(begin, ids) << '\n';
         if (encoded_image.slices) {
             size_t token_idx = 0;
             const ov::Shape& slices_shape = encoded_image.slices.get_shape();
@@ -440,9 +447,11 @@ DecodedResults VLMPipeline::generate(
                     ov::Tensor encoded_view{ov::element::f32, {1, d2, d3}, encoded_image.slices.data<float>() + (i * slices_shape.at(1) + ja) * d2 * d3};
                     const ov::Tensor& vision_embed_tensor_i_j = resample(*this, encoded_view, {sliced_sizes.at(i * slices_shape.at(1) + ja)});
                     ids = std::find(ids, end, slice_start_id);
+        std::cout << std::distance(begin, ids) << '\n';
                     OPENVINO_ASSERT(end != ids);
                     std::copy_n(vision_embed_tensor_i_j.data<float>(), vision_embed_tensor_i_j.get_size(), inputs_embeds_data + std::distance(begin, ids) * m_vlm_config.hidden_size);
-                    ids += m_vlm_config.hidden_size;
+                    ids += m_vlm_config.query_num;
+        std::cout << std::distance(begin, ids) << '\n';
                 }
             }
         }