Add matching test

Author: Wovchena

.github/workflows/causal_lm_cpp.yml

@@ -714,10 +714,36 @@ jobs:
         run: |
           source ./ov/setupvars.sh
           python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
-          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
+          python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt opencv-python --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
           python -m pip install -U "optimum<1.23" --no-dependencies
           optimum-cli export openvino -m openbmb/MiniCPM-V-2_6 MiniCPM-V-2_6 --trust-remote-code
           wget https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11 --output-document cat.jpg
+      - name: Generate reference
+        shell: python
+        run: |
+          from optimum.intel.openvino import OVModelForVisualCausalLM
+          from transformers import AutoProcessor
+          from PIL import Image
+          import requests
+          import cv2
+          import numpy as np
+          res = 448, 448
+          im = np.arange(res[0] * res[1] * 3, dtype=np.uint8) % 255
+          im = im.reshape([*res, 3])
+          cv2.imwrite("lines.png", im)
+          model_id = "openbmb/MiniCPM-V-2_6"
+          processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+          prompt = processor.tokenizer.apply_chat_template([{"role": "user", "content": "(<image>./</image>)\nWhat is unusual on this image?"}], tokenize=False, add_generation_prompt=True)
+          image = Image.open("/home/vzlobin/r/g/g.png").convert('RGB')
+          # image = Image.open(requests.get("https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11", stream=True).raw).convert('RGB')
+          model = OVModelForVisualCausalLM.from_pretrained("MiniCPM-V-2_6", trust_remote_code=True)
+          inputs = processor([prompt], [image], return_tensors="pt")
+          result = model.generate(**inputs, max_new_tokens=200)
+          decoded = processor.tokenizer.batch_decode(result[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)[0]
+          print(decoded)
+          with open("ref.txt", "w") as f:
+              f.write(decoded)
+
       - name: Run visual_language_chat sample - MiniCPM-V-2_6
         run: >
           source ./ov/setupvars.sh

miniCPM-V-2_6.py

@@ -1,21 +1,22 @@
 from optimum.intel.openvino import OVModelForVisualCausalLM
-from transformers import AutoProcessor, AutoTokenizer
+from transformers import AutoProcessor
 from PIL import Image
 import requests
-
+import cv2
+import numpy as np
+res = 448, 448
+im = np.arange(res[0] * res[1] * 3, dtype=np.uint8) % 255
+im = im.reshape([*res, 3])
+cv2.imwrite("lines.png", im)
 model_id = "openbmb/MiniCPM-V-2_6"
-
 processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
-
-tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
-prompt = tokenizer.apply_chat_template([{"role": "user", "content": "(<image>./</image>)\nWhat is unusual on this image?"}], tokenize=False, add_generation_prompt=True)
-# image = Image.open(requests.get("https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11", stream=True).raw).convert('RGB')
+prompt = processor.tokenizer.apply_chat_template([{"role": "user", "content": "(<image>./</image>)\nWhat is unusual on this image?"}], tokenize=False, add_generation_prompt=True)
 image = Image.open("/home/vzlobin/r/g/g.png").convert('RGB')
-
+# image = Image.open(requests.get("https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11", stream=True).raw).convert('RGB')
 model = OVModelForVisualCausalLM.from_pretrained("MiniCPM-V-2_6", trust_remote_code=True)
-
 inputs = processor([prompt], [image], return_tensors="pt")
-
 result = model.generate(**inputs, max_new_tokens=200)
-
-print(processor.tokenizer.batch_decode(result[:, inputs["input_ids"].shape[1]:]))
+decoded = processor.tokenizer.batch_decode(result[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)[0]
+print(decoded)
+with open("ref.txt", "w") as f:
+    f.write(decoded)

src/cpp/src/visual_language/pipeline.cpp

@@ -413,6 +413,8 @@ class ov::genai::VLMPipeline::VLMPipelineImpl {
 
         int64_t sequence_len = m_language.get_tensor("logits").get_shape().at(1) - 1;
         size_t vocab_size = m_language.get_tensor("logits").get_shape().back();
+        float* logits = m_language.get_tensor("logits").data<float>() + sequence_len * vocab_size;
+        int64_t out_token = std::max_element(logits, logits + vocab_size) - logits;
 
         m_language.get_tensor("inputs_embeds").set_shape({BATCH_SIZE, 1, m_vlm_config.hidden_size});
         m_language.get_tensor("position_ids").set_shape({ BATCH_SIZE, 1 });
@@ -435,16 +437,6 @@ class ov::genai::VLMPipeline::VLMPipelineImpl {
         }, streamer);
         std::vector<int64_t> generated;
         while (true) {  //(out_token != eos_token_id)
-            float *logits = m_language.get_tensor("logits").data<float>();
-            int64_t out_token = std::max_element(logits, logits + vocab_size) - logits;
-            generated.push_back(out_token);
-            // if (streamer_ptr && streamer_ptr->put(out_token)) {
-            //     break;
-            // }
-            std::cout << out_token << ", ";
-            if (out_token == eos_token_id) {
-                break;
-            }
             m_embedding.get_input_tensor().data<int64_t>()[0] = out_token;
             m_embedding.infer();
             const ov::Tensor& embed_prompt_tensor = m_embedding.get_output_tensor();
@@ -459,6 +451,17 @@ class ov::genai::VLMPipeline::VLMPipelineImpl {
             m_language.get_tensor("position_ids").data<int64_t>()[0] = int64_t(m_language.get_tensor("attention_mask").get_size() - 1);
 
             m_language.infer();
+
+            generated.push_back(out_token);
+            if (streamer_ptr && streamer_ptr->put(out_token)) {
+                break;
+            }
+            logits = m_language.get_tensor("logits").data<float>();
+
+            out_token = std::max_element(logits, logits + vocab_size) - logits;
+            if (out_token == eos_token_id) {
+                break;
+            }
         }
 
         if (streamer_ptr) {