attempt to resolve 4.49 compatibility issues and fix input processing for internvl2

Author: eaidova

optimum/intel/openvino/modeling_visual_language.py

@@ -1426,12 +1426,18 @@ def load_image(image, input_size=448, max_num=12):
             pixel_values = [transform(image) for image in images]
             pixel_values = torch.stack(pixel_values)
             return pixel_values
+        
+        if image is not None and "<image>" not in text:
+            text = "<image>\n" + text
 
+        if tokenizer.chat_template is not None:
+            text = tokenizer.apply_chat_template([{"role": "user", "content": text}], add_generation_prompt=True, tokenize=False)
+        
+        inputs = {}
+            
         if image is not None:
             if config is None:
                 raise ValueError("Config is required.")
-            if "<image>" not in text:
-                text = "<image>\n" + text
             pixel_values = load_image(image, input_size=config.vision_config.image_size)
             num_patches = pixel_values.shape[0]
             num_image_token = int(
@@ -1440,11 +1446,9 @@ def load_image(image, input_size=448, max_num=12):
             )
             image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * num_image_token * num_patches + IMG_END_TOKEN
             text = text.replace("<image>", image_tokens, 1)
-            text_inputs = tokenizer(text, return_tensors="pt")
-            inputs = dict(text_inputs)
+            logger.warn(text)
             inputs.update({"pixel_values": pixel_values})
-        else:
-            inputs = tokenizer(text, return_tensors="pt")
+        inputs.update(tokenizer(text, return_tensors="pt"))
         return inputs
 
     # internvl has issue with check  _get_non_default_parameters, as wrkaraund overide _prepare_generation_config

tests/openvino/test_modeling.py

@@ -1040,12 +1040,6 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
     def test_compare_to_transformers(self, model_arch):
         model_id = MODEL_NAMES[model_arch]
 
-        # TODO: add back once dtype fixed everywhere
-        # https://huggingface.co/katuni4ka/tiny-random-chatglm2/blob/main/modeling_chatglm.py#L720
-        # https://huggingface.co/katuni4ka/tiny-random-chatglm2/blob/main/modeling_chatglm.py#L759
-        if model_arch in {"chatglm", "glm4"} and is_transformers_version(">=", "4.49"):
-            self.skipTest("Incompatible modeling code")
-
         not_stateful = []
         if is_openvino_version("<", "2024.0"):
             not_stateful.append("mixtral")
@@ -1126,7 +1120,7 @@ def test_compare_to_transformers(self, model_arch):
         ov_outputs = ov_model.generate(**tokens, generation_config=gen_config)
 
         # TODO: add back once https://huggingface.co/katuni4ka/tiny-random-minicpm3/discussions/1 merged (for all models) as current mdoeling incompatible with transformers >= v4.49
-        if model_arch in {"minicpm", "minicpm3", "arctic", "deepseek"} and is_transformers_version(">=", "4.49"):
+        if model_arch in {"deepseek"} and is_transformers_version(">=", "4.49"):
             self.skipTest("Incompatible modeling code")
 
         additional_inputs = {}
@@ -2240,10 +2234,6 @@ def test_compare_to_transformers(self, model_arch):
         ov_outputs = ov_model.generate(**inputs, generation_config=gen_config)
         set_seed(SEED)
 
-        # TODO: add back once https://huggingface.co/katuni4ka/tiny-random-minicpm3/discussions/1 merged for all models as current mdoeling incompatible with transformers >= v4.49
-        if model_arch in {"phi3_v", "nanollava"} and is_transformers_version(">=", "4.49"):
-            self.skipTest("Incompatible modeling code")
-
         with torch.no_grad():
             transformers_outputs = transformers_model.generate(**transformers_inputs, generation_config=gen_config)