apply review comments

Author: eaidova

optimum/exporters/openvino/model_configs.py

@@ -132,13 +132,15 @@ def init_model_configs():
         "transformers",
         "Qwen2VLForConditionalGeneration",
     )
-    TasksManager._CUSTOM_CLASSES[("pt", "qwen2-5-vl", "image-text-to-text")] = (
-        "transformers",
-        "Qwen2_5_VLForConditionalGeneration",
+
+    TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["image-text-to-text"] = (
+        (
+            "AutoModelForImageTextToText",
+            "AutoModelForCausalLM",
+        )
+        if is_transformers_version(">=", "4.46")
+        else TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["text-generation"]
     )
-    TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS[
-        "image-text-to-text"
-    ] = TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["text-generation"]
 
     if is_diffusers_available() and "fill" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS:
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS["fill"] = "FluxFillPipeline"
@@ -2571,6 +2573,10 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
             return self.random_float_tensor([grid_h * grid_t * grid_w, dim], framework=framework, dtype=float_dtype)
 
         if input_name == "window_index":
+            if self.spatial_merge_size:
+                raise ValueError(
+                    "`spatial_merge_size` parameter is not found in model config. Can not generate dummy input data for `window_index` input"
+                )
             spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
             hidden_size = (grid_t * grid_h * grid_w) // spatial_merge_unit
             return self.random_int_tensor([hidden_size], max_value=hidden_size)

optimum/intel/openvino/modeling_visual_language.py

@@ -2386,6 +2386,7 @@ def get_rope_index(
         second_per_grid_ts: Optional[torch.Tensor] = None,
         attention_mask: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, torch.Tensor]:
+        # modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L1546
         """
         Calculate the 3D rope index based on image and video's temporal, height and width in LLM.
         """
@@ -2597,6 +2598,7 @@ def get_multimodal_embeddings(
         second_per_grid_ts: Optional[torch.Tensor] = None,
         **kwargs,
     ):
+        # Adopted from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L1791-L1861
         inputs_embeds = torch.from_numpy(self.get_text_embeddings(input_ids))
         if pixel_values is not None and input_ids.shape[1] != 1:
             image_embeds = torch.from_numpy(self.get_vision_embeddings(pixel_values, image_grid_thw))

tests/openvino/test_modeling.py

@@ -2148,7 +2148,11 @@ class OVModelForVisualCausalLMIntegrationTest(unittest.TestCase):
     )
 
     def get_transformer_model_class(self, model_arch):
-        if model_arch == "llava":
+        if is_transformers_version(">=", "4.46") and model_arch in ["llava", "llava_next", "qwen2_vl", "qwen2_5_vl"]:
+            from transformers import AutoModelForImageTextToText
+
+            return AutoModelForImageTextToText
+        if model_arch in "llava":
             from transformers import LlavaForConditionalGeneration
 
             return LlavaForConditionalGeneration
@@ -2160,10 +2164,6 @@ def get_transformer_model_class(self, model_arch):
             from transformers import Qwen2VLForConditionalGeneration
 
             return Qwen2VLForConditionalGeneration
-        if model_arch == "qwen2_5_vl":
-            from transformers import Qwen2_5_VLForConditionalGeneration
-
-            return Qwen2_5_VLForConditionalGeneration
         return AutoModelForCausalLM
 
     def _check_device_and_request(self, ov_model, expected_device, has_request):