Merge branch 'main' into ns/q-config-kwargs

Author: nikita-savelyevv

.github/workflows/test_ipex.yml

@@ -18,8 +18,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        transformers-version: ["4.47.0", "4.47.1"]
-        torch-version: ["2.4.0", "2.5.*"]
+        transformers-version: ["4.47.*"]
+        torch-version: ["2.6.0"]
 
     runs-on: ubuntu-22.04
 

docs/source/openvino/models.mdx

@@ -106,6 +106,7 @@ Here is the list of the supported architectures :
 - Qwen2(Qwen1.5, Qwen2.5)
 - Qwen2MoE
 - Qwen2VL
+- Qwen2.5VL
 - ResNet
 - Roberta
 - Roformer

optimum/commands/export/openvino.py

@@ -429,7 +429,7 @@ def run(self):
                 maybe_convert_tokenizers(library_name, self.args.output, model, task=task)
         elif (
             quantize_with_dataset
-            and (task.startswith("text-generation") or task == "automatic-speech-recognition")
+            and (task.startswith("text-generation") or "automatic-speech-recognition" in task)
             or (task == "image-text-to-text" and quantization_config is not None)
         ):
             if task.startswith("text-generation"):

optimum/exporters/openvino/model_configs.py

@@ -105,6 +105,7 @@
     PersimmonModelPatcher,
     Phi3ModelPatcher,
     Phi3VisionImageEmbeddingsPatcher,
+    Qwen2_5_VLVisionEmbMergerPatcher,
     Qwen2VLLanguageModelPatcher,
     Qwen2VLVisionEmbMergerPatcher,
     QwenModelPatcher,
@@ -131,10 +132,14 @@ def init_model_configs():
         "transformers",
         "Qwen2VLForConditionalGeneration",
     )
+    TasksManager._CUSTOM_CLASSES[("pt", "qwen2-5-vl", "image-text-to-text")] = (
+        "transformers",
+        "AutoModelForImageTextToText",
+    )
+
     TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS[
         "image-text-to-text"
     ] = TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["text-generation"]
-
     if is_diffusers_available() and "fill" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS:
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS["fill"] = "FluxFillPipeline"
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["fill"] = {"flux": "FluxFillPipeline"}
@@ -2510,7 +2515,13 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
 
 
 class DummyQwen2VLVisionEmbedInputGenerator(DummyVisionInputGenerator):
-    SUPPORTED_INPUT_NAMES = ("hidden_states", "attention_mask", "rotary_pos_emb")
+    SUPPORTED_INPUT_NAMES = (
+        "hidden_states",
+        "attention_mask",
+        "window_attention_mask",
+        "window_index",
+        "rotary_pos_emb",
+    )
 
     def __init__(
         self,
@@ -2529,10 +2540,17 @@ def __init__(
         self.temporal_patch_size = normalized_config.config.temporal_patch_size
         self.patch_size = normalized_config.config.patch_size
         if normalized_config.use_embed_dim:
-            self.embed_dim = normalized_config.config.embed_dim
+            self.embed_dim = (
+                normalized_config.config.embed_dim
+                if hasattr(normalized_config.config, "embed_dim")
+                else normalized_config.hidden_size
+            )
         else:
             self.embed_dim = self.num_channels * self.temporal_patch_size * self.patch_size * self.patch_size
         self.num_heads = normalized_config.config.num_heads
+        self.spatial_merge_size = None
+        if hasattr(normalized_config.config, "spatial_merge_size"):
+            self.spatial_merge_size = normalized_config.config.spatial_merge_size
 
     def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int64", float_dtype: str = "fp32"):
         grid_h, grid_w = self.height // self.patch_size, self.width // self.patch_size
@@ -2543,7 +2561,7 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
                 [grid_t * grid_h * grid_w, self.embed_dim], framework=framework, dtype=float_dtype
             )
 
-        if input_name == "attention_mask":
+        if input_name in ["attention_mask", "window_attention_mask"]:
             return self.random_mask_tensor(
                 [1, grid_t * grid_h * grid_w, grid_t * grid_h * grid_w], framework=framework, dtype=float_dtype
             )
@@ -2552,6 +2570,15 @@ def generate(self, input_name: str, framework: str = "pt", int_dtype: str = "int
             dim = self.embed_dim // self.num_heads // 2
             return self.random_float_tensor([grid_h * grid_t * grid_w, dim], framework=framework, dtype=float_dtype)
 
+        if input_name == "window_index":
+            if self.spatial_merge_size is None:
+                raise ValueError(
+                    "`spatial_merge_size` parameter is not found in model config. Can not generate dummy input data for `window_index` input"
+                )
+            spatial_merge_unit = self.spatial_merge_size * self.spatial_merge_size
+            hidden_size = (grid_t * grid_h * grid_w) // spatial_merge_unit
+            return self.random_int_tensor([hidden_size], max_value=hidden_size)
+
 
 class Qwen2VLConfigBehavior(str, enum.Enum):
     LANGUAGE = "language"
@@ -2674,7 +2701,7 @@ def patch_model_for_export(
 
     @property
     def inputs(self) -> Dict[str, Dict[int, str]]:
-        if self._behavior == Phi3VisionConfigBehavior.VISION_EMBEDDINGS:
+        if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS:
             return {"hidden_states": {0: "patch_thw_grid", 1: "patch_temporal_channels"}}
         if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER:
             return {
@@ -2690,6 +2717,31 @@ def outputs(self) -> Dict[str, Dict[int, str]]:
         return {}
 
 
+@register_in_tasks_manager("qwen2-5-vl", *["image-text-to-text"], library_name="transformers")
+class Qwen2_5_VLOpenVINOConfig(Qwen2VLOpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = version.parse("4.49.0")
+
+    @property
+    def inputs(self) -> Dict[str, Dict[int, str]]:
+        if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return {
+                "hidden_states": {0: "sequence_length"},
+                "attention_mask": {1: "sequence_length", 2: "sequence_length"},
+                "window_attention_mask": {1: "sequence_length", 2: "sequence_length"},
+                "window_index": {0: "unit_sequence_length"},
+                "rotary_pos_emb": {0: "sequence_length"},
+            }
+        return super().inputs
+
+    def patch_model_for_export(
+        self, model: Union["PreTrainedModel", "TFPreTrainedModel"], model_kwargs: Optional[Dict[str, Any]] = None
+    ):
+        model_kwargs = model_kwargs or {}
+        if self._behavior == Qwen2VLConfigBehavior.VISION_EMBEDDINGS_MERGER:
+            return Qwen2_5_VLVisionEmbMergerPatcher(self, model, model_kwargs)
+        return super().patch_model_for_export(model, model_kwargs)
+
+
 @register_in_tasks_manager(
     "glm",
     *[

optimum/exporters/openvino/model_patcher.py

@@ -3909,29 +3909,8 @@ def __exit__(self, exc_type, exc_value, traceback):
         self._model.forward = self._model.__orig_forward
 
 
-class Qwen2VLVisionEmbMergerPatcher(ModelPatcher):
-    def __init__(
-        self,
-        config: "OnnxConfig",
-        model: Union["PreTrainedModel", "TFPreTrainedModel"],
-        model_kwargs: Dict[str, Any] = None,
-    ):
-        model.__orig_forward = model.forward
-
-        # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L1118
-        # added attention_mask input instead cu_lens for its internal calculation model (unsupported by tracing due to cycle with dynamic len)
-        # separated patch_embed and rot_pos_emb calls for performing as part of another model
-        def image_embed_forward(
-            self, hidden_states: torch.Tensor, attention_mask: torch.Tensor, rotary_pos_emb: torch.Tensor
-        ) -> torch.Tensor:
-            for blk in self.blocks:
-                hidden_states = blk(hidden_states, attention_mask=attention_mask, rotary_pos_emb=rotary_pos_emb)
-            return self.merger(hidden_states)
-
-        model.forward = types.MethodType(image_embed_forward, model)
-        super().__init__(config, model, model_kwargs)
-
-    def __enter__(self):
+def patch_qwen2vl_vision_blocks(model, force_new_behaviour=False):
+    if not force_new_behaviour and is_transformers_version("<=", "4.48.99"):
         # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L390
         # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len)
         def sdpa_attn_forward(
@@ -3976,11 +3955,165 @@ def block_forward(self, hidden_states, attention_mask, rotary_pos_emb) -> torch.
             hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
             return hidden_states
 
+    else:
+        # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L391
+        # added attention_mask input instead of internal calculation (unsupported by tracing due to cycle with dynamic len)
+        def sdpa_attn_forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: torch.Tensor,
+            rotary_pos_emb: torch.Tensor = None,
+            position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        ):
+            def rotate_half(x):
+                """Rotates half the hidden dims of the input."""
+                x1 = x[..., : x.shape[-1] // 2]
+                x2 = x[..., x.shape[-1] // 2 :]
+                return torch.cat((-x2, x1), dim=-1)
+
+            def apply_rotary_pos_emb_vision(
+                q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
+            ) -> Tuple[torch.Tensor, torch.Tensor]:
+                orig_q_dtype = q.dtype
+                orig_k_dtype = k.dtype
+                q, k = q.float(), k.float()
+                cos, sin = cos.unsqueeze(-2), sin.unsqueeze(-2)
+                q_embed = (q * cos) + (rotate_half(q) * sin)
+                k_embed = (k * cos) + (rotate_half(k) * sin)
+                q_embed = q_embed.to(orig_q_dtype)
+                k_embed = k_embed.to(orig_k_dtype)
+                return q_embed, k_embed
+
+            seq_length = hidden_states.shape[0]
+            q, k, v = self.qkv(hidden_states).reshape(seq_length, 3, self.num_heads, -1).permute(1, 0, 2, 3).unbind(0)
+            if position_embeddings is None:
+                emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)
+                cos = emb.cos().float()
+                sin = emb.sin().float()
+            else:
+                cos, sin = position_embeddings
+            q, k = apply_rotary_pos_emb_vision(q, k, cos, sin)
+            q = q.transpose(0, 1)
+            k = k.transpose(0, 1)
+            v = v.transpose(0, 1)
+            attn_output = torch.nn.functional.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0)
+            attn_output = attn_output.transpose(0, 1)
+            attn_output = attn_output.reshape(seq_length, -1)
+            attn_output = self.proj(attn_output)
+            return attn_output
+
+        # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L446
+        # added attention_mask input propagation to self.attn
+        def block_forward(
+            self,
+            hidden_states,
+            attention_mask,
+            rotary_pos_emb: Optional[torch.Tensor] = None,
+            position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        ) -> torch.Tensor:
+            hidden_states = hidden_states + self.attn(
+                self.norm1(hidden_states),
+                attention_mask=attention_mask,
+                rotary_pos_emb=rotary_pos_emb,
+                position_embeddings=position_embeddings,
+            )
+            hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))
+            return hidden_states
+
+    for block in model.blocks:
+        block._orig_forward = block.forward
+        block.forward = types.MethodType(block_forward, block)
+        block.attn._orig_forward = block.attn.forward
+        block.attn.forward = types.MethodType(sdpa_attn_forward, block.attn)
+
+
+class Qwen2VLVisionEmbMergerPatcher(ModelPatcher):
+    def __init__(
+        self,
+        config: "OnnxConfig",
+        model: Union["PreTrainedModel", "TFPreTrainedModel"],
+        model_kwargs: Dict[str, Any] = None,
+    ):
+        model.__orig_forward = model.forward
+
+        # Modified from https://github.com/huggingface/transformers/blob/v4.45.2/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L1118
+        # added attention_mask input instead cu_lens for its internal calculation model (unsupported by tracing due to cycle with dynamic len)
+        # separated patch_embed and rot_pos_emb calls for performing as part of another model
+        def image_embed_forward(
+            self, hidden_states: torch.Tensor, attention_mask: torch.Tensor, rotary_pos_emb: torch.Tensor
+        ) -> torch.Tensor:
+            for blk in self.blocks:
+                hidden_states = blk(hidden_states, attention_mask=attention_mask, rotary_pos_emb=rotary_pos_emb)
+            return self.merger(hidden_states)
+
+        model.forward = types.MethodType(image_embed_forward, model)
+        super().__init__(config, model, model_kwargs)
+
+    def __enter__(self):
+        patch_qwen2vl_vision_blocks(self._model)
+        super().__enter__()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super().__exit__(exc_type, exc_value, traceback)
+        self._model.forward = self._model.__orig_forward
         for block in self._model.blocks:
-            block._orig_forward = block.forward
-            block.forward = types.MethodType(block_forward, block)
-            block.attn._orig_forward = block.attn.forward
-            block.attn.forward = types.MethodType(sdpa_attn_forward, block.attn)
+            block.forward = block._orig_forward
+            block.attn.forward = block.attn._orig_forward
+
+
+class Qwen2_5_VLVisionEmbMergerPatcher(ModelPatcher):
+    def __init__(
+        self,
+        config: "OnnxConfig",
+        model: Union["PreTrainedModel", "TFPreTrainedModel"],
+        model_kwargs: Dict[str, Any] = None,
+    ):
+        super().__init__(config, model, model_kwargs)
+
+        model.__orig_forward = model.forward
+
+        # Modified from https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py#L405
+        # added attention_mask and window_attention_mask inputs instead cu_lens and window_cu_lens processing for its internal calculation model
+        # (unsupported by tracing due to cycle with dynamic len)
+        # separated patch_embed and rot_pos_emb calls for performing as part of another model
+        def image_embed_forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: torch.Tensor,
+            window_attention_mask: torch.Tensor,
+            window_index: torch.Tensor,
+            rotary_pos_emb: torch.Tensor,
+        ) -> torch.Tensor:
+            seq_len = hidden_states.shape[0]
+            hidden_states = hidden_states.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
+            hidden_states = hidden_states[window_index, :, :]
+            hidden_states = hidden_states.reshape(seq_len, -1)
+            rotary_pos_emb = rotary_pos_emb.reshape(seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
+            rotary_pos_emb = rotary_pos_emb[window_index, :, :]
+            rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
+            emb = torch.cat((rotary_pos_emb, rotary_pos_emb), dim=-1)
+            position_embeddings = (emb.cos(), emb.sin())
+            for layer_num, blk in enumerate(self.blocks):
+                if layer_num in self.fullatt_block_indexes:
+                    attention_mask_now = attention_mask
+                else:
+                    attention_mask_now = window_attention_mask
+                hidden_states = blk(
+                    hidden_states, attention_mask=attention_mask_now, position_embeddings=position_embeddings
+                )
+
+            hidden_states = self.merger(hidden_states)
+            reverse_indices = torch.argsort(window_index)
+            hidden_states = hidden_states[reverse_indices, :]
+
+            return hidden_states
+
+        model.forward = types.MethodType(image_embed_forward, model)
+        super().__init__(config, model, model_kwargs)
+
+    def __enter__(self):
+        patch_qwen2vl_vision_blocks(self._model, force_new_behaviour=True)
+        super().__enter__()
 
     def __exit__(self, exc_type, exc_value, traceback):
         super().__exit__(exc_type, exc_value, traceback)

optimum/exporters/openvino/utils.py

@@ -226,6 +226,7 @@ def get_submodels(model):
     "minicpmv",
     "phi3-v",
     "qwen2-vl",
+    "qwen2-5-vl",
 ]