add support llava-next-video

Author: eaidova

optimum/exporters/openvino/__main__.py

@@ -311,7 +311,7 @@ def main_export(
             and framework == "pt"
             and (
                 task.startswith("text-generation")
-                or getattr(config, "model_type", None) in MULTI_MODAL_TEXT_GENERATION_MODELS
+                or getattr(config, "model_type", "").replace("_", "-") in MULTI_MODAL_TEXT_GENERATION_MODELS
             )
             and getattr(config, "torch_dtype", torch.float32) in [torch.float16, torch.bfloat16]
         ):

optimum/exporters/openvino/model_configs.py

@@ -94,6 +94,7 @@
     JaisModelPatcher,
     LlamaModelPatcher,
     LlavaImageEmbeddingModelPatcher,
+    LlavaNextVideoImageEmbeddingModelPatcher,
     LlavaQwen2ImageEmbeddingsModelPatcher,
     MiniCPM3Patcher,
     MiniCPMModelPatcher,
@@ -140,6 +141,9 @@ def init_model_configs():
     TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS[
         "image-text-to-text"
     ] = TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["text-generation"]
+
+    TasksManager._TRANSFORMERS_TASKS_TO_MODEL_LOADERS["video-text-to-text"] = "AutoModelForVision2Seq"
+
     if is_diffusers_available() and "fill" not in TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS:
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_LOADERS["fill"] = "FluxFillPipeline"
         TasksManager._DIFFUSERS_TASKS_TO_MODEL_MAPPINGS["fill"] = {"flux": "FluxFillPipeline"}
@@ -1591,6 +1595,118 @@ class LlavaNextOpenVINOConfig(LlavaOpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = version.parse("4.40.0")
 
 
+class DummyLLavaMultiModalProjectorInputGenerator(DummyInputGenerator):
+    SUPPORTED_INPUT_NAMES = ["image_features"]
+
+    def __init__(
+        self,
+        task: str,
+        normalized_config: NormalizedTextConfig,
+        batch_size: int = DEFAULT_DUMMY_SHAPES["batch_size"],
+        random_batch_size_range: Optional[Tuple[int, int]] = None,
+        **kwargs,
+    ):
+        self.task = task
+
+        self.batch_size = batch_size
+        self.hidden_size = normalized_config.hidden_size
+        self.num_patches = (normalized_config.image_size // normalized_config.patch_size) ** 2
+        self.normalized_config = normalized_config
+
+    def generate(
+        self,
+        input_name: str,
+        framework: str = "pt",
+        int_dtype: str = "int64",
+        float_dtype: str = "fp32",
+    ):
+        shape = [self.batch_size, self.num_patches, self.hidden_size]
+        return self.random_float_tensor(shape, framework=framework, dtype=float_dtype)
+
+
+class LLavaMultimodalProjectorOpenVINOConfig(OnnxConfig):
+    DUMMY_INPUT_GENERATOR_CLASSES = (DummyLLavaMultiModalProjectorInputGenerator,)
+    NORMALIZED_CONFIG_CLASS = NormalizedVisionConfig
+
+    @property
+    def inputs(self) -> Dict[str, Dict[int, str]]:
+        return {"image_features": {0: "batch_size", 1: "sequence_length"}}
+
+    @property
+    def outputs(self) -> Dict[str, Dict[int, str]]:
+        return {"hidden_states": {0: "batch_size", 1: "sequence_length"}}
+
+
+class LlavaNextVideoConfigBehavior(str, enum.Enum):
+    LANGUAGE = "language"
+    VISION_EMBEDDINGS = "vision_embeddings"
+    VISION_RESAMPLER = "vision_resampler"
+    MULTI_MODAL_PROJECTOR = "multi_modal_projector"
+    TEXT_EMBEDDINGS = "text_embeddings"
+
+
+@register_in_tasks_manager(
+    "llava-next-video", *["image-text-to-text", "video-text-to-text"], library_name="transformers"
+)
+class LlavaNextVideoOpenVINOConfig(LlavaOpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = version.parse("4.42.0")
+    SUPPORTED_BEHAVIORS = [model_type.value for model_type in LlavaNextVideoConfigBehavior]
+
+    def with_behavior(
+        self,
+        behavior: Union[str, LlavaNextVideoConfigBehavior],
+    ):
+        """
+        Creates a config for different behaviour.
+
+        Args:
+            behavior ([`ConfigBehavior`]):
+                The behavior to use for the new instance.
+        """
+        if isinstance(behavior, str) and not isinstance(behavior, LlavaNextVideoConfigBehavior):
+            behavior = LlavaNextVideoConfigBehavior(behavior)
+
+        if behavior == LlavaNextVideoConfigBehavior.MULTI_MODAL_PROJECTOR:
+            export_config = LLavaMultimodalProjectorOpenVINOConfig(
+                self._orig_config.vision_config,
+                task="feature-extraction",
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+            )
+            return export_config
+
+        if behavior == LlavaNextVideoConfigBehavior.VISION_RESAMPLER:
+            export_config = LLavaMultimodalProjectorOpenVINOConfig(
+                self._orig_config.vision_config,
+                task="feature-extraction",
+                int_dtype=self.int_dtype,
+                float_dtype=self.float_dtype,
+            )
+            return export_config
+
+        return super().with_behavior(behavior)
+
+    def get_model_for_behavior(self, model, behavior: Union[str, LlavaNextVideoConfigBehavior]):
+        if isinstance(behavior, str) and not isinstance(behavior, LlavaNextVideoConfigBehavior):
+            behavior = LlavaNextVideoConfigBehavior(behavior)
+
+        if behavior == LlavaNextVideoConfigBehavior.MULTI_MODAL_PROJECTOR:
+            return model.multi_modal_projector
+
+        if behavior == LlavaNextVideoConfigBehavior.VISION_RESAMPLER:
+            return model.vision_resampler
+
+        return super().get_model_for_behavior(model, behavior)
+
+    def patch_model_for_export(
+        self, model: Union["PreTrainedModel", "TFPreTrainedModel"], model_kwargs: Optional[Dict[str, Any]] = None
+    ):
+        model_kwargs = model_kwargs or {}
+        if self._behavior != LlavaConfigBehavior.VISION_EMBEDDINGS:
+            return super().patch_model_for_export(model, model_kwargs)
+        return LlavaNextVideoImageEmbeddingModelPatcher(self, model, model_kwargs)
+
+
 @register_in_tasks_manager(
     "maira2", *["image-text-to-text", "text-generation", "text-generation-with-past"], library_name="transformers"
 )

optimum/exporters/openvino/model_patcher.py

@@ -3111,6 +3111,28 @@ def llava_vision_embed_forward(self, pixel_values):
     return image_features
 
 
+def llava_next_video_vision_embed_forward(self, pixel_values):
+    # copied from https://github.com/huggingface/transformers/blob/v4.44.2/src/transformers/models/llava/modeling_llava.py#L428-L441
+    # these changes does not bring any difference from original, it only packs model subcomponent inference together
+    # that allow us avoid memory overheads and their inference results handling on code-level
+    image_features = self.vision_tower(pixel_values, output_hidden_states=True)
+    # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
+    vision_feature_layer = self.config.vision_feature_layer
+    if isinstance(vision_feature_layer, int):
+        selected_image_feature = image_features.hidden_states[vision_feature_layer]
+    else:
+        hs_pool = [image_features.hidden_states[layer_idx] for layer_idx in vision_feature_layer]
+        selected_image_feature = torch.cat(hs_pool, dim=-1)
+
+    if self.config.vision_feature_select_strategy == "default":
+        selected_image_feature = selected_image_feature[:, 1:]
+    elif self.config.vision_feature_select_strategy == "full":
+        selected_image_feature = selected_image_feature
+    else:
+        raise ValueError(f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}")
+    return selected_image_feature
+
+
 class LlavaImageEmbeddingModelPatcher(ModelPatcher):
     def __init__(
         self,
@@ -3128,6 +3150,23 @@ def __exit__(self, exc_type, exc_value, traceback):
         self._model.forward = self._model.__orig_forward
 
 
+class LlavaNextVideoImageEmbeddingModelPatcher(ModelPatcher):
+    def __init__(
+        self,
+        config: "OnnxConfig",
+        model: Union["PreTrainedModel", "TFPreTrainedModel"],
+        model_kwargs: Dict[str, Any],
+    ):
+        model.__orig_forward = model.forward
+        model.forward = types.MethodType(llava_next_video_vision_embed_forward, model)
+
+        super().__init__(config, model, model_kwargs)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        super().__exit__(exc_type, exc_value, traceback)
+        self._model.forward = self._model.__orig_forward
+
+
 def _embednb_forward(self, ids: torch.Tensor) -> torch.Tensor:
     def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor:
         assert dim % 2 == 0, "The dimension must be even."

optimum/exporters/openvino/utils.py

@@ -220,6 +220,7 @@ def get_submodels(model):
 MULTI_MODAL_TEXT_GENERATION_MODELS = [
     "llava",
     "llava-next",
+    "llava-next-video",
     "llava-qwen2",
     "internvl-chat",
     "maira2",
@@ -299,7 +300,7 @@ def save_preprocessors(
                 preprocessors[1].chat_template = getattr(preprocessors[0], "chat_template", None)
         if (
             is_transformers_version(">=", "4.45")
-            and model_type in ["llava", "llava-next"]
+            and model_type in ["llava", "llava-next", "llava-next-video"]
             and preprocessors is not None
         ):
             if getattr(preprocessors[1], "patch_size", None) is None: