maira2 support

Author: eaidova

optimum/exporters/openvino/model_configs.py

@@ -1488,6 +1488,7 @@ def __init__(
         float_dtype: str = "fp32",
         behavior: LlavaConfigBehavior = LlavaConfigBehavior.VISION_EMBEDDINGS,
         preprocessors: Optional[List[Any]] = None,
+        **kwargs,
     ):
         super().__init__(
             config=config,
@@ -1584,6 +1585,14 @@ class LlavaNextOpenVINOConfig(LlavaOpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = version.parse("4.40.0")
 
 
+@register_in_tasks_manager(
+    "maira2", *["image-text-to-text", "text-generation", "text-generation-with-past"], library_name="transformers"
+)
+class MairaOpenVINOConfig(LlavaOpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = version.parse("4.46.0")
+    SUPPORTS_PAST = True
+
+
 class InternVLChatConfigBehavior(str, enum.Enum):
     LANGUAGE = "language"
     VISION_EMBEDDINGS = "vision_embeddings"

optimum/exporters/openvino/utils.py

@@ -222,6 +222,7 @@ def get_submodels(model):
     "llava-next",
     "llava-qwen2",
     "internvl-chat",
+    "maira2",
     "minicpmv",
     "phi3-v",
     "qwen2-vl",

optimum/intel/openvino/modeling_visual_language.py

@@ -2331,11 +2331,31 @@ def preprocess_inputs(
         return inputs
 
 
+class _OVMaira2ForCausalLM(_OVLlavaForCausalLM):
+    @staticmethod
+    def preprocess_inputs(
+        text: str,
+        image: Optional["Image"] = None,
+        processor: Optional[AutoImageProcessor] = None,
+        tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
+    ):
+        if processor is None:
+            raise ValueError("processor is required")
+        processed_inputs = processor.format_and_preprocess_phrase_grounding_input(
+            frontal_image=image,
+            phrase=text,
+            return_tensors="pt",
+        )
+        return processed_inputs
+
+
 MODEL_TYPE_TO_CLS_MAPPING = {
     "llava": _OVLlavaForCausalLM,
     "llava_next": _OVLlavaNextForCausalLM,
     "minicpmv": _OVMiniCPMVForCausalLM,
     "llava-qwen2": _OVNanoLlavaForCausalLM,
+    "maira2": _OVMaira2ForCausalLM,
     "phi3_v": _OVPhi3VisionForCausalLM,
     "internvl_chat": _OVInternVLForCausalLM,
     "qwen2_vl": _OVQwen2VLForCausalLM,