Merge branch 'main' into ea/deepseek

Author: eaidova

.github/workflows/test_openvino_slow.yml

@@ -26,6 +26,11 @@ jobs:
       matrix:
         os: ["ubuntu-22.04", "windows-2019"]
         transformers-version: ["4.36.0", "latest"]
+        include:
+          - transformers-version: "4.40.0"
+            os: "ubuntu-22.04"
+          - transformers-version: "4.45.0"
+            os: "ubuntu-22.04"
 
     runs-on: ${{ matrix.os }}
 
@@ -65,10 +70,6 @@ jobs:
         run: |
           pip install .[nncf]
 
-      - if: ${{ matrix.transformers-version != 'latest' }}
-        name: Downgrade Transformers and Accelerate
-        run: pip install transformers==${{ matrix.transformers-version }} accelerate==0.*
-
       - name: Test with Pytest (slow)
         run: |
           pytest tests/openvino -m "run_slow" --durations=0

docs/source/openvino/models.mdx

@@ -75,6 +75,7 @@ Here is the list of the supported architectures :
 - Llava
 - Llava-Next
 - M2-M100
+- MAIRA-2
 - MBart
 - MPNet
 - MPT

optimum/exporters/openvino/model_configs.py

@@ -1489,6 +1489,7 @@ def __init__(
         float_dtype: str = "fp32",
         behavior: LlavaConfigBehavior = LlavaConfigBehavior.VISION_EMBEDDINGS,
         preprocessors: Optional[List[Any]] = None,
+        **kwargs,
     ):
         super().__init__(
             config=config,
@@ -1585,6 +1586,14 @@ class LlavaNextOpenVINOConfig(LlavaOpenVINOConfig):
     MIN_TRANSFORMERS_VERSION = version.parse("4.40.0")
 
 
+@register_in_tasks_manager(
+    "maira2", *["image-text-to-text", "text-generation", "text-generation-with-past"], library_name="transformers"
+)
+class MairaOpenVINOConfig(LlavaOpenVINOConfig):
+    MIN_TRANSFORMERS_VERSION = version.parse("4.46.0")
+    SUPPORTS_PAST = True
+
+
 class InternVLChatConfigBehavior(str, enum.Enum):
     LANGUAGE = "language"
     VISION_EMBEDDINGS = "vision_embeddings"

optimum/exporters/openvino/utils.py

@@ -223,6 +223,7 @@ def get_submodels(model):
     "llava-next",
     "llava-qwen2",
     "internvl-chat",
+    "maira2",
     "minicpmv",
     "phi3-v",
     "qwen2-vl",

optimum/intel/openvino/modeling_diffusion.py

@@ -294,8 +294,12 @@ def _save_pretrained(self, save_directory: Union[str, Path]):
                 dst_path = save_path / OV_XML_FILE_NAME
                 dst_path.parent.mkdir(parents=True, exist_ok=True)
                 openvino.save_model(model.model, dst_path, compress_to_fp16=False)
-                model_dir = model.config.get("_name_or_path", None) or model.model_save_dir
-                config_path = Path(model_dir) / CONFIG_NAME
+                model_dir = (
+                    self.model_save_dir
+                    if not isinstance(self.model_save_dir, TemporaryDirectory)
+                    else self.model_save_dir.name
+                )
+                config_path = Path(model_dir) / save_path.name / CONFIG_NAME
                 if config_path.is_file():
                     config_save_path = save_path / CONFIG_NAME
                     shutil.copyfile(config_path, config_save_path)

optimum/intel/openvino/modeling_visual_language.py

@@ -2331,11 +2331,33 @@ def preprocess_inputs(
         return inputs
 
 
+class _OVMaira2ForCausalLM(_OVLlavaForCausalLM):
+    @staticmethod
+    def preprocess_inputs(
+        text: str,
+        image: Optional["Image"] = None,
+        processor: Optional[AutoImageProcessor] = None,
+        tokenizer: Optional[PreTrainedTokenizer] = None,
+        config: Optional[PretrainedConfig] = None,
+    ):
+        if processor is None:
+            raise ValueError("processor is required")
+        if image is None:
+            return processor(text=text, return_tensors="pt")
+        processed_inputs = processor.format_and_preprocess_phrase_grounding_input(
+            frontal_image=image,
+            phrase=text,
+            return_tensors="pt",
+        )
+        return processed_inputs
+
+
 MODEL_TYPE_TO_CLS_MAPPING = {
     "llava": _OVLlavaForCausalLM,
     "llava_next": _OVLlavaNextForCausalLM,
     "minicpmv": _OVMiniCPMVForCausalLM,
     "llava-qwen2": _OVNanoLlavaForCausalLM,
+    "maira2": _OVMaira2ForCausalLM,
     "phi3_v": _OVPhi3VisionForCausalLM,
     "internvl_chat": _OVInternVLForCausalLM,
     "qwen2_vl": _OVQwen2VLForCausalLM,

tests/openvino/test_diffusion.py

@@ -359,6 +359,13 @@ def test_load_and_save_pipeline_with_safety_checker(self):
                     self.assertTrue(model_lib in ["diffusers", "transformers"])
                     self.assertFalse(model_class.startswith("OV"))
             loaded_pipeline = self.OVMODEL_CLASS.from_pretrained(tmpdirname)
+            for component in ["text_encoder", "unet", "vae_encoder", "vae_decoder"]:
+                config = getattr(getattr(ov_pipeline, component), "config", None)
+                if config is not None:
+                    loaded_config = getattr(getattr(loaded_pipeline, component), "config")
+                    self.assertDictEqual(
+                        config, loaded_config, f"Expected config:\n{config}\nLoaded config:|n{loaded_config}"
+                    )
             self.assertTrue(loaded_pipeline.safety_checker is not None)
             self.assertIsInstance(loaded_pipeline.safety_checker, StableDiffusionSafetyChecker)
             del loaded_pipeline

tests/openvino/test_modeling.py

@@ -980,24 +980,27 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
     if is_transformers_version(">=", "4.40.0"):
         SUPPORTED_ARCHITECTURES += (
             "gemma",
-            "gemma2",
             "olmo",
             "stablelm",
             "starcoder2",
             "dbrx",
-            "phi3",
             "cohere",
             "qwen2",
             "qwen2-moe",
             "arctic",
-            "exaone",
-            "mistral-nemo",
-            "minicpm3",
-            "glm",
-            "granite",
-            "granite-moe",
         )
 
+    if is_transformers_version(">=", "4.41.0"):
+        SUPPORTED_ARCHITECTURES += ("phi3",)
+
+    if is_transformers_version(">=", "4.43.0"):
+        SUPPORTED_ARCHITECTURES += ("gemma2", "exaone")
+
+    if is_transformers_version(">=", "4.44.0"):
+        SUPPORTED_ARCHITECTURES += ("granite", "granite-moe")
+
+    if is_transformers_version(">=", "4.46.0"):
+        SUPPORTED_ARCHITECTURES += ("glm", "mistral-nemo", "minicpm3")
         # openvino 2025.0 required for disabling check_trace
         if is_openvino_version(">=", "2025.0"):
             SUPPORTED_ARCHITECTURES += ("deepseek",)
@@ -2115,9 +2118,9 @@ class OVModelForVisualCausalLMIntegrationTest(unittest.TestCase):
     if is_transformers_version(">=", "4.40.0"):
         SUPPORTED_ARCHITECTURES += ["llava_next", "nanollava"]
     if is_transformers_version(">=", "4.45.0"):
-        SUPPORTED_ARCHITECTURES += ["minicpmv", "internvl2", "phi3_v", "qwen2_vl"]
+        SUPPORTED_ARCHITECTURES += ["minicpmv", "internvl2", "phi3_v", "qwen2_vl", "maira2"]
     TASK = "image-text-to-text"
-    REMOTE_CODE_MODELS = ["internvl2", "minicpmv", "nanollava", "phi3_v"]
+    REMOTE_CODE_MODELS = ["internvl2", "minicpmv", "nanollava", "phi3_v", "maira2"]
 
     IMAGE = Image.open(
         requests.get(
@@ -2197,7 +2200,7 @@ def test_compare_to_transformers(self, model_arch):
             with torch.no_grad():
                 transformers_outputs = transformers_model(**transformers_inputs)
             self.assertTrue(
-                torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=1e-4),
+                torch.allclose(ov_outputs.logits, transformers_outputs.logits, atol=4e-3),
                 f"Max abs diff {(torch.abs(ov_outputs.logits - transformers_outputs.logits).max())}",
             )
 

tests/openvino/utils_tests.py

@@ -92,6 +92,7 @@
     "opt": "hf-internal-testing/tiny-random-OPTModel",
     "opt125m": "facebook/opt-125m",
     "opt_gptq": "ybelkada/opt-125m-gptq-4bit",
+    "maira2": "katuni4ka/tiny-random-maira2",
     "marian": "sshleifer/tiny-marian-en-de",
     "mbart": "hf-internal-testing/tiny-random-mbart",
     "minicpm": "katuni4ka/tiny-random-minicpm",