test for video

Author: eaidova

optimum/intel/openvino/modeling_visual_language.py

@@ -6,7 +6,7 @@
 from dataclasses import dataclass
 from pathlib import Path
 from types import MethodType
-from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 import numpy as np
 import openvino as ov
@@ -55,7 +55,11 @@
 
 if TYPE_CHECKING:
     from PIL.Image import Image
-    from transformers.image_utils import VideoInput
+
+    if is_transformers_version(">=", "4.42.0"):
+        from transformers.image_utils import VideoInput
+    else:
+        VideoInput = List[Image]
 
 
 logger = logging.getLogger(__name__)

tests/openvino/test_modeling.py

@@ -31,7 +31,7 @@
 import torch
 from datasets import load_dataset
 from evaluate import evaluator
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, hf_hub_download
 from parameterized import parameterized
 from PIL import Image
 from sentence_transformers import SentenceTransformer
@@ -2126,21 +2126,25 @@ def test_compare_with_and_without_past_key_values(self):
 
 class OVModelForVisualCausalLMIntegrationTest(unittest.TestCase):
     SUPPORTED_ARCHITECTURES = ["llava"]
+    SUPPORT_VIDEO = []
 
-    # if is_transformers_version(">=", "4.40.0"):
-    #     SUPPORTED_ARCHITECTURES += ["llava_next", "nanollava"]
+    if is_transformers_version(">=", "4.40.0"):
+        SUPPORTED_ARCHITECTURES += ["llava_next", "nanollava"]
 
     if is_transformers_version(">=", "4.42.0"):
         SUPPORTED_ARCHITECTURES += ["llava_next_video"]
+        SUPPORT_VIDEO.append("llava_next_video")
 
-    # if is_transformers_version(">=", "4.45.0"):
-    #     SUPPORTED_ARCHITECTURES += ["minicpmv", "internvl2", "phi3_v", "qwen2_vl"]
+    if is_transformers_version(">=", "4.45.0"):
+        SUPPORTED_ARCHITECTURES += ["minicpmv", "internvl2", "phi3_v", "qwen2_vl"]
+        SUPPORT_VIDEO.append("qwen2_vl")
 
-    # if is_transformers_version(">=", "4.46.0"):
-    #     SUPPORTED_ARCHITECTURES += ["maira2"]
+    if is_transformers_version(">=", "4.46.0"):
+        SUPPORTED_ARCHITECTURES += ["maira2"]
 
-    # if is_transformers_version(">=", "4.49.0"):
-    #     SUPPORTED_ARCHITECTURES += ["qwen2_5_vl"]
+    if is_transformers_version(">=", "4.49.0"):
+        SUPPORTED_ARCHITECTURES += ["qwen2_5_vl"]
+        SUPPORT_VIDEO.append("qwen2_5_vl")
     TASK = "image-text-to-text"
     REMOTE_CODE_MODELS = ["internvl2", "minicpmv", "nanollava", "phi3_v", "maira2"]
 
@@ -2350,6 +2354,22 @@ def test_generate_utils(self, model_arch):
         outputs = outputs[:, inputs["input_ids"].shape[1] :]
         outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)
         self.assertIsInstance(outputs[0], str)
+
+        # video laoder helper only available for transformers >= 4.49
+        if model_arch in self.SUPPORT_VIDEO and is_transformers_version(">=", "4.49"):
+            from transformers.image_utils import load_video
+
+            video_path = hf_hub_download(
+                repo_id="raushan-testing-hf/videos-test", filename="sample_demo_1.mp4", repo_type="dataset"
+            )
+            input_video = load_video(video_path, num_frames=4)
+            question = "Why is this video funny?"
+            inputs = model.preprocess_inputs(**preprocessors, text=question, video=input_video)
+            outputs = model.generate(**inputs, max_new_tokens=10)
+            # filter out original prompt becuase it may contains out of tokenizer tokens e.g. in nanollva text separator = -200
+            outputs = outputs[:, inputs["input_ids"].shape[1] :]
+            outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+            self.assertIsInstance(outputs[0], str)
         del model
 
         gc.collect()