Set openvino slow test (#774)

* fix test

* set slow test

* add slow

* register custom marks

Author: echarlaix

.github/workflows/test_openvino_basic.yml

@@ -41,7 +41,7 @@ jobs:
         # optimum or transformers to a specific version
         # Install PyTorch CPU to prevent unnecessary downloading/installing of CUDA packages
         pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
-        pip install .[tests] openvino onnxruntime ${{ matrix.optimum}}
+        pip install .[tests] openvino ${{ matrix.optimum}}
 
     - name: Pip freeze
       run: pip freeze
@@ -52,6 +52,7 @@ jobs:
 
     - name: Slow tests
       run: |
-        pytest tests/openvino/test_modeling.py -s -m "run_slow" --durations=0
+        pip install nncf
+        pytest tests/openvino -s -m "run_slow" --durations=0
       env:
         RUN_SLOW: 1

pyproject.toml

@@ -29,3 +29,8 @@ line-length = 119
 [tool.ruff.isort]
 lines-after-imports = 2
 known-first-party = ["optimum"]
+
+[tool.pytest.ini_options]
+markers = [
+    "run_slow",
+]

tests/openvino/test_modeling.py

@@ -990,14 +990,21 @@ def test_beam_search(self, model_arch):
         ov_model_stateless.config.eos_token_id = None
         transformers_model.config.eos_token_id = None
 
-        for idx, gen_config in enumerate(gen_configs):
+        for gen_config in gen_configs:
             if gen_config.do_sample and model_arch in ["baichuan2-13b", "olmo"]:
                 continue
+
             transformers_outputs = transformers_model.generate(**tokens, generation_config=gen_config)
             ov_stateful_outputs = ov_model_stateful.generate(**tokens, generation_config=gen_config)
-            self.assertTrue(torch.allclose(ov_stateful_outputs, transformers_outputs), f"generation config : {idx}")
+            self.assertTrue(
+                torch.equal(ov_stateful_outputs, transformers_outputs),
+                f"generation config : {gen_config}, transformers output {transformers_outputs}, ov_model_stateful output {ov_stateful_outputs}",
+            )
             ov_stateless_outputs = ov_model_stateless.generate(**tokens, generation_config=gen_config)
-            self.assertTrue(torch.allclose(ov_stateless_outputs, transformers_outputs), f"generation config : {idx}")
+            self.assertTrue(
+                torch.equal(ov_stateless_outputs, transformers_outputs),
+                f"generation config : {gen_config}, transformers output {transformers_outputs}, ov_model_stateless output {ov_stateless_outputs}",
+            )
 
 
 class OVModelForMaskedLMIntegrationTest(unittest.TestCase):

tests/openvino/test_quantization.py

@@ -22,7 +22,7 @@
 from enum import Enum
 from functools import partial
 from typing import Union
-
+import pytest
 import evaluate
 import numpy as np
 import torch
@@ -37,6 +37,7 @@
     TrainingArguments,
     default_data_collator,
 )
+from transformers.testing_utils import slow
 from transformers.utils.quantization_config import QuantizationMethod
 
 from optimum.intel import (
@@ -173,15 +174,13 @@ def preprocess_function(examples, tokenizer):
 
 
 class OVWeightCompressionTest(unittest.TestCase):
-    # TODO : add models
     SUPPORTED_ARCHITECTURES_WITH_EXPECTED_8BIT_COMPRESSED_MATMULS = (
         (OVModelForSequenceClassification, "bert", 70, 70),
         (OVModelForCausalLM, "gpt2", 44, 44),
     )
 
     SUPPORTED_ARCHITECTURES_WITH_EXPECTED_4BIT_COMPRESSED_MATMULS = ((OVModelForCausalLM, "opt125m", 62, 86),)
     SUPPORTED_ARCHITECTURES_WITH_EXPECTED_4BIT_AUTOCOMPRESSED_MATMULS = ((OVModelForCausalLM, "opt125m", 0, 148),)
-
     SUPPORTED_ARCHITECTURES_STATEFUL_WITH_EXPECTED_8BIT_COMPRESSED_MATMULS = ((OVModelForCausalLM, "gpt2", 44, 44),)
 
     LOAD_IN_4_BITS_SCOPE = (
@@ -347,7 +346,6 @@ def test_ovmodel_4bit_weight_compression(self, model_cls, model_name, expected_i
             self.assertEqual(ov_config.quantization_config.to_dict(), loaded_config.quantization_config.to_dict())
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES_STATEFUL_WITH_EXPECTED_8BIT_COMPRESSED_MATMULS)
-    @unittest.skipIf(not IS_SUPPORT_STATEFUL, "Stateful models supported only in 2023.3 and above")
     def test_ovmodel_8bit_weight_compression_stateful(self, model_cls, model_name, expected_pt_int8, expected_ov_int8):
         task = model_cls.export_feature
         model_id = MODEL_NAMES[model_name]
@@ -473,7 +471,6 @@ def test_ovmodel_4bit_auto_compression_with_config(
             self.assertEqual(openvino_config.dtype, "int4")
 
     @parameterized.expand(((OVModelForCausalLM, "gpt2"),))
-    @unittest.skipIf(not IS_SUPPORT_STATEFUL, "Stateful models supported only in 2023.3 and above")
     def test_ovmodel_stateful_load_with_compressed_weights(self, model_cls, model_type):
         model = model_cls.from_pretrained(MODEL_NAMES[model_type], export=True, load_in_8bit=True, stateful=True)
         self.assertTrue(model.stateful)
@@ -588,9 +585,11 @@ def test_ovmodel_4bit_dynamic_with_config(self, model_cls, model_name, quantizat
 
 
 class OVQuantizerQATest(unittest.TestCase):
-    SUPPORTED_ARCHITECTURES = (("hf-internal-testing/tiny-random-BertForQuestionAnswering",),)
+    SUPPORTED_ARCHITECTURES = ("hf-internal-testing/tiny-random-BertForQuestionAnswering",)
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_automodel_static_quantization(self, model_name):
         def preprocess_function(examples, tokenizer):
             return tokenizer(
@@ -630,6 +629,8 @@ def preprocess_function(examples, tokenizer):
             self.assertEqual(ov_config.quantization_config.to_dict(), loaded_config.quantization_config.to_dict())
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_ovmodel_static_quantization(self, model_name):
         def preprocess_function(examples, tokenizer):
             return tokenizer(
@@ -670,12 +671,13 @@ def preprocess_function(examples, tokenizer):
 
 
 class OVTrainerTest(unittest.TestCase):
-    SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = (("distilbert-base-uncased", 67, 38),)
+    SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS = (("albert", 65, 39),)
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES_WITH_EXPECTED_QUANTIZED_MATMULS)
     def test_aware_training_quantization(self, model_name, expected_fake_quantize, expected_int8):
-        model = AutoModelForSequenceClassification.from_pretrained(model_name)
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model_id = MODEL_NAMES[model_name]
+        model = AutoModelForSequenceClassification.from_pretrained(model_id)
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
         ov_config = OVConfig()
         dataset = load_dataset("glue", "sst2")
         dataset = dataset.map(

tests/openvino/test_stable_diffusion.py

@@ -19,6 +19,7 @@
 
 import numpy as np
 import PIL
+import pytest
 import torch
 from diffusers import (
     StableDiffusionPipeline,
@@ -29,6 +30,7 @@
 from diffusers.utils.testing_utils import floats_tensor
 from openvino.runtime.ie_api import CompiledModel
 from parameterized import parameterized
+from transformers.testing_utils import slow
 from utils_tests import MODEL_NAMES, SEED
 
 from optimum.intel import (
@@ -106,6 +108,8 @@ def test_num_images_per_prompt(self, model_arch: str):
                 self.assertEqual(outputs.shape, (batch_size * num_images, height, width, 3))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_callback(self, model_arch: str):
         MODEL_NAMES[model_arch]
 
@@ -178,6 +182,8 @@ def test_compare_diffusers_pipeline(self, model_arch: str):
         self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-1))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]
         pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False)
@@ -260,6 +266,8 @@ def test_image_reproducibility(self, model_arch: str):
         self.assertFalse(np.array_equal(ov_outputs_1.images[0], ov_outputs_3.images[0]))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]
         pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False)
@@ -332,6 +340,8 @@ def test_compare_diffusers_pipeline(self, model_arch: str):
         self.assertTrue(np.allclose(outputs[0, -3:, -3:, -1].flatten(), expected_slice, atol=1e-1))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]
         pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False)
@@ -420,6 +430,8 @@ def test_image_reproducibility(self, model_arch: str):
         self.assertFalse(np.array_equal(ov_outputs_1.images[0], ov_outputs_3.images[0]))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]
         pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False)
@@ -458,6 +470,8 @@ def test_inference(self):
         self.assertTrue(np.allclose(output.flatten(), expected_slice, atol=1e-3))
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]
         pipeline = self.MODEL_CLASS.from_pretrained(model_id, export=True, compile=False, dynamic_shapes=False)
@@ -525,6 +539,8 @@ def test_compare_to_diffusers(self, model_arch: str):
         self.assertEqual(pipeline.device.type, ov_pipeline.device)
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    @pytest.mark.run_slow
+    @slow
     @unittest.skipIf(is_diffusers_version("<=", "0.21.4"), "not supported with this diffusers version")
     def test_num_images_per_prompt_static_model(self, model_arch: str):
         model_id = MODEL_NAMES[model_arch]

tests/openvino/test_training.py

@@ -28,6 +28,7 @@
 import cpuinfo
 import evaluate
 import numpy as np
+import pytest
 import torch
 from datasets import load_dataset
 from nncf.experimental.torch.sparsity.movement.algo import MovementSparsityController
@@ -41,6 +42,7 @@
     AutoTokenizer,
     default_data_collator,
 )
+from transformers.testing_utils import slow
 from transformers.trainer_utils import EvalPrediction, TrainOutput
 from transformers.utils import WEIGHTS_NAME
 
@@ -613,6 +615,8 @@ class OVTrainerImageClassificationTrainingTest(OVTrainerBaseTrainingTest):
     task = "image-classification"
 
     @parameterized.expand(OVTRAINER_IMAGE_CLASSIFICATION_TEST_DESCRIPTORS.items())
+    @pytest.mark.run_slow
+    @slow
     @unittest.skipIf(is_transformers_version("<", "4.41.0"), reason="Mismatch in expected fake quantized op")
     def test_training(self, _, desc: OVTrainerTestDescriptor):
         self.run_ovtrainer_training_checks(desc)
@@ -791,6 +795,8 @@ class OVTrainerAudioClassificationTrainingTest(OVTrainerBaseTrainingTest):
     task = "audio-classification"
 
     @parameterized.expand(OVTRAINER_AUDIO_CLASSIFICATION_TEST_DESCRIPTORS.items())
+    @pytest.mark.run_slow
+    @slow
     def test_training(self, _, desc: OVTrainerTestDescriptor):
         self.run_ovtrainer_training_checks(desc)