Add SD quant API test

Author: l-bat

optimum/intel/openvino/quantization.py

@@ -460,6 +460,9 @@ def _quantize_ovbasemodel(
             if calibration_dataset is None:
                 raise ValueError("Calibration dataset is required to run quantization.")
 
+            if is_diffusers_available() and isinstance(self.model, OVDiffusionPipeline):
+                raise NotImplementedError(f"Mixed precision quantization isn't supported for diffusers.")
+
             quantized_model = _mixed_quantization(self.model.model, quantization_config, calibration_dataset, **kwargs)
             self.model.model = quantized_model
             self.model.request = None

tests/openvino/test_exporters_cli.py

@@ -269,10 +269,10 @@ class OVCLIExportTestCase(unittest.TestCase):
             "int8",
             "--dataset conceptual_captions --num-samples 1 --trust-remote-code",
             [
-                0, 0, 0, 112
+                112, 0, 0, 0,
             ],
             [
-                {"int8": 34}, {"int8": 42}, {"int8": 64}, {"int8": 121},
+                {"int8": 121}, {"int8": 42}, {"int8": 34}, {"int8": 64},
             ],
         ),
         (
@@ -281,10 +281,10 @@ class OVCLIExportTestCase(unittest.TestCase):
             "f8e5m2",
             "--dataset laion/220k-GPT4Vision-captions-from-LIVIS --num-samples 1 --trust-remote-code",
             [
-                0, 0, 0, 174
+                174, 0, 0, 0, 0,
             ],
             [
-                {"int8": 34}, {"int8": 42}, {"int8": 64}, {"f8e5m2": 183},
+                {"f8e5m2": 183}, {"int8": 42}, {"int8": 34}, {"int8": 64}, {"int8": 66},
             ],
         ),
         (
@@ -293,10 +293,10 @@ class OVCLIExportTestCase(unittest.TestCase):
             "f8e4m3",
             "--dataset laion/filtered-wit --num-samples 1 --trust-remote-code",
             [
-                0, 0, 0, 79
+                79, 0, 0, 0,
             ],
             [
-                {"int8": 34}, {"int8": 42}, {"int8": 40}, {"f8e4m3": 84},
+                {"f8e4m3": 84}, {"int8": 42}, {"int8": 34}, {"int8": 40},
             ],
         ),
     ]
@@ -733,7 +733,7 @@ def test_exporters_cli_full_quantization(
             elif "text-generation" in task:
                 submodels = [model]
             elif any(x in task for x in ("stable-diffusion", "latent-consistency")):
-                submodels = [model.vae_encoder, model.vae_decoder, model.text_encoder, model.unet]
+                submodels = model.ov_submodels.values()
             else:
                 raise Exception("Unexpected task.")
 

tests/openvino/test_quantization.py

@@ -19,26 +19,22 @@
 import itertools
 import logging
 import unittest
-from collections import defaultdict, Iterable
+from collections import defaultdict
+from collections.abc import Iterable
 from enum import Enum
 from functools import partial
 from typing import Union, Type
 
 import openvino as ov
 import pytest
-import evaluate
 import numpy as np
 import torch
-from datasets import load_dataset
 from parameterized import parameterized
 import nncf
 from transformers import (
     AutoModelForQuestionAnswering,
-    AutoModelForSequenceClassification,
     AutoTokenizer,
     AutoProcessor,
-    TrainingArguments,
-    default_data_collator,
 )
 from transformers.testing_utils import slow
 from transformers.utils.quantization_config import QuantizationMethod
@@ -234,6 +230,57 @@ class OVQuantizerTest(unittest.TestCase):
                 {"f8e5m2": 2, "int4": 28},
             ],
         ),
+        (
+            OVStableDiffusionPipeline,
+            "stable-diffusion",
+            dict(
+                weight_only=False,
+                dataset="conceptual_captions",
+                num_samples=1,
+                processor=MODEL_NAMES["stable-diffusion"],
+                trust_remote_code=True,
+            ),
+            [
+                112, 0, 0, 0,
+            ],
+            [
+                {"int8": 121}, {"int8": 42}, {"int8": 34}, {"int8": 64},
+            ],
+        ),
+        (
+            OVStableDiffusionXLPipeline,
+            "stable-diffusion-xl",
+            dict(
+                weight_only=False,
+                dtype="f8e5m2",
+                dataset="laion/220k-GPT4Vision-captions-from-LIVIS",
+                num_samples=1,
+                processor=MODEL_NAMES["stable-diffusion-xl"],
+                trust_remote_code=True,
+            ),
+            [
+                174, 0, 0, 0, 0,
+            ],
+            [
+                {"f8e5m2": 183}, {"int8": 42}, {"int8": 34}, {"int8": 64}, {"int8": 66},
+            ],
+        ),
+        (
+            OVLatentConsistencyModelPipeline,
+            "latent-consistency",
+            OVQuantizationConfig(
+                dtype="f8e4m3",
+                dataset="laion/filtered-wit",
+                num_samples=1,
+                trust_remote_code=True,
+            ),
+            [
+                79, 0, 0, 0,
+            ],
+            [
+                {"f8e4m3": 84}, {"int8": 42}, {"int8": 34}, {"int8": 40},
+            ],
+        ),
     ]
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES_TORCH_MODEL)
@@ -359,6 +406,8 @@ def test_ov_model_static_quantization_with_auto_dataset(
                 tokens = tokenizer("This is a sample input", return_tensors="pt")
                 outputs = ov_model(**tokens)
                 self.assertTrue("logits" in outputs)
+            elif any(x == model_cls for x in (OVStableDiffusionPipeline, OVStableDiffusionXLPipeline, OVLatentConsistencyModelPipeline)):
+                submodels = ov_model.ov_submodels.values()
             else:
                 raise Exception("Unexpected model class.")