OV Tokenizer Leftovers

- Support SD Pipeline Slow Tokenizer Conversion
- Support SD Mixed Quantization
- Move Converted OV Tokenizers to a Separate Folder

Author: apaniukov

optimum/commands/export/openvino.py

@@ -21,6 +21,7 @@
 from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
 
 from ...exporters import TasksManager
+from ...exporters.openvino.convert import export_tokenizer
 from ...intel.utils.import_utils import DIFFUSERS_IMPORT_ERROR, is_diffusers_available
 from ..base import BaseOptimumCLICommand, CommandInfo
 
@@ -261,6 +262,14 @@ def run(self):
             )
             model.save_pretrained(self.args.output)
 
+            output = Path(self.args.output)
+            tokenizer = getattr(model, "tokenizer", None)
+            if tokenizer is not None:
+                export_tokenizer(tokenizer, output / "tokenizer")
+
+            tokenizer_2 = getattr(model, "tokenizer_2", None)
+            if tokenizer_2 is not None:
+                export_tokenizer(tokenizer_2, output / "tokenizer_2")
         else:
             if self.args.convert_tokenizer:
                 logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")

optimum/exporters/openvino/__main__.py

@@ -25,6 +25,7 @@
 from optimum.exporters.onnx.base import OnnxConfig
 from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED
 from optimum.exporters.openvino.convert import export_from_model, export_tokenizer
+from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER
 from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
 from optimum.utils.save_utils import maybe_load_preprocessors
 
@@ -364,7 +365,7 @@ class StoreAttr(object):
 
             if tokenizer is not None:
                 try:
-                    export_tokenizer(tokenizer, output)
+                    export_tokenizer(tokenizer, output / OV_TOKENIZER_FLOLDER)
                 except Exception as exception:
                     logger.warning(
                         "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "
@@ -373,11 +374,11 @@ class StoreAttr(object):
         else:
             tokenizer = getattr(model, "tokenizer", None)
             if tokenizer is not None:
-                export_tokenizer(tokenizer, output)
+                export_tokenizer(tokenizer, output / "tokenizer")
 
             tokenizer_2 = getattr(model, "tokenizer_2", None)
             if tokenizer_2 is not None:
-                export_tokenizer(tokenizer_2, output, suffix="_2")
+                export_tokenizer(tokenizer_2, output / "tokenizer_2")
     elif convert_tokenizer and not is_openvino_tokenizers_available():
         logger.warning("Tokenizer won't be converted.")
 

optimum/exporters/openvino/convert.py

@@ -667,20 +667,21 @@ def export_tokenizer(
     output: Union[str, Path],
     suffix: Optional[str] = "",
 ):
-    from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME  # avoid circular imports
+    # avoid circular imports
+    from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME
+    from optimum.intel.openvino.utils import maybe_convert_tokenizer_to_fast
 
     try:
         from openvino_tokenizers import convert_tokenizer
     except ModuleNotFoundError:
-        # avoid this message before tokenizers are part of the openvino dependencies
-        # logger.info(
-        #     "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models."
-        # )
         return
 
     if not isinstance(output, Path):
         output = Path(output)
 
+    if output.exists():
+        tokenizer = maybe_convert_tokenizer_to_fast(tokenizer, output)
+
     try:
         converted = convert_tokenizer(tokenizer, with_detokenizer=True)
     except NotImplementedError:

optimum/intel/openvino/utils.py

@@ -17,10 +17,13 @@
 import logging
 import os
 from glob import glob
+from pathlib import Path
+from typing import List, Union
 
 import numpy as np
 from huggingface_hub import model_info
 from openvino.runtime import Core, Type, properties
+from transformers import AutoTokenizer, CLIPTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
 from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size
 
 
@@ -31,6 +34,7 @@
 OV_DECODER_NAME = "openvino_decoder_model.xml"
 OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml"
 
+OV_TOKENIZER_FLOLDER = "openvino_tokenizer"
 OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml"
 OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml"
 
@@ -107,6 +111,26 @@
 }
 
 
+NEED_CONVERT_TO_FAST_TOKENIZER: List[PreTrainedTokenizer] = [
+    CLIPTokenizer,
+]
+
+
+def maybe_convert_tokenizer_to_fast(
+    hf_tokenizer: PreTrainedTokenizer, tokenizer_path: Path
+) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
+    if isinstance(hf_tokenizer, PreTrainedTokenizerFast):
+        return hf_tokenizer
+
+    if any(isinstance(type(hf_tokenizer), slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER):
+        try:
+            return AutoTokenizer.from_pretrained(tokenizer_path)
+        except Exception:
+            return hf_tokenizer
+
+    return hf_tokenizer
+
+
 def use_external_data_format(num_parameters: int) -> bool:
     """
     Returns whether or not the model requires using external data format for the ONNX export

tests/openvino/test_exporters_cli.py

@@ -74,8 +74,8 @@ class OVCLIExportTestCase(unittest.TestCase):
         "wav2vec2": 0,  # no tokenizer
         "bert": 1,  # no detokenizer
         "blenderbot": 2,
-        "stable-diffusion": 0,  # not supported
-        "stable-diffusion-xl": 0,  # not supported
+        "stable-diffusion": 2,
+        "stable-diffusion-xl": 2,
     }
 
     SUPPORTED_SD_HYBRID_ARCHITECTURES = (