From d02e281f2f94c791890f99277d0926e2ae7810e0 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 14:23:57 +0100 Subject: [PATCH 1/9] OV Tokenizer Leftovers - Support SD Pipeline Slow Tokenizer Conversion - Support SD Mixed Quantization - Move Converted OV Tokenizers to a Separate Folder --- optimum/commands/export/openvino.py | 9 +++++++++ optimum/exporters/openvino/__main__.py | 7 ++++--- optimum/exporters/openvino/convert.py | 11 ++++++----- optimum/intel/openvino/utils.py | 24 ++++++++++++++++++++++++ tests/openvino/test_exporters_cli.py | 4 ++-- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index 4fed3f6f88..c225c50d71 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -21,6 +21,7 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from ...exporters import TasksManager +from ...exporters.openvino.convert import export_tokenizer from ...intel.utils.import_utils import DIFFUSERS_IMPORT_ERROR, is_diffusers_available from ..base import BaseOptimumCLICommand, CommandInfo @@ -261,6 +262,14 @@ def run(self): ) model.save_pretrained(self.args.output) + output = Path(self.args.output) + tokenizer = getattr(model, "tokenizer", None) + if tokenizer is not None: + export_tokenizer(tokenizer, output / "tokenizer") + + tokenizer_2 = getattr(model, "tokenizer_2", None) + if tokenizer_2 is not None: + export_tokenizer(tokenizer_2, output / "tokenizer_2") else: if self.args.convert_tokenizer: logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.") diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 8908c430b3..3fa4fb0eb1 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -25,6 +25,7 @@ from optimum.exporters.onnx.base import OnnxConfig from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED from optimum.exporters.openvino.convert import export_from_model, export_tokenizer +from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version from optimum.utils.save_utils import maybe_load_preprocessors @@ -364,7 +365,7 @@ class StoreAttr(object): if tokenizer is not None: try: - export_tokenizer(tokenizer, output) + export_tokenizer(tokenizer, output / OV_TOKENIZER_FLOLDER) except Exception as exception: logger.warning( "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " @@ -373,11 +374,11 @@ class StoreAttr(object): else: tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: - export_tokenizer(tokenizer, output) + export_tokenizer(tokenizer, output / "tokenizer") tokenizer_2 = getattr(model, "tokenizer_2", None) if tokenizer_2 is not None: - export_tokenizer(tokenizer_2, output, suffix="_2") + export_tokenizer(tokenizer_2, output / "tokenizer_2") elif convert_tokenizer and not is_openvino_tokenizers_available(): logger.warning("Tokenizer won't be converted.") diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 6c86c2c2df..bb781a6904 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -667,20 +667,21 @@ def export_tokenizer( output: Union[str, Path], suffix: Optional[str] = "", ): - from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME # avoid circular imports + # avoid circular imports + from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME + from optimum.intel.openvino.utils import maybe_convert_tokenizer_to_fast try: from openvino_tokenizers import convert_tokenizer except ModuleNotFoundError: - # avoid this message before tokenizers are part of the openvino dependencies - # logger.info( - # "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models." - # ) return if not isinstance(output, Path): output = Path(output) + if output.exists(): + tokenizer = maybe_convert_tokenizer_to_fast(tokenizer, output) + try: converted = convert_tokenizer(tokenizer, with_detokenizer=True) except NotImplementedError: diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 4d1479f733..89994a7ac8 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -17,10 +17,13 @@ import logging import os from glob import glob +from pathlib import Path +from typing import List, Union import numpy as np from huggingface_hub import model_info from openvino.runtime import Core, Type, properties +from transformers import AutoTokenizer, CLIPTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size @@ -31,6 +34,7 @@ OV_DECODER_NAME = "openvino_decoder_model.xml" OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml" +OV_TOKENIZER_FLOLDER = "openvino_tokenizer" OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml" OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml" @@ -107,6 +111,26 @@ } +NEED_CONVERT_TO_FAST_TOKENIZER: List[PreTrainedTokenizer] = [ + CLIPTokenizer, +] + + +def maybe_convert_tokenizer_to_fast( + hf_tokenizer: PreTrainedTokenizer, tokenizer_path: Path +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + if isinstance(hf_tokenizer, PreTrainedTokenizerFast): + return hf_tokenizer + + if any(isinstance(type(hf_tokenizer), slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER): + try: + return AutoTokenizer.from_pretrained(tokenizer_path) + except Exception: + return hf_tokenizer + + return hf_tokenizer + + def use_external_data_format(num_parameters: int) -> bool: """ Returns whether or not the model requires using external data format for the ONNX export diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 09fad5d773..c91f28ba5c 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -74,8 +74,8 @@ class OVCLIExportTestCase(unittest.TestCase): "wav2vec2": 0, # no tokenizer "bert": 1, # no detokenizer "blenderbot": 2, - "stable-diffusion": 0, # not supported - "stable-diffusion-xl": 0, # not supported + "stable-diffusion": 2, + "stable-diffusion-xl": 2, } SUPPORTED_SD_HYBRID_ARCHITECTURES = ( From 135c2e9b8f96e54b95baa7c626fc4be3fb0cdc08 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 15:01:49 +0100 Subject: [PATCH 2/9] Fix Circular Import --- optimum/commands/export/openvino.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index c225c50d71..5a6cfeb029 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -21,7 +21,6 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from ...exporters import TasksManager -from ...exporters.openvino.convert import export_tokenizer from ...intel.utils.import_utils import DIFFUSERS_IMPORT_ERROR, is_diffusers_available from ..base import BaseOptimumCLICommand, CommandInfo @@ -262,6 +261,9 @@ def run(self): ) model.save_pretrained(self.args.output) + # avoid circular import + from ...exporters.openvino.convert import export_tokenizer + output = Path(self.args.output) tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: From 1766570e78ca7a3f38e1d8d47326c6fb70e7ba7c Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 15:12:20 +0100 Subject: [PATCH 3/9] Fix Circular Import --- optimum/commands/export/openvino.py | 4 +--- optimum/exporters/openvino/__main__.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index 5a6cfeb029..c225c50d71 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -21,6 +21,7 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from ...exporters import TasksManager +from ...exporters.openvino.convert import export_tokenizer from ...intel.utils.import_utils import DIFFUSERS_IMPORT_ERROR, is_diffusers_available from ..base import BaseOptimumCLICommand, CommandInfo @@ -261,9 +262,6 @@ def run(self): ) model.save_pretrained(self.args.output) - # avoid circular import - from ...exporters.openvino.convert import export_tokenizer - output = Path(self.args.output) tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 3fa4fb0eb1..41eb0200d5 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -25,7 +25,6 @@ from optimum.exporters.onnx.base import OnnxConfig from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED from optimum.exporters.openvino.convert import export_from_model, export_tokenizer -from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version from optimum.utils.save_utils import maybe_load_preprocessors @@ -356,6 +355,9 @@ class StoreAttr(object): **kwargs_shapes, ) + # avoid circular import + from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER + if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": tokenizer = next( From a1ee74970357e5c8ab2164d0bd381993cc035f35 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 17:07:30 +0100 Subject: [PATCH 4/9] Fix Tests --- optimum/intel/openvino/utils.py | 4 ++-- tests/openvino/test_exporters_cli.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 89994a7ac8..6b49f7a833 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -111,7 +111,7 @@ } -NEED_CONVERT_TO_FAST_TOKENIZER: List[PreTrainedTokenizer] = [ +NEED_CONVERT_TO_FAST_TOKENIZER: List[type(PreTrainedTokenizer)] = [ CLIPTokenizer, ] @@ -122,7 +122,7 @@ def maybe_convert_tokenizer_to_fast( if isinstance(hf_tokenizer, PreTrainedTokenizerFast): return hf_tokenizer - if any(isinstance(type(hf_tokenizer), slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER): + if any(isinstance(hf_tokenizer, slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER): try: return AutoTokenizer.from_pretrained(tokenizer_path) except Exception: diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index c91f28ba5c..cac79abaee 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -75,7 +75,7 @@ class OVCLIExportTestCase(unittest.TestCase): "bert": 1, # no detokenizer "blenderbot": 2, "stable-diffusion": 2, - "stable-diffusion-xl": 2, + "stable-diffusion-xl": 4, } SUPPORTED_SD_HYBRID_ARCHITECTURES = ( From ef9e5df7d23596a1b95a02698ab58d5e018634c0 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 18:40:37 +0100 Subject: [PATCH 5/9] Fix INC Tests --- optimum/commands/export/openvino.py | 4 +++- optimum/exporters/openvino/__main__.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index c225c50d71..a7302ef88f 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -21,7 +21,6 @@ from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE from ...exporters import TasksManager -from ...exporters.openvino.convert import export_tokenizer from ...intel.utils.import_utils import DIFFUSERS_IMPORT_ERROR, is_diffusers_available from ..base import BaseOptimumCLICommand, CommandInfo @@ -262,6 +261,9 @@ def run(self): ) model.save_pretrained(self.args.output) + # not export when using other exporters + from ...exporters.openvino.convert import export_tokenizer + output = Path(self.args.output) tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 41eb0200d5..234e34aa94 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -24,7 +24,7 @@ from optimum.exporters import TasksManager from optimum.exporters.onnx.base import OnnxConfig from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED -from optimum.exporters.openvino.convert import export_from_model, export_tokenizer +from optimum.exporters.openvino.convert import export_from_model from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version from optimum.utils.save_utils import maybe_load_preprocessors @@ -357,6 +357,8 @@ class StoreAttr(object): # avoid circular import from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER + # hide openvino import when using other exporters + from optimum.exporters.openvino.convert import export_tokenizer if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": From 1f44ce9da35249f08276ac8affa56ec0d63ac503 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 2 May 2024 18:41:27 +0100 Subject: [PATCH 6/9] Make Style --- optimum/exporters/openvino/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 234e34aa94..a43c42e441 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -355,10 +355,10 @@ class StoreAttr(object): **kwargs_shapes, ) - # avoid circular import - from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER # hide openvino import when using other exporters + # avoid circular import from optimum.exporters.openvino.convert import export_tokenizer + from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": From ed5cbb91e02c56e53784351f3befb69c56903171 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 7 May 2024 14:55:10 +0100 Subject: [PATCH 7/9] Apply Review Comments --- optimum/commands/export/openvino.py | 9 ++++++--- optimum/exporters/openvino/__main__.py | 4 ++-- optimum/intel/openvino/utils.py | 10 ++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index a7302ef88f..56abc6b7ce 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -226,6 +226,9 @@ def run(self): ) library_name = "transformers" + if self.args.convert_tokenizer: + logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.") + if ( library_name == "diffusers" and ov_config @@ -261,6 +264,9 @@ def run(self): ) model.save_pretrained(self.args.output) + if self.args.disable_convert_tokenizer: + return + # not export when using other exporters from ...exporters.openvino.convert import export_tokenizer @@ -273,9 +279,6 @@ def run(self): if tokenizer_2 is not None: export_tokenizer(tokenizer_2, output / "tokenizer_2") else: - if self.args.convert_tokenizer: - logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.") - # TODO : add input shapes main_export( model_name_or_path=self.args.model, diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index a43c42e441..0d80101a5e 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -358,7 +358,7 @@ class StoreAttr(object): # hide openvino import when using other exporters # avoid circular import from optimum.exporters.openvino.convert import export_tokenizer - from optimum.intel.openvino.utils import OV_TOKENIZER_FLOLDER + from optimum.intel.openvino.utils import OV_TOKENIZER_FOLDER if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": @@ -369,7 +369,7 @@ class StoreAttr(object): if tokenizer is not None: try: - export_tokenizer(tokenizer, output / OV_TOKENIZER_FLOLDER) + export_tokenizer(tokenizer, output / OV_TOKENIZER_FOLDER) except Exception as exception: logger.warning( "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 6b49f7a833..3bf00f0713 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -18,7 +18,7 @@ import os from glob import glob from pathlib import Path -from typing import List, Union +from typing import Tuple, Union import numpy as np from huggingface_hub import model_info @@ -34,7 +34,7 @@ OV_DECODER_NAME = "openvino_decoder_model.xml" OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml" -OV_TOKENIZER_FLOLDER = "openvino_tokenizer" +OV_TOKENIZER_FOLDER = "openvino_tokenizer" OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml" OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml" @@ -111,9 +111,7 @@ } -NEED_CONVERT_TO_FAST_TOKENIZER: List[type(PreTrainedTokenizer)] = [ - CLIPTokenizer, -] +NEED_CONVERT_TO_FAST_TOKENIZER: Tuple[type(PreTrainedTokenizer)] = (CLIPTokenizer,) def maybe_convert_tokenizer_to_fast( @@ -122,7 +120,7 @@ def maybe_convert_tokenizer_to_fast( if isinstance(hf_tokenizer, PreTrainedTokenizerFast): return hf_tokenizer - if any(isinstance(hf_tokenizer, slow_class) for slow_class in NEED_CONVERT_TO_FAST_TOKENIZER): + if isinstance(hf_tokenizer, NEED_CONVERT_TO_FAST_TOKENIZER): try: return AutoTokenizer.from_pretrained(tokenizer_path) except Exception: From 4fa40a259faf85fd30d7d930565533f4b1e11f32 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 7 May 2024 14:58:44 +0100 Subject: [PATCH 8/9] Apply Review Comments --- optimum/commands/export/openvino.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index 56abc6b7ce..025a40e057 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -267,7 +267,7 @@ def run(self): if self.args.disable_convert_tokenizer: return - # not export when using other exporters + # avoid import when using other exporters (IPEX, INC) from ...exporters.openvino.convert import export_tokenizer output = Path(self.args.output) From 0029e9165a2dad4cfcf787aa63181d9dc0cd49d5 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 9 May 2024 13:06:20 +0100 Subject: [PATCH 9/9] Move OV tokenizer to the root folder --- optimum/exporters/openvino/__main__.py | 4 +--- optimum/intel/openvino/utils.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 0d80101a5e..31abd0f327 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -356,9 +356,7 @@ class StoreAttr(object): ) # hide openvino import when using other exporters - # avoid circular import from optimum.exporters.openvino.convert import export_tokenizer - from optimum.intel.openvino.utils import OV_TOKENIZER_FOLDER if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": @@ -369,7 +367,7 @@ class StoreAttr(object): if tokenizer is not None: try: - export_tokenizer(tokenizer, output / OV_TOKENIZER_FOLDER) + export_tokenizer(tokenizer, output) except Exception as exception: logger.warning( "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 3bf00f0713..69a750fb65 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -34,7 +34,6 @@ OV_DECODER_NAME = "openvino_decoder_model.xml" OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml" -OV_TOKENIZER_FOLDER = "openvino_tokenizer" OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml" OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml"