From 94bc226ba20cba68a3dfad12d389262c37c0f117 Mon Sep 17 00:00:00 2001 From: Sergey Lyalin Date: Fri, 5 Jan 2024 13:44:09 +0000 Subject: [PATCH 01/22] Convert tokenizers with openvino_tokenizers --- optimum/exporters/openvino/__main__.py | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 54fe1193e5..cc35620357 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -19,6 +19,7 @@ from requests.exceptions import ConnectionError as RequestsConnectionError from transformers import AutoConfig, AutoTokenizer +from openvino import save_model from optimum.exporters import TasksManager from optimum.exporters.onnx import __main__ as optimum_main @@ -46,6 +47,24 @@ logger = logging.getLogger(__name__) +def tokenizer_export( + tokenizer, + output: Union[str, Path], + suffix: Optional[str] = "" +): + try: + from openvino_tokenizers import convert_tokenizer + ov_tokenizer, ov_detokenizer = convert_tokenizer(tokenizer, with_detokenizer=True) + if isinstance(output, str): + output = Path(output) + tokenizer_path = output.joinpath("openvino_tokenizer" + suffix + ".xml") + detokenizer_path = output.joinpath("openvino_detokenizer" + suffix + ".xml") + save_model(ov_tokenizer, tokenizer_path) + save_model(ov_detokenizer, detokenizer_path) + except Exception as exception: + print("[ WARNING ] OpenVINO tokenizer/detokenizer models couldn't be exported because of exception:", exception) + + def main_export( model_name_or_path: str, output: Union[str, Path], @@ -328,6 +347,12 @@ class StoreAttr(object): if generation_config is not None: generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) + try: + # Avoid loding it for the second time if loaded before + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + tokenizer_export(tokenizer, output) + except: + print("[ WARNING ] Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer models won't be generated.") if model.config.is_encoder_decoder and task.startswith("text-generation"): raise ValueError( @@ -358,10 +383,12 @@ class StoreAttr(object): tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: tokenizer.save_pretrained(output.joinpath("tokenizer")) + tokenizer_export(tokenizer, output) tokenizer_2 = getattr(model, "tokenizer_2", None) if tokenizer_2 is not None: tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) + tokenizer_export(tokenizer, output, "_2") model.save_config(output) From 6bb395f536613f1d0b86b97d49bc3a18afa1e676 Mon Sep 17 00:00:00 2001 From: Sergey Lyalin Date: Fri, 5 Jan 2024 19:22:21 +0400 Subject: [PATCH 02/22] Update optimum/exporters/openvino/__main__.py --- optimum/exporters/openvino/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index cc35620357..3c594679dd 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -348,7 +348,7 @@ class StoreAttr(object): generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) try: - # Avoid loding it for the second time if loaded before + # TODO: Avoid loading the tokenizer again if loaded before tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) tokenizer_export(tokenizer, output) except: From 7d16ec7d91cc9b42cbb3bb6b70d5ce4a69c945d8 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 9 Jan 2024 15:31:50 +0000 Subject: [PATCH 03/22] Refactor and Add Tests --- optimum/exporters/openvino/__main__.py | 46 ++++++++++++++++++-------- setup.py | 1 + tests/openvino/test_exporters_cli.py | 44 ++++++++++-------------- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 3c594679dd..d4245e01a7 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -41,6 +41,8 @@ ] OV_XML_FILE_NAME = "openvino_model.xml" +OV_TOKENIZER_FILE_NAME = "openvino_tokenizer{}.xml" +OV_DETOKENIZER_FILE_NAME = "openvino_detokenizer{}.xml" _MAX_UNCOMPRESSED_SIZE = 1e9 @@ -49,20 +51,33 @@ def tokenizer_export( tokenizer, - output: Union[str, Path], - suffix: Optional[str] = "" + output_path: Union[str, Path], + suffix: Optional[str] = "", ): + from openvino.runtime.exceptions import OVTypeError + try: from openvino_tokenizers import convert_tokenizer - ov_tokenizer, ov_detokenizer = convert_tokenizer(tokenizer, with_detokenizer=True) - if isinstance(output, str): - output = Path(output) - tokenizer_path = output.joinpath("openvino_tokenizer" + suffix + ".xml") - detokenizer_path = output.joinpath("openvino_detokenizer" + suffix + ".xml") - save_model(ov_tokenizer, tokenizer_path) - save_model(ov_detokenizer, detokenizer_path) - except Exception as exception: - print("[ WARNING ] OpenVINO tokenizer/detokenizer models couldn't be exported because of exception:", exception) + except ModuleNotFoundError: + logger.info("Run `pip install openvino-tokenizers` to get OpenVINO tokenizer/detokenizer models.") + + if not isinstance(output_path, Path): + output_path = Path(output_path) + + try: + converted = convert_tokenizer(tokenizer, with_detokenizer=True) + except NotImplementedError: + logger.info("Detokenizer is not supported, convert tokenizer only.") + converted = convert_tokenizer(tokenizer, with_detokenizer=False) + except OVTypeError: + logger.info("OpenVINO Tokenizer for this model is not supported.") + return + + if not isinstance(converted, tuple): + converted = (converted,) + + for model, file_name in zip(converted, (OV_TOKENIZER_FILE_NAME, OV_DETOKENIZER_FILE_NAME)): + save_model(model, output_path / file_name.format(suffix)) def main_export( @@ -351,8 +366,11 @@ class StoreAttr(object): # TODO: Avoid loading the tokenizer again if loaded before tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) tokenizer_export(tokenizer, output) - except: - print("[ WARNING ] Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer models won't be generated.") + except Exception as exception: + logger.warning( + "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer models won't " + f"be generated. Exception: {exception}" + ) if model.config.is_encoder_decoder and task.startswith("text-generation"): raise ValueError( @@ -388,7 +406,7 @@ class StoreAttr(object): tokenizer_2 = getattr(model, "tokenizer_2", None) if tokenizer_2 is not None: tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) - tokenizer_export(tokenizer, output, "_2") + tokenizer_export(tokenizer, output, suffix="_2") model.save_config(output) diff --git a/setup.py b/setup.py index 5c83182476..983a63ed4c 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ "transformers>=4.34.0", ], "openvino": ["openvino>=2023.2", "onnx", "onnxruntime", "transformers>=4.34.0"], + "openvino-tokenizers": ["openvino-tokenizers"], "nncf": ["nncf>=2.7.0"], "ipex": ["transformers<4.32.0", "intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index b90490d610..50cd987a37 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -13,6 +13,7 @@ # limitations under the License. import subprocess import unittest +from pathlib import Path from tempfile import TemporaryDirectory from parameterized import parameterized @@ -61,6 +62,18 @@ class OVCLIExportTestCase(unittest.TestCase): ("stable-diffusion-xl", "stable-diffusion-xl"), ("stable-diffusion-xl", "stable-diffusion-xl-refiner"), ) + EXPECTED_NUMBER_OF_TOKENIZER_MODELS = { + "gpt2": 2, + "t5": 2, # bug for t5 tokenizer + "albert": 0, # not supported yet + "distilbert": 1, # no detokenizer + "roberta": 2, + "vit": 0, # no tokenizer for image model + "wav2vec2": 0, # no tokenizer + "bert": 1, # no detokenizer + "blenderbot": 2, + "stable-diffusion": 4 # two tokenizers + } SUPPORTED_4BIT_ARCHITECTURES = (("text-generation-with-past", "opt125m"),) @@ -84,38 +97,17 @@ def _openvino_export( ) @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_export(self, task: str, model_type: str): - self._openvino_export(MODEL_NAMES[model_type], task) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli(self, task: str, model_type: str): + def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: subprocess.run( f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", shell=True, check=True, ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli_fp16(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", - shell=True, - check=True, - ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli_int8(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", - shell=True, - check=True, + save_dir = Path(tmpdir) + self.assertEqual( + self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], + sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))), ) model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) From f0933ad6996d5a904f87fcf6faf41bb96518d35b Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Wed, 10 Jan 2024 11:32:59 +0000 Subject: [PATCH 04/22] Fix t5 Test --- tests/openvino/test_exporters_cli.py | 37 +++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 50cd987a37..7b027fab23 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -64,7 +64,7 @@ class OVCLIExportTestCase(unittest.TestCase): ) EXPECTED_NUMBER_OF_TOKENIZER_MODELS = { "gpt2": 2, - "t5": 2, # bug for t5 tokenizer + "t5": 0, # failed internal sentencepiece check - no token in the vocab "albert": 0, # not supported yet "distilbert": 1, # no detokenizer "roberta": 2, @@ -96,6 +96,21 @@ def _openvino_export( compression_ratio=compression_ratio, ) + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_export(self, task: str, model_type: str): + self._openvino_export(MODEL_NAMES[model_type], task) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: @@ -109,6 +124,26 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str): self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))), ) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_fp16(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_int8(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", + shell=True, + check=True, + ) model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) From 24cc6164a82bd492b34d9682f9810723371f9719 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Wed, 10 Jan 2024 12:04:43 +0000 Subject: [PATCH 05/22] Add Warning --- optimum/exporters/openvino/__main__.py | 3 + tests/openvino/test_exporters_cli.py | 143 +++++++++++++------------ 2 files changed, 77 insertions(+), 69 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index d4245e01a7..5343d80bf0 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -72,6 +72,9 @@ def tokenizer_export( except OVTypeError: logger.info("OpenVINO Tokenizer for this model is not supported.") return + except Exception as exception: + logger.warning(f"OpenVINO Tokenizer for this model is not supported. Exception: {exception}") + return if not isinstance(converted, tuple): converted = (converted,) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 7b027fab23..064d180abb 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -72,7 +72,8 @@ class OVCLIExportTestCase(unittest.TestCase): "wav2vec2": 0, # no tokenizer "bert": 1, # no detokenizer "blenderbot": 2, - "stable-diffusion": 4 # two tokenizers + "stable-diffusion": 0, # not supported + "stable-diffusion-xl": 0, # not supported } SUPPORTED_4BIT_ARCHITECTURES = (("text-generation-with-past", "opt125m"),) @@ -96,22 +97,26 @@ def _openvino_export( compression_ratio=compression_ratio, ) - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_export(self, task: str, model_type: str): - self._openvino_export(MODEL_NAMES[model_type], task) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", - shell=True, - check=True, - ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) + # @parameterized.expand(SUPPORTED_ARCHITECTURES) + # def test_export(self, task: str, model_type: str): + # self._openvino_export(MODEL_NAMES[model_type], task) + # + # @parameterized.expand(SUPPORTED_ARCHITECTURES) + # def test_exporters_cli(self, task: str, model_type: str): + # with TemporaryDirectory() as tmpdir: + # subprocess.run( + # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", + # shell=True, + # check=True, + # ) + # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + # eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + @parameterized.expand( + arch + for arch in SUPPORTED_ARCHITECTURES + if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner") + ) def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: subprocess.run( @@ -125,55 +130,55 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str): sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))), ) - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli_fp16(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", - shell=True, - check=True, - ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - @parameterized.expand(SUPPORTED_ARCHITECTURES) - def test_exporters_cli_int8(self, task: str, model_type: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", - shell=True, - check=True, - ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - if task.startswith("text2text-generation"): - models = [model.encoder, model.decoder] - if task.endswith("with-past"): - models.append(model.decoder_with_past) - elif task.startswith("stable-diffusion"): - models = [model.unet, model.vae_encoder, model.vae_decoder] - models.append(model.text_encoder if task == "stable-diffusion" else model.text_encoder_2) - else: - models = [model] - - expected_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] - for i, model in enumerate(models): - _, num_int8, _ = get_num_quantized_nodes(model) - self.assertEqual(expected_int8[i], num_int8) - - @parameterized.expand(TEST_4BIT_CONFIGURATONS) - def test_exporters_cli_int4(self, task: str, model_type: str, option: str): - with TemporaryDirectory() as tmpdir: - subprocess.run( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format {option} {tmpdir}", - shell=True, - check=True, - ) - model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - - expected_int8, expected_int4 = _ARCHITECTURES_TO_EXPECTED_INT4_INT8[model_type] - _, num_int8, num_int4 = get_num_quantized_nodes(model) - self.assertEqual(expected_int8, num_int8) - self.assertEqual(expected_int4, num_int4) + # @parameterized.expand(SUPPORTED_ARCHITECTURES) + # def test_exporters_cli_fp16(self, task: str, model_type: str): + # with TemporaryDirectory() as tmpdir: + # subprocess.run( + # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", + # shell=True, + # check=True, + # ) + # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + # eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + # + # @parameterized.expand(SUPPORTED_ARCHITECTURES) + # def test_exporters_cli_int8(self, task: str, model_type: str): + # with TemporaryDirectory() as tmpdir: + # subprocess.run( + # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", + # shell=True, + # check=True, + # ) + # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + # model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + # + # if task.startswith("text2text-generation"): + # models = [model.encoder, model.decoder] + # if task.endswith("with-past"): + # models.append(model.decoder_with_past) + # elif task.startswith("stable-diffusion"): + # models = [model.unet, model.vae_encoder, model.vae_decoder] + # models.append(model.text_encoder if task == "stable-diffusion" else model.text_encoder_2) + # else: + # models = [model] + # + # expected_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] + # for i, model in enumerate(models): + # _, num_int8, _ = get_num_quantized_nodes(model) + # self.assertEqual(expected_int8[i], num_int8) + # + # @parameterized.expand(TEST_4BIT_CONFIGURATONS) + # def test_exporters_cli_int4(self, task: str, model_type: str, option: str): + # with TemporaryDirectory() as tmpdir: + # subprocess.run( + # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format {option} {tmpdir}", + # shell=True, + # check=True, + # ) + # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + # model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + # + # expected_int8, expected_int4 = _ARCHITECTURES_TO_EXPECTED_INT4_INT8[model_type] + # _, num_int8, num_int4 = get_num_quantized_nodes(model) + # self.assertEqual(expected_int8, num_int8) + # self.assertEqual(expected_int4, num_int4) From 49337b0eb4786b4917adecbfd715f13eccbc00ab Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Wed, 10 Jan 2024 12:09:11 +0000 Subject: [PATCH 06/22] Return Tests --- tests/openvino/test_exporters_cli.py | 132 +++++++++++++-------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 064d180abb..dbb1fa8fd4 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -97,20 +97,20 @@ def _openvino_export( compression_ratio=compression_ratio, ) - # @parameterized.expand(SUPPORTED_ARCHITECTURES) - # def test_export(self, task: str, model_type: str): - # self._openvino_export(MODEL_NAMES[model_type], task) - # - # @parameterized.expand(SUPPORTED_ARCHITECTURES) - # def test_exporters_cli(self, task: str, model_type: str): - # with TemporaryDirectory() as tmpdir: - # subprocess.run( - # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", - # shell=True, - # check=True, - # ) - # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - # eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_export(self, task: str, model_type: str): + self._openvino_export(MODEL_NAMES[model_type], task) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) @parameterized.expand( arch @@ -130,55 +130,55 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str): sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))), ) - # @parameterized.expand(SUPPORTED_ARCHITECTURES) - # def test_exporters_cli_fp16(self, task: str, model_type: str): - # with TemporaryDirectory() as tmpdir: - # subprocess.run( - # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", - # shell=True, - # check=True, - # ) - # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - # eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - # - # @parameterized.expand(SUPPORTED_ARCHITECTURES) - # def test_exporters_cli_int8(self, task: str, model_type: str): - # with TemporaryDirectory() as tmpdir: - # subprocess.run( - # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", - # shell=True, - # check=True, - # ) - # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - # model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - # - # if task.startswith("text2text-generation"): - # models = [model.encoder, model.decoder] - # if task.endswith("with-past"): - # models.append(model.decoder_with_past) - # elif task.startswith("stable-diffusion"): - # models = [model.unet, model.vae_encoder, model.vae_decoder] - # models.append(model.text_encoder if task == "stable-diffusion" else model.text_encoder_2) - # else: - # models = [model] - # - # expected_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] - # for i, model in enumerate(models): - # _, num_int8, _ = get_num_quantized_nodes(model) - # self.assertEqual(expected_int8[i], num_int8) - # - # @parameterized.expand(TEST_4BIT_CONFIGURATONS) - # def test_exporters_cli_int4(self, task: str, model_type: str, option: str): - # with TemporaryDirectory() as tmpdir: - # subprocess.run( - # f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format {option} {tmpdir}", - # shell=True, - # check=True, - # ) - # model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} - # model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) - # - # expected_int8, expected_int4 = _ARCHITECTURES_TO_EXPECTED_INT4_INT8[model_type] - # _, num_int8, num_int4 = get_num_quantized_nodes(model) - # self.assertEqual(expected_int8, num_int8) - # self.assertEqual(expected_int4, num_int4) + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_fp16(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format fp16 {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + @parameterized.expand(SUPPORTED_ARCHITECTURES) + def test_exporters_cli_int8(self, task: str, model_type: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format int8 {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + if task.startswith("text2text-generation"): + models = [model.encoder, model.decoder] + if task.endswith("with-past"): + models.append(model.decoder_with_past) + elif task.startswith("stable-diffusion"): + models = [model.unet, model.vae_encoder, model.vae_decoder] + models.append(model.text_encoder if task == "stable-diffusion" else model.text_encoder_2) + else: + models = [model] + + expected_int8 = _ARCHITECTURES_TO_EXPECTED_INT8[model_type] + for i, model in enumerate(models): + _, num_int8, _ = get_num_quantized_nodes(model) + self.assertEqual(expected_int8[i], num_int8) + + @parameterized.expand(TEST_4BIT_CONFIGURATONS) + def test_exporters_cli_int4(self, task: str, model_type: str, option: str): + with TemporaryDirectory() as tmpdir: + subprocess.run( + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} --weight-format {option} {tmpdir}", + shell=True, + check=True, + ) + model_kwargs = {"use_cache": task.endswith("with-past")} if "generation" in task else {} + model = eval(_HEAD_TO_AUTOMODELS[task.replace("-with-past", "")]).from_pretrained(tmpdir, **model_kwargs) + + expected_int8, expected_int4 = _ARCHITECTURES_TO_EXPECTED_INT4_INT8[model_type] + _, num_int8, num_int4 = get_num_quantized_nodes(model) + self.assertEqual(expected_int8, num_int8) + self.assertEqual(expected_int4, num_int4) From 7709043e825ded517b37a0ef9498275ef9d77cd9 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Wed, 10 Jan 2024 20:26:59 +0000 Subject: [PATCH 07/22] Move export_tokenizer to convert.py Reuse existing preprocessors --- optimum/exporters/openvino/__main__.py | 68 ++++++-------------------- optimum/exporters/openvino/convert.py | 48 ++++++++++++++++++ 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 5343d80bf0..8ba9f66080 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -18,8 +18,7 @@ from typing import Any, Callable, Dict, Optional, Union from requests.exceptions import ConnectionError as RequestsConnectionError -from transformers import AutoConfig, AutoTokenizer -from openvino import save_model +from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase from optimum.exporters import TasksManager from optimum.exporters.onnx import __main__ as optimum_main @@ -28,8 +27,7 @@ from optimum.utils.save_utils import maybe_load_preprocessors, maybe_save_preprocessors from ...intel.utils.import_utils import is_nncf_available, is_optimum_version, is_transformers_version -from .convert import export_models - +from .convert import export_models, export_tokenizer if is_optimum_version(">=", "1.16.0"): from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED @@ -41,48 +39,11 @@ ] OV_XML_FILE_NAME = "openvino_model.xml" -OV_TOKENIZER_FILE_NAME = "openvino_tokenizer{}.xml" -OV_DETOKENIZER_FILE_NAME = "openvino_detokenizer{}.xml" - _MAX_UNCOMPRESSED_SIZE = 1e9 logger = logging.getLogger(__name__) -def tokenizer_export( - tokenizer, - output_path: Union[str, Path], - suffix: Optional[str] = "", -): - from openvino.runtime.exceptions import OVTypeError - - try: - from openvino_tokenizers import convert_tokenizer - except ModuleNotFoundError: - logger.info("Run `pip install openvino-tokenizers` to get OpenVINO tokenizer/detokenizer models.") - - if not isinstance(output_path, Path): - output_path = Path(output_path) - - try: - converted = convert_tokenizer(tokenizer, with_detokenizer=True) - except NotImplementedError: - logger.info("Detokenizer is not supported, convert tokenizer only.") - converted = convert_tokenizer(tokenizer, with_detokenizer=False) - except OVTypeError: - logger.info("OpenVINO Tokenizer for this model is not supported.") - return - except Exception as exception: - logger.warning(f"OpenVINO Tokenizer for this model is not supported. Exception: {exception}") - return - - if not isinstance(converted, tuple): - converted = (converted,) - - for model, file_name in zip(converted, (OV_TOKENIZER_FILE_NAME, OV_DETOKENIZER_FILE_NAME)): - save_model(model, output_path / file_name.format(suffix)) - - def main_export( model_name_or_path: str, output: Union[str, Path], @@ -353,7 +314,7 @@ class StoreAttr(object): model.config.pad_token_id = pad_token_id else: try: - tok = AutoTokenizer.from_pretrained(model_name_or_path) + tok = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code) model.config.pad_token_id = tok.pad_token_id except Exception: raise ValueError( @@ -365,15 +326,16 @@ class StoreAttr(object): if generation_config is not None: generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) - try: - # TODO: Avoid loading the tokenizer again if loaded before - tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) - tokenizer_export(tokenizer, output) - except Exception as exception: - logger.warning( - "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer models won't " - f"be generated. Exception: {exception}" - ) + + for preprocessor in preprocessors: + if isinstance(preprocessor, PreTrainedTokenizerBase): + try: + export_tokenizer(preprocessor, output) + except Exception as exception: + logger.warning( + "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " + f"models won't be generated. Exception: {exception}" + ) if model.config.is_encoder_decoder and task.startswith("text-generation"): raise ValueError( @@ -404,12 +366,12 @@ class StoreAttr(object): tokenizer = getattr(model, "tokenizer", None) if tokenizer is not None: tokenizer.save_pretrained(output.joinpath("tokenizer")) - tokenizer_export(tokenizer, output) + export_tokenizer(tokenizer, output) tokenizer_2 = getattr(model, "tokenizer_2", None) if tokenizer_2 is not None: tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) - tokenizer_export(tokenizer, output, suffix="_2") + export_tokenizer(tokenizer, output, suffix="_2") model.save_config(output) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 56c5a10e5d..35075c0b00 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union +from transformers import T5Tokenizer, T5TokenizerFast from transformers.utils import is_tf_available, is_torch_available from openvino.runtime import PartialShape, save_model @@ -486,3 +487,50 @@ def export_models( outputs = list(map(list, zip(*outputs))) return outputs + + +OV_TOKENIZER_FILE_NAME = "openvino_tokenizer{}.xml" +OV_DETOKENIZER_FILE_NAME = "openvino_detokenizer{}.xml" +UNSUPPORTED_TOKENZIER_CLASSES = ( + T5Tokenizer, + T5TokenizerFast, +) + + +def export_tokenizer( + tokenizer, + output_path: Union[str, Path], + suffix: Optional[str] = "", +): + from openvino.runtime.exceptions import OVTypeError + + + if isinstance(tokenizer, UNSUPPORTED_TOKENZIER_CLASSES): + logger.info("OpenVINO Tokenizer for this model is not supported.") + return + + try: + from openvino_tokenizers import convert_tokenizer + except ModuleNotFoundError: + logger.info("Run `pip install openvino-tokenizers` to get OpenVINO tokenizer/detokenizer models.") + + if not isinstance(output_path, Path): + output_path = Path(output_path) + + try: + converted = convert_tokenizer(tokenizer, with_detokenizer=True) + except NotImplementedError: + logger.info("Detokenizer is not supported, convert tokenizer only.") + converted = convert_tokenizer(tokenizer, with_detokenizer=False) + except OVTypeError: + logger.info("OpenVINO Tokenizer for this model is not supported.") + return + except Exception as exception: + logger.warning(f"OpenVINO Tokenizer for this model is not supported. Exception: {exception}") + return + + if not isinstance(converted, tuple): + converted = (converted,) + + for model, file_name in zip(converted, (OV_TOKENIZER_FILE_NAME, OV_DETOKENIZER_FILE_NAME)): + save_model(model, output_path / file_name.format(suffix)) From dbd609b10dad819e5e472d9625f817f2dad2b2ef Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Fri, 12 Jan 2024 16:12:34 +0000 Subject: [PATCH 08/22] Avoid Double Tokenizer Save --- optimum/exporters/openvino/__main__.py | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 8ba9f66080..f9070f8912 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -309,13 +309,18 @@ class StoreAttr(object): and getattr(model.config, "pad_token_id", None) is None and task in ["text-classification"] ) + + tokenizer = next( + (preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)), + None + ) + if needs_pad_token_id: if pad_token_id is not None: model.config.pad_token_id = pad_token_id - else: + elif tokenizer is not None: try: - tok = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code) - model.config.pad_token_id = tok.pad_token_id + model.config.pad_token_id = tokenizer.pad_token_id except Exception: raise ValueError( "Could not infer the pad token id, which is needed in this case, please provide it with the --pad_token_id argument" @@ -327,15 +332,14 @@ class StoreAttr(object): generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) - for preprocessor in preprocessors: - if isinstance(preprocessor, PreTrainedTokenizerBase): - try: - export_tokenizer(preprocessor, output) - except Exception as exception: - logger.warning( - "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " - f"models won't be generated. Exception: {exception}" - ) + if tokenizer is not None: + try: + export_tokenizer(tokenizer, output) + except Exception as exception: + logger.warning( + "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer " + f"models won't be generated. Exception: {exception}" + ) if model.config.is_encoder_decoder and task.startswith("text-generation"): raise ValueError( From 7e24f100823c888e3a613e290a0edfa321984c51 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Fri, 12 Jan 2024 17:41:17 +0000 Subject: [PATCH 09/22] Fix Style --- optimum/exporters/openvino/__main__.py | 6 +++--- optimum/exporters/openvino/convert.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index f9070f8912..3f54d6c188 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -18,7 +18,7 @@ from typing import Any, Callable, Dict, Optional, Union from requests.exceptions import ConnectionError as RequestsConnectionError -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase +from transformers import AutoConfig, PreTrainedTokenizerBase from optimum.exporters import TasksManager from optimum.exporters.onnx import __main__ as optimum_main @@ -29,6 +29,7 @@ from ...intel.utils.import_utils import is_nncf_available, is_optimum_version, is_transformers_version from .convert import export_models, export_tokenizer + if is_optimum_version(">=", "1.16.0"): from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED else: @@ -311,8 +312,7 @@ class StoreAttr(object): ) tokenizer = next( - (preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)), - None + (preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)), None ) if needs_pad_token_id: diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 35075c0b00..036d5f4c33 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -504,7 +504,6 @@ def export_tokenizer( ): from openvino.runtime.exceptions import OVTypeError - if isinstance(tokenizer, UNSUPPORTED_TOKENZIER_CLASSES): logger.info("OpenVINO Tokenizer for this model is not supported.") return From 2cf460d4de619784b41ef55a526f3e38d780feca Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 18 Jan 2024 11:22:34 +0000 Subject: [PATCH 10/22] Refactor After Review --- optimum/exporters/openvino/convert.py | 27 +++++++++++++-------------- optimum/intel/openvino/__init__.py | 9 ++++++++- optimum/intel/openvino/utils.py | 3 +++ 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 036d5f4c33..f31e0f2b20 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -23,6 +23,7 @@ from transformers.utils import is_tf_available, is_torch_available from openvino.runtime import PartialShape, save_model +from openvino.runtime.exceptions import OVTypeError from openvino.runtime.utils.types import get_element_type from openvino.tools.ovc import convert_model from optimum.exporters.onnx.base import OnnxConfig @@ -489,9 +490,7 @@ def export_models( return outputs -OV_TOKENIZER_FILE_NAME = "openvino_tokenizer{}.xml" -OV_DETOKENIZER_FILE_NAME = "openvino_detokenizer{}.xml" -UNSUPPORTED_TOKENZIER_CLASSES = ( +UNSUPPORTED_TOKENIZER_CLASSES = ( T5Tokenizer, T5TokenizerFast, ) @@ -499,13 +498,13 @@ def export_models( def export_tokenizer( tokenizer, - output_path: Union[str, Path], + output: Union[str, Path], suffix: Optional[str] = "", ): - from openvino.runtime.exceptions import OVTypeError + from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME # avoid circular imports - if isinstance(tokenizer, UNSUPPORTED_TOKENZIER_CLASSES): - logger.info("OpenVINO Tokenizer for this model is not supported.") + if isinstance(tokenizer, UNSUPPORTED_TOKENIZER_CLASSES): + logger.info(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.") return try: @@ -513,23 +512,23 @@ def export_tokenizer( except ModuleNotFoundError: logger.info("Run `pip install openvino-tokenizers` to get OpenVINO tokenizer/detokenizer models.") - if not isinstance(output_path, Path): - output_path = Path(output_path) + if not isinstance(output, Path): + output = Path(output) try: converted = convert_tokenizer(tokenizer, with_detokenizer=True) except NotImplementedError: - logger.info("Detokenizer is not supported, convert tokenizer only.") + logger.warning("Detokenizer is not supported, convert tokenizer only.") converted = convert_tokenizer(tokenizer, with_detokenizer=False) except OVTypeError: - logger.info("OpenVINO Tokenizer for this model is not supported.") + logger.warning(f"OpenVINO Tokenizer for {type(tokenizer).__name__} is not supported.") return except Exception as exception: - logger.warning(f"OpenVINO Tokenizer for this model is not supported. Exception: {exception}") + logger.warning(f"OpenVINO Tokenizer {type(tokenizer).__name__} is not supported. Exception: {exception}") return if not isinstance(converted, tuple): converted = (converted,) - for model, file_name in zip(converted, (OV_TOKENIZER_FILE_NAME, OV_DETOKENIZER_FILE_NAME)): - save_model(model, output_path / file_name.format(suffix)) + for model, file_name in zip(converted, (OV_TOKENIZER_NAME, OV_DETOKENIZER_NAME)): + save_model(model, output / file_name.format(suffix)) diff --git a/optimum/intel/openvino/__init__.py b/optimum/intel/openvino/__init__.py index 6999c6b48f..a15edb9cc3 100644 --- a/optimum/intel/openvino/__init__.py +++ b/optimum/intel/openvino/__init__.py @@ -14,7 +14,14 @@ import logging from ..utils.import_utils import is_diffusers_available, is_nncf_available -from .utils import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME +from .utils import ( + OV_DECODER_NAME, + OV_DECODER_WITH_PAST_NAME, + OV_DETOKENIZER_NAME, + OV_ENCODER_NAME, + OV_TOKENIZER_NAME, + OV_XML_FILE_NAME, +) if is_nncf_available(): diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 9548c750bf..1c34d98d94 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -31,6 +31,9 @@ OV_DECODER_NAME = "openvino_decoder_model.xml" OV_DECODER_WITH_PAST_NAME = "openvino_decoder_with_past_model.xml" +OV_TOKENIZER_NAME = "openvino_tokenizer{}.xml" +OV_DETOKENIZER_NAME = "openvino_detokenizer{}.xml" + ONNX_WEIGHTS_NAME = "model.onnx" ONNX_ENCODER_NAME = "encoder_model.onnx" ONNX_DECODER_NAME = "decoder_model.onnx" From 57782d1cd7427119477a25818d7834c81c3f8a68 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 18 Jan 2024 12:48:54 +0000 Subject: [PATCH 11/22] Skip Tokenizers Tests If No Package Installed Check logs from tokneizers test --- optimum/exporters/openvino/convert.py | 10 +++++++--- tests/openvino/test_exporters_cli.py | 25 ++++++++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index f31e0f2b20..05b1056326 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -510,7 +510,9 @@ def export_tokenizer( try: from openvino_tokenizers import convert_tokenizer except ModuleNotFoundError: - logger.info("Run `pip install openvino-tokenizers` to get OpenVINO tokenizer/detokenizer models.") + logger.info( + "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models." + ) if not isinstance(output, Path): output = Path(output) @@ -521,10 +523,12 @@ def export_tokenizer( logger.warning("Detokenizer is not supported, convert tokenizer only.") converted = convert_tokenizer(tokenizer, with_detokenizer=False) except OVTypeError: - logger.warning(f"OpenVINO Tokenizer for {type(tokenizer).__name__} is not supported.") + logger.warning(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.") return except Exception as exception: - logger.warning(f"OpenVINO Tokenizer {type(tokenizer).__name__} is not supported. Exception: {exception}") + logger.warning( + f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported. Exception: {exception}" + ) return if not isinstance(converted, tuple): diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index dbb1fa8fd4..955249889a 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -41,6 +41,13 @@ from optimum.intel.openvino.utils import _HEAD_TO_AUTOMODELS +try: + + OV_TOKENIZERS_NOT_AVAILABLE = False +except Exception: + OV_TOKENIZERS_NOT_AVAILABLE = True + + class OVCLIExportTestCase(unittest.TestCase): """ Integration tests ensuring supported models are correctly exported. @@ -117,18 +124,22 @@ def test_exporters_cli(self, task: str, model_type: str): for arch in SUPPORTED_ARCHITECTURES if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner") ) + @unittest.skipIf(OV_TOKENIZERS_NOT_AVAILABLE, reason="OpenVINO Tokenizers not available") def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: - subprocess.run( + output = subprocess.check_output( f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", shell=True, - check=True, - ) + stderr=subprocess.STDOUT, + ).decode() save_dir = Path(tmpdir) - self.assertEqual( - self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], - sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))), - ) + number_of_tokenizers = sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))) + self.assertEqual(self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], number_of_tokenizers) + + if number_of_tokenizers == 1: + self.assertFalse("Detokenizer is not supported, convert tokenizer only." in output) + elif number_of_tokenizers == 0: + self.assertFalse("OpenVINO Tokenizer export for" in output and "is not supported." in output) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_exporters_cli_fp16(self, task: str, model_type: str): From e7cd70f375a235816599ca42a30b64ad79b58c9d Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Fri, 19 Jan 2024 09:16:56 +0000 Subject: [PATCH 12/22] Style Fix --- tests/openvino/test_exporters_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 46f6dbef9c..8bff28cb26 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -42,7 +42,6 @@ try: - OV_TOKENIZERS_NOT_AVAILABLE = False except Exception: OV_TOKENIZERS_NOT_AVAILABLE = True From 40cf1171319b36512638122491aba550debf7986 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Fri, 19 Jan 2024 09:24:43 +0000 Subject: [PATCH 13/22] Fix OV Tokenizers Check --- tests/openvino/test_exporters_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 8bff28cb26..faf56f8d54 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -42,6 +42,7 @@ try: + import openvino_tokenizers OV_TOKENIZERS_NOT_AVAILABLE = False except Exception: OV_TOKENIZERS_NOT_AVAILABLE = True From 901f48a50c5e24e8977c774ef00e4ce51e703193 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Fri, 19 Jan 2024 16:53:34 +0000 Subject: [PATCH 14/22] Fix Tests --- optimum/exporters/openvino/__main__.py | 1 - optimum/intel/utils/import_utils.py | 13 +++++++++++++ tests/openvino/test_exporters_cli.py | 22 ++++++++++------------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index a8925e9c22..0a4a3f5a5b 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -27,7 +27,6 @@ from optimum.utils.save_utils import maybe_load_preprocessors, maybe_save_preprocessors from ...intel.utils.import_utils import is_nncf_available, is_optimum_version, is_transformers_version - from .convert import export_models, export_tokenizer from .stateful import ensure_export_task_support_stateful diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index 3f3fa6c55b..b993d2a394 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -92,6 +92,15 @@ _nncf_available = False +_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None +_openvino_tokenizers_version = "N/A" +if _openvino_tokenizers_available: + try: + _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers") + except importlib_metadata.PackageNotFoundError: + _openvino_tokenizers_available = False + + _diffusers_available = importlib.util.find_spec("diffusers") is not None _diffusers_version = "N/A" if _diffusers_available: @@ -135,6 +144,10 @@ def is_openvino_available(): return _openvino_available +def is_openvino_tokenizers_available(): + return _openvino_tokenizers_available + + def is_nncf_available(): return _nncf_available diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index faf56f8d54..822a47a6f6 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -39,13 +39,7 @@ OVStableDiffusionXLPipeline, ) from optimum.intel.openvino.utils import _HEAD_TO_AUTOMODELS - - -try: - import openvino_tokenizers - OV_TOKENIZERS_NOT_AVAILABLE = False -except Exception: - OV_TOKENIZERS_NOT_AVAILABLE = True +from optimum.intel.utils.import_utils import is_openvino_tokenizers_available class OVCLIExportTestCase(unittest.TestCase): @@ -124,7 +118,7 @@ def test_exporters_cli(self, task: str, model_type: str): for arch in SUPPORTED_ARCHITECTURES if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner") ) - @unittest.skipIf(OV_TOKENIZERS_NOT_AVAILABLE, reason="OpenVINO Tokenizers not available") + @unittest.skipIf(not is_openvino_tokenizers_available(), reason="OpenVINO Tokenizers not available") def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: output = subprocess.check_output( @@ -134,12 +128,16 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str): ).decode() save_dir = Path(tmpdir) number_of_tokenizers = sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))) - self.assertEqual(self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], number_of_tokenizers) + self.assertEqual( + self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], + number_of_tokenizers, + f"OVT: {is_openvino_tokenizers_available() }", + ) if number_of_tokenizers == 1: - self.assertFalse("Detokenizer is not supported, convert tokenizer only." in output) - elif number_of_tokenizers == 0: - self.assertFalse("OpenVINO Tokenizer export for" in output and "is not supported." in output) + self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output) + elif number_of_tokenizers == 0 and task not in ("image-classification", "audio-classification"): + self.assertTrue(("OpenVINO Tokenizer export for" in output and "is not supported." in output), output) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_exporters_cli_fp16(self, task: str, model_type: str): From 1a76bd417c11e547d5363d6c463f722e639de888 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 23 Jan 2024 17:55:09 +0000 Subject: [PATCH 15/22] Add Missing return --- optimum/exporters/openvino/convert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 2c75125848..5efdd363ac 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -558,6 +558,7 @@ def export_tokenizer( logger.info( "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models." ) + return if not isinstance(output, Path): output = Path(output) From f2b2237c3bef1e0b8f2e30d37b942f06bf58617b Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 23 Jan 2024 18:06:57 +0000 Subject: [PATCH 16/22] Turn off tokenizer message if not installed --- optimum/exporters/openvino/convert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 5efdd363ac..192ae76bbe 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -555,9 +555,10 @@ def export_tokenizer( try: from openvino_tokenizers import convert_tokenizer except ModuleNotFoundError: - logger.info( - "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models." - ) + # avoid this message before tokenizers are part of the openvino dependencies + # logger.info( + # "Run `pip install openvino-tokenizers[transformers]` to get OpenVINO tokenizer/detokenizer models." + # ) return if not isinstance(output, Path): From 7ee347e5992858c609ef35845f9ae32599c56315 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Mon, 29 Jan 2024 15:33:43 +0000 Subject: [PATCH 17/22] Move tokenizers to OV dependencies --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d6f435c419..c942ff9246 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,14 @@ "onnxruntime<1.15.0", "transformers>=4.34.0", ], - "openvino": ["openvino>=2023.2", "onnx", "onnxruntime", "transformers>=4.36.0", "optimum>=1.16.1"], - "openvino-tokenizers": ["openvino-tokenizers"], + "openvino": [ + "openvino>=2023.2", + "onnx", + "onnxruntime", + "transformers>=4.36.0", + "optimum>=1.16.1", + "openvino-tokenizers[transformers]", + ], "nncf": ["nncf>=2.7.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], From 8d2ec413c83aff0cb7352ed7e8e9dc74784657c0 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 30 Jan 2024 13:03:57 +0000 Subject: [PATCH 18/22] Check OV Compatibility --- optimum/exporters/openvino/__main__.py | 13 +++++--- optimum/intel/utils/import_utils.py | 42 ++++++++++++++++++-------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 0a4a3f5a5b..72124b220d 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -26,7 +26,12 @@ from optimum.utils import DEFAULT_DUMMY_SHAPES from optimum.utils.save_utils import maybe_load_preprocessors, maybe_save_preprocessors -from ...intel.utils.import_utils import is_nncf_available, is_optimum_version, is_transformers_version +from ...intel.utils.import_utils import ( + is_nncf_available, + is_openvino_tokenizers_available, + is_optimum_version, + is_transformers_version, +) from .convert import export_models, export_tokenizer from .stateful import ensure_export_task_support_stateful @@ -339,7 +344,7 @@ class StoreAttr(object): generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) - if tokenizer is not None: + if tokenizer is not None and is_openvino_tokenizers_available(): try: export_tokenizer(tokenizer, output) except Exception as exception: @@ -375,12 +380,12 @@ class StoreAttr(object): feature_extractor.save_pretrained(output.joinpath("feature_extractor")) tokenizer = getattr(model, "tokenizer", None) - if tokenizer is not None: + if tokenizer is not None and is_openvino_tokenizers_available(): tokenizer.save_pretrained(output.joinpath("tokenizer")) export_tokenizer(tokenizer, output) tokenizer_2 = getattr(model, "tokenizer_2", None) - if tokenizer_2 is not None: + if tokenizer_2 is not None and is_openvino_tokenizers_available(): tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) export_tokenizer(tokenizer, output, suffix="_2") diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index b993d2a394..e444e4e29a 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import importlib.util +import logging import operator as op import sys from collections import OrderedDict @@ -27,6 +28,8 @@ import importlib.metadata as importlib_metadata +logger = logging.getLogger(__name__) + STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt} _optimum_version = importlib_metadata.version("optimum") @@ -75,13 +78,38 @@ version = get_version() # avoid invalid format if "-" in version: - major_version, dev_info = version.split("-", 1) + ov_major_version, dev_info = version.split("-", 1) commit_id = dev_info.split("-")[0] - version = f"{major_version}-{commit_id}" + version = f"{ov_major_version}-{commit_id}" _openvino_version = version except ImportError: _openvino_available = False +_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available +_openvino_tokenizers_version = "N/A" +if _openvino_tokenizers_available: + try: + _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers") + except importlib_metadata.PackageNotFoundError: + _openvino_tokenizers_available = False + +if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A": + _compatible_openvino_version = next( + ( + requirement.split("==")[-1] + for requirement in importlib_metadata.requires("openvino-tokenizers") + if requirement.startswith("openvino==") + ), + "", + ) + _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version + if not _openvino_tokenizers_available: + logger.warning( + "OpenVINO Tokenizer version is not compatible with OpenVINO version. " + f"Installed OpenVINO version: {ov_major_version}," + f"OpenVINO Tokenizers requires {_compatible_openvino_version}. " + f"OpenVINO Tokenizers models will not be added during export." + ) _nncf_available = importlib.util.find_spec("nncf") is not None _nncf_version = "N/A" @@ -91,16 +119,6 @@ except importlib_metadata.PackageNotFoundError: _nncf_available = False - -_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None -_openvino_tokenizers_version = "N/A" -if _openvino_tokenizers_available: - try: - _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers") - except importlib_metadata.PackageNotFoundError: - _openvino_tokenizers_available = False - - _diffusers_available = importlib.util.find_spec("diffusers") is not None _diffusers_version = "N/A" if _diffusers_available: From 32a7274ba2cb02138b5185b6501f6c1fbb1d345e Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Tue, 30 Jan 2024 16:48:43 +0000 Subject: [PATCH 19/22] Bump OV Version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c942ff9246..68919a6101 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ "transformers>=4.34.0", ], "openvino": [ - "openvino>=2023.2", + "openvino>=2023.3", "onnx", "onnxruntime", "transformers>=4.36.0", From 8c029e02c28236aae09ef7adc649553995373ef4 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 1 Feb 2024 11:46:42 +0000 Subject: [PATCH 20/22] Move OpenVINO Tokenizers To Optional Dependencies --- .github/workflows/test_openvino.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_openvino.yml b/.github/workflows/test_openvino.yml index deea61e913..bf9460c75a 100644 --- a/.github/workflows/test_openvino.yml +++ b/.github/workflows/test_openvino.yml @@ -32,7 +32,7 @@ jobs: python -m pip install --upgrade pip # install PyTorch CPU version to avoid installing CUDA packages on GitHub runner without GPU pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu - pip install .[openvino,nncf,tests,diffusers] + pip install .[openvino,openvino-tokenizers,nncf,tests,diffusers] - name: Test with Pytest run: | pytest tests/openvino/ --ignore test_modeling_basic diff --git a/setup.py b/setup.py index 68919a6101..7223e731c8 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,8 @@ "onnxruntime", "transformers>=4.36.0", "optimum>=1.16.1", - "openvino-tokenizers[transformers]", ], + "openvino-tokenizers": ["openvino-tokenizers[transformers]"], "nncf": ["nncf>=2.7.0"], "ipex": ["intel-extension-for-pytorch", "onnx"], "diffusers": ["diffusers"], From 09b067fa1245cc12ce7117174fb9c8a114118380 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 8 Feb 2024 14:12:13 +0000 Subject: [PATCH 21/22] Add --convert-tokenizer Option to CLI --- optimum/commands/export/openvino.py | 6 ++++++ optimum/exporters/openvino/__main__.py | 7 ++++--- tests/openvino/test_exporters_cli.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index c2f8f85758..0f1f71d252 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -103,6 +103,11 @@ def parse_args_openvino(parser: "ArgumentParser"): "OpenVINO native inference code that expects kv-cache inputs and outputs in the model." ), ) + optional_group.add_argument( + "--convert-tokenizer", + action="store_true", + help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers", + ) class OVExportCommand(BaseOptimumCLICommand): @@ -151,5 +156,6 @@ def run(self): compression_option=self.args.weight_format, compression_ratio=self.args.ratio, stateful=not self.args.disable_stateful, + convert_tokenizer=self.args.convert_tokenizer, # **input_shapes, ) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 72124b220d..717b850d3e 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -71,6 +71,7 @@ def main_export( compression_option: Optional[str] = None, compression_ratio: Optional[float] = None, stateful: bool = True, + convert_tokenizer: bool = False, **kwargs_shapes, ): """ @@ -344,7 +345,7 @@ class StoreAttr(object): generation_config.save_pretrained(output) maybe_save_preprocessors(model_name_or_path, output) - if tokenizer is not None and is_openvino_tokenizers_available(): + if convert_tokenizer and tokenizer is not None and is_openvino_tokenizers_available(): try: export_tokenizer(tokenizer, output) except Exception as exception: @@ -380,12 +381,12 @@ class StoreAttr(object): feature_extractor.save_pretrained(output.joinpath("feature_extractor")) tokenizer = getattr(model, "tokenizer", None) - if tokenizer is not None and is_openvino_tokenizers_available(): + if convert_tokenizer and tokenizer is not None and is_openvino_tokenizers_available(): tokenizer.save_pretrained(output.joinpath("tokenizer")) export_tokenizer(tokenizer, output) tokenizer_2 = getattr(model, "tokenizer_2", None) - if tokenizer_2 is not None and is_openvino_tokenizers_available(): + if convert_tokenizer and tokenizer_2 is not None and is_openvino_tokenizers_available(): tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) export_tokenizer(tokenizer, output, suffix="_2") diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 822a47a6f6..46c6e3c69a 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -122,7 +122,7 @@ def test_exporters_cli(self, task: str, model_type: str): def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: output = subprocess.check_output( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}", shell=True, stderr=subprocess.STDOUT, ).decode() From 3c27fbddd551ce741548b39486b6ca6ef71529f8 Mon Sep 17 00:00:00 2001 From: Artur Paniukov Date: Thu, 8 Feb 2024 16:59:40 +0000 Subject: [PATCH 22/22] Fix SD Tokenizer --- optimum/exporters/openvino/__main__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 717b850d3e..96d57ff3b1 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -381,14 +381,16 @@ class StoreAttr(object): feature_extractor.save_pretrained(output.joinpath("feature_extractor")) tokenizer = getattr(model, "tokenizer", None) - if convert_tokenizer and tokenizer is not None and is_openvino_tokenizers_available(): + if tokenizer is not None: tokenizer.save_pretrained(output.joinpath("tokenizer")) - export_tokenizer(tokenizer, output) + if convert_tokenizer and is_openvino_tokenizers_available(): + export_tokenizer(tokenizer, output) tokenizer_2 = getattr(model, "tokenizer_2", None) - if convert_tokenizer and tokenizer_2 is not None and is_openvino_tokenizers_available(): + if tokenizer_2 is not None: tokenizer_2.save_pretrained(output.joinpath("tokenizer_2")) - export_tokenizer(tokenizer, output, suffix="_2") + if convert_tokenizer and is_openvino_tokenizers_available(): + export_tokenizer(tokenizer, output, suffix="_2") model.save_config(output)