diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py index 40901fbf90..3015d7b5b5 100644 --- a/optimum/commands/export/openvino.py +++ b/optimum/commands/export/openvino.py @@ -126,10 +126,15 @@ def parse_args_openvino(parser: "ArgumentParser"): "OpenVINO native inference code that expects kv-cache inputs and outputs in the model." ), ) + optional_group.add_argument( + "--disable-convert-tokenizer", + action="store_true", + help="Do not add converted tokenizer and detokenizer OpenVINO models.", + ) optional_group.add_argument( "--convert-tokenizer", action="store_true", - help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers", + help="[Deprecated] Add converted tokenizer and detokenizer with OpenVINO Tokenizers.", ) optional_group.add_argument( @@ -247,6 +252,9 @@ def run(self): model.save_pretrained(self.args.output) else: + if self.args.convert_tokenizer: + logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.") + # TODO : add input shapes main_export( model_name_or_path=self.args.model, @@ -258,7 +266,7 @@ def run(self): pad_token_id=self.args.pad_token_id, ov_config=ov_config, stateful=not self.args.disable_stateful, - convert_tokenizer=self.args.convert_tokenizer, + convert_tokenizer=not self.args.disable_convert_tokenizer, library_name=library_name, # **input_shapes, ) diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py index 5f74c1de8b..d7b29584d6 100644 --- a/optimum/exporters/openvino/__main__.py +++ b/optimum/exporters/openvino/__main__.py @@ -22,11 +22,10 @@ from optimum.exporters import TasksManager from optimum.exporters.onnx.base import OnnxConfig from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED +from optimum.exporters.openvino.convert import export_from_model, export_tokenizer +from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version from optimum.utils.save_utils import maybe_load_preprocessors -from ...intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version -from .convert import export_from_model, export_tokenizer - if TYPE_CHECKING: from optimum.intel.openvino.configuration import OVConfig @@ -187,12 +186,6 @@ def main_export( f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}" ) - if convert_tokenizer and not is_openvino_tokenizers_available(): - logger.warning( - "`convert_tokenizer` requires openvino-tokenizers, please install it with `pip install optimum-intel[openvino-tokenizers]`" - ) - convert_tokenizer = False - do_gptq_patching = False custom_architecture = False loading_kwargs = {} @@ -348,7 +341,7 @@ class StoreAttr(object): **kwargs_shapes, ) - if convert_tokenizer: + if convert_tokenizer and is_openvino_tokenizers_available(): if library_name != "diffusers": tokenizer = next( (preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)), @@ -371,6 +364,8 @@ class StoreAttr(object): tokenizer_2 = getattr(model, "tokenizer_2", None) if tokenizer_2 is not None: export_tokenizer(tokenizer_2, output, suffix="_2") + elif convert_tokenizer and not is_openvino_tokenizers_available(): + logger.warning("Tokenizer won't be converted.") # Unpatch modules after GPTQ export if do_gptq_patching: diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py index 5dd7c7bd90..55e3318017 100644 --- a/optimum/exporters/openvino/convert.py +++ b/optimum/exporters/openvino/convert.py @@ -20,7 +20,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union -from transformers import T5Tokenizer, T5TokenizerFast from transformers.utils import is_tf_available, is_torch_available from openvino.runtime import PartialShape, save_model @@ -49,9 +48,6 @@ ) -UNSUPPORTED_TOKENIZER_CLASSES = (T5Tokenizer, T5TokenizerFast) - - logger = logging.getLogger(__name__) if is_torch_available(): @@ -662,10 +658,6 @@ def export_tokenizer( ): from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME # avoid circular imports - if isinstance(tokenizer, UNSUPPORTED_TOKENIZER_CLASSES): - logger.info(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.") - return - try: from openvino_tokenizers import convert_tokenizer except ModuleNotFoundError: @@ -681,13 +673,13 @@ def export_tokenizer( try: converted = convert_tokenizer(tokenizer, with_detokenizer=True) except NotImplementedError: - logger.warning("Detokenizer is not supported, convert tokenizer only.") + logger.info("Detokenizer is not supported, convert tokenizer only.") converted = convert_tokenizer(tokenizer, with_detokenizer=False) except OVTypeError: - logger.warning(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.") + logger.debug(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.") return except Exception as exception: - logger.warning( + logger.debug( f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported. Exception: {exception}" ) return diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py index 08a9ec1f88..fcdf932a28 100644 --- a/optimum/intel/utils/import_utils.py +++ b/optimum/intel/utils/import_utils.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import functools import importlib.util import logging import operator as op @@ -95,32 +95,6 @@ except ImportError: _openvino_available = False -_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available -_openvino_tokenizers_version = "N/A" -if _openvino_tokenizers_available: - try: - _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers") - except importlib_metadata.PackageNotFoundError: - _openvino_tokenizers_available = False - -if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A": - _compatible_openvino_version = next( - ( - requirement.split("==")[-1] - for requirement in importlib_metadata.requires("openvino-tokenizers") - if requirement.startswith("openvino==") - ), - "", - ) - _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version - if not _openvino_tokenizers_available: - logger.warning( - "OpenVINO Tokenizer version is not compatible with OpenVINO version. " - f"Installed OpenVINO version: {ov_major_version}," - f"OpenVINO Tokenizers requires {_compatible_openvino_version}. " - f"OpenVINO Tokenizers models will not be added during export." - ) - _nncf_available = importlib.util.find_spec("nncf") is not None _nncf_version = "N/A" if _nncf_available: @@ -196,8 +170,81 @@ def is_openvino_available(): return _openvino_available +@functools.lru_cache(1) def is_openvino_tokenizers_available(): - return _openvino_tokenizers_available + if not is_openvino_available(): + return False + + if importlib.util.find_spec("openvino_tokenizers") is None: + logger.info( + "OpenVINO Tokenizers is not available. To deploy models in production " + "with C++ code, please follow installation instructions: " + "https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n" + ) + return False + + try: + pip_metadata_version = importlib_metadata.version("openvino") + except importlib_metadata.PackageNotFoundError: + pip_metadata_version = False + try: + pip_metadata_version = importlib_metadata.version("openvino-nightly") + is_nightly = True + except importlib_metadata.PackageNotFoundError: + is_nightly = False + + try: + import openvino_tokenizers + + openvino_tokenizers._get_factory() + except RuntimeError: + tokenizers_version = openvino_tokenizers.__version__ + + if tokenizers_version == "0.0.0.0": + try: + tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version + except importlib_metadata.PackageNotFoundError: + pass + message = ( + "OpenVINO and OpenVINO Tokenizers versions are not binary compatible.\n" + f"OpenVINO version: {_openvino_version}\n" + f"OpenVINO Tokenizers version: {tokenizers_version}\n" + "First 3 numbers should be the same. Update OpenVINO Tokenizers to compatible version. " + ) + if not pip_metadata_version: + message += ( + "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: " + "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file" + "#build-and-install-from-source" + ) + if sys.platform == "linux": + message += ( + "\nThe PyPI version of OpenVINO Tokenizers is built on CentOS and may not be compatible with other " + "Linux distributions; rebuild OpenVINO Tokenizers from source." + ) + else: + message += ( + "It is recommended to use the same day builds for pre-release version. " + "To install both OpenVINO and OpenVINO Tokenizers release version perform:\n" + ) + if is_nightly: + message += "pip uninstall -y openvino-nightly && " + message += "pip install --force-reinstall openvino openvino-tokenizers\n" + if is_nightly: + message += ( + "openvino-nightly package will be deprecated in the future - use pre-release drops instead. " + ) + message += "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n" + if is_nightly: + message += "pip uninstall -y openvino-nightly && " + message += ( + "pip install --pre -U openvino openvino-tokenizers " + "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly" + ) + logger.warning(message) + return False + + return True def is_nncf_available(): diff --git a/setup.py b/setup.py index 3978fd1fd6..0c794aaeb5 100644 --- a/setup.py +++ b/setup.py @@ -58,13 +58,8 @@ QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"] EXTRAS_REQUIRE = { - "neural-compressor": [ - "neural-compressor>=2.2.0", - "onnxruntime<1.15.0", - "accelerate", - ], - "openvino": ["openvino>=2023.3", "nncf>=2.8.1"], - "openvino-tokenizers": ["openvino-tokenizers[transformers]"], + "neural-compressor": ["neural-compressor>=2.2.0", "onnxruntime<1.15.0", "accelerate"], + "openvino": ["openvino>=2023.3", "nncf>=2.8.1", "openvino-tokenizers[transformers]"], "nncf": ["nncf>=2.8.1"], "ipex": ["intel-extension-for-pytorch", "transformers>=4.36.0,<4.39.0"], "diffusers": ["diffusers"], diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py index 7d618c530e..6e1c7a56bd 100644 --- a/tests/openvino/test_exporters_cli.py +++ b/tests/openvino/test_exporters_cli.py @@ -66,7 +66,7 @@ class OVCLIExportTestCase(unittest.TestCase): ) EXPECTED_NUMBER_OF_TOKENIZER_MODELS = { "gpt2": 2, - "t5": 0, # failed internal sentencepiece check - no token in the vocab + "t5": 0, # no .model file in the repository "albert": 0, # not supported yet "distilbert": 1, # no detokenizer "roberta": 2, @@ -125,26 +125,26 @@ def test_exporters_cli(self, task: str, model_type: str): for arch in SUPPORTED_ARCHITECTURES if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner") ) - @unittest.skipIf(not is_openvino_tokenizers_available(), reason="OpenVINO Tokenizers not available") def test_exporters_cli_tokenizers(self, task: str, model_type: str): with TemporaryDirectory() as tmpdir: output = subprocess.check_output( - f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}", + f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}", shell=True, stderr=subprocess.STDOUT, ).decode() - save_dir = Path(tmpdir) - number_of_tokenizers = sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml"))) - self.assertEqual( - self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], - number_of_tokenizers, - f"OVT: {is_openvino_tokenizers_available() }", - ) + if not is_openvino_tokenizers_available(): + self.assertTrue( + "OpenVINO Tokenizers is not available." in output + or "OpenVINO and OpenVINO Tokenizers versions are not binary compatible." in output, + msg=output, + ) + return + + number_of_tokenizers = sum("tokenizer" in file for file in map(str, Path(tmpdir).rglob("*.xml"))) + self.assertEqual(self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], number_of_tokenizers, output) if number_of_tokenizers == 1: self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output) - elif number_of_tokenizers == 0 and task not in ("image-classification", "audio-classification"): - self.assertTrue(("OpenVINO Tokenizer export for" in output and "is not supported." in output), output) @parameterized.expand(SUPPORTED_ARCHITECTURES) def test_exporters_cli_fp16(self, task: str, model_type: str):