diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
index 40901fbf90..3015d7b5b5 100644
--- a/optimum/commands/export/openvino.py
+++ b/optimum/commands/export/openvino.py
@@ -126,10 +126,15 @@ def parse_args_openvino(parser: "ArgumentParser"):
"OpenVINO native inference code that expects kv-cache inputs and outputs in the model."
),
)
+ optional_group.add_argument(
+ "--disable-convert-tokenizer",
+ action="store_true",
+ help="Do not add converted tokenizer and detokenizer OpenVINO models.",
+ )
optional_group.add_argument(
"--convert-tokenizer",
action="store_true",
- help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers",
+ help="[Deprecated] Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
)
optional_group.add_argument(
@@ -247,6 +252,9 @@ def run(self):
model.save_pretrained(self.args.output)
else:
+ if self.args.convert_tokenizer:
+ logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")
+
# TODO : add input shapes
main_export(
model_name_or_path=self.args.model,
@@ -258,7 +266,7 @@ def run(self):
pad_token_id=self.args.pad_token_id,
ov_config=ov_config,
stateful=not self.args.disable_stateful,
- convert_tokenizer=self.args.convert_tokenizer,
+ convert_tokenizer=not self.args.disable_convert_tokenizer,
library_name=library_name,
# **input_shapes,
)
diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index 5f74c1de8b..d7b29584d6 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -22,11 +22,10 @@
from optimum.exporters import TasksManager
from optimum.exporters.onnx.base import OnnxConfig
from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED
+from optimum.exporters.openvino.convert import export_from_model, export_tokenizer
+from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
from optimum.utils.save_utils import maybe_load_preprocessors
-from ...intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
-from .convert import export_from_model, export_tokenizer
-
if TYPE_CHECKING:
from optimum.intel.openvino.configuration import OVConfig
@@ -187,12 +186,6 @@ def main_export(
f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}"
)
- if convert_tokenizer and not is_openvino_tokenizers_available():
- logger.warning(
- "`convert_tokenizer` requires openvino-tokenizers, please install it with `pip install optimum-intel[openvino-tokenizers]`"
- )
- convert_tokenizer = False
-
do_gptq_patching = False
custom_architecture = False
loading_kwargs = {}
@@ -348,7 +341,7 @@ class StoreAttr(object):
**kwargs_shapes,
)
- if convert_tokenizer:
+ if convert_tokenizer and is_openvino_tokenizers_available():
if library_name != "diffusers":
tokenizer = next(
(preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)),
@@ -371,6 +364,8 @@ class StoreAttr(object):
tokenizer_2 = getattr(model, "tokenizer_2", None)
if tokenizer_2 is not None:
export_tokenizer(tokenizer_2, output, suffix="_2")
+ elif convert_tokenizer and not is_openvino_tokenizers_available():
+ logger.warning("Tokenizer won't be converted.")
# Unpatch modules after GPTQ export
if do_gptq_patching:
diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py
index 5dd7c7bd90..55e3318017 100644
--- a/optimum/exporters/openvino/convert.py
+++ b/optimum/exporters/openvino/convert.py
@@ -20,7 +20,6 @@
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
-from transformers import T5Tokenizer, T5TokenizerFast
from transformers.utils import is_tf_available, is_torch_available
from openvino.runtime import PartialShape, save_model
@@ -49,9 +48,6 @@
)
-UNSUPPORTED_TOKENIZER_CLASSES = (T5Tokenizer, T5TokenizerFast)
-
-
logger = logging.getLogger(__name__)
if is_torch_available():
@@ -662,10 +658,6 @@ def export_tokenizer(
):
from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME # avoid circular imports
- if isinstance(tokenizer, UNSUPPORTED_TOKENIZER_CLASSES):
- logger.info(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
- return
-
try:
from openvino_tokenizers import convert_tokenizer
except ModuleNotFoundError:
@@ -681,13 +673,13 @@ def export_tokenizer(
try:
converted = convert_tokenizer(tokenizer, with_detokenizer=True)
except NotImplementedError:
- logger.warning("Detokenizer is not supported, convert tokenizer only.")
+ logger.info("Detokenizer is not supported, convert tokenizer only.")
converted = convert_tokenizer(tokenizer, with_detokenizer=False)
except OVTypeError:
- logger.warning(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
+ logger.debug(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
return
except Exception as exception:
- logger.warning(
+ logger.debug(
f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported. Exception: {exception}"
)
return
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 08a9ec1f88..fcdf932a28 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+import functools
import importlib.util
import logging
import operator as op
@@ -95,32 +95,6 @@
except ImportError:
_openvino_available = False
-_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available
-_openvino_tokenizers_version = "N/A"
-if _openvino_tokenizers_available:
- try:
- _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
- except importlib_metadata.PackageNotFoundError:
- _openvino_tokenizers_available = False
-
-if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
- _compatible_openvino_version = next(
- (
- requirement.split("==")[-1]
- for requirement in importlib_metadata.requires("openvino-tokenizers")
- if requirement.startswith("openvino==")
- ),
- "",
- )
- _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version
- if not _openvino_tokenizers_available:
- logger.warning(
- "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
- f"Installed OpenVINO version: {ov_major_version},"
- f"OpenVINO Tokenizers requires {_compatible_openvino_version}. "
- f"OpenVINO Tokenizers models will not be added during export."
- )
-
_nncf_available = importlib.util.find_spec("nncf") is not None
_nncf_version = "N/A"
if _nncf_available:
@@ -196,8 +170,81 @@ def is_openvino_available():
return _openvino_available
+@functools.lru_cache(1)
def is_openvino_tokenizers_available():
- return _openvino_tokenizers_available
+ if not is_openvino_available():
+ return False
+
+ if importlib.util.find_spec("openvino_tokenizers") is None:
+ logger.info(
+ "OpenVINO Tokenizers is not available. To deploy models in production "
+ "with C++ code, please follow installation instructions: "
+ "https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n"
+ )
+ return False
+
+ try:
+ pip_metadata_version = importlib_metadata.version("openvino")
+ except importlib_metadata.PackageNotFoundError:
+ pip_metadata_version = False
+ try:
+ pip_metadata_version = importlib_metadata.version("openvino-nightly")
+ is_nightly = True
+ except importlib_metadata.PackageNotFoundError:
+ is_nightly = False
+
+ try:
+ import openvino_tokenizers
+
+ openvino_tokenizers._get_factory()
+ except RuntimeError:
+ tokenizers_version = openvino_tokenizers.__version__
+
+ if tokenizers_version == "0.0.0.0":
+ try:
+ tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+ except importlib_metadata.PackageNotFoundError:
+ pass
+ message = (
+ "OpenVINO and OpenVINO Tokenizers versions are not binary compatible.\n"
+ f"OpenVINO version: {_openvino_version}\n"
+ f"OpenVINO Tokenizers version: {tokenizers_version}\n"
+ "First 3 numbers should be the same. Update OpenVINO Tokenizers to compatible version. "
+ )
+ if not pip_metadata_version:
+ message += (
+ "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: "
+ "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
+ "#build-and-install-from-source"
+ )
+ if sys.platform == "linux":
+ message += (
+ "\nThe PyPI version of OpenVINO Tokenizers is built on CentOS and may not be compatible with other "
+ "Linux distributions; rebuild OpenVINO Tokenizers from source."
+ )
+ else:
+ message += (
+ "It is recommended to use the same day builds for pre-release version. "
+ "To install both OpenVINO and OpenVINO Tokenizers release version perform:\n"
+ )
+ if is_nightly:
+ message += "pip uninstall -y openvino-nightly && "
+ message += "pip install --force-reinstall openvino openvino-tokenizers\n"
+ if is_nightly:
+ message += (
+ "openvino-nightly package will be deprecated in the future - use pre-release drops instead. "
+ )
+ message += "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n"
+ if is_nightly:
+ message += "pip uninstall -y openvino-nightly && "
+ message += (
+ "pip install --pre -U openvino openvino-tokenizers "
+ "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+ )
+ logger.warning(message)
+ return False
+
+ return True
def is_nncf_available():
diff --git a/setup.py b/setup.py
index 3978fd1fd6..0c794aaeb5 100644
--- a/setup.py
+++ b/setup.py
@@ -58,13 +58,8 @@
QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241"]
EXTRAS_REQUIRE = {
- "neural-compressor": [
- "neural-compressor>=2.2.0",
- "onnxruntime<1.15.0",
- "accelerate",
- ],
- "openvino": ["openvino>=2023.3", "nncf>=2.8.1"],
- "openvino-tokenizers": ["openvino-tokenizers[transformers]"],
+ "neural-compressor": ["neural-compressor>=2.2.0", "onnxruntime<1.15.0", "accelerate"],
+ "openvino": ["openvino>=2023.3", "nncf>=2.8.1", "openvino-tokenizers[transformers]"],
"nncf": ["nncf>=2.8.1"],
"ipex": ["intel-extension-for-pytorch", "transformers>=4.36.0,<4.39.0"],
"diffusers": ["diffusers"],
diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index 7d618c530e..6e1c7a56bd 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -66,7 +66,7 @@ class OVCLIExportTestCase(unittest.TestCase):
)
EXPECTED_NUMBER_OF_TOKENIZER_MODELS = {
"gpt2": 2,
- "t5": 0, # failed internal sentencepiece check - no <s> token in the vocab
+ "t5": 0, # no .model file in the repository
"albert": 0, # not supported yet
"distilbert": 1, # no detokenizer
"roberta": 2,
@@ -125,26 +125,26 @@ def test_exporters_cli(self, task: str, model_type: str):
for arch in SUPPORTED_ARCHITECTURES
if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner")
)
- @unittest.skipIf(not is_openvino_tokenizers_available(), reason="OpenVINO Tokenizers not available")
def test_exporters_cli_tokenizers(self, task: str, model_type: str):
with TemporaryDirectory() as tmpdir:
output = subprocess.check_output(
- f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}",
+ f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
shell=True,
stderr=subprocess.STDOUT,
).decode()
- save_dir = Path(tmpdir)
- number_of_tokenizers = sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml")))
- self.assertEqual(
- self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type],
- number_of_tokenizers,
- f"OVT: {is_openvino_tokenizers_available() }",
- )
+ if not is_openvino_tokenizers_available():
+ self.assertTrue(
+ "OpenVINO Tokenizers is not available." in output
+ or "OpenVINO and OpenVINO Tokenizers versions are not binary compatible." in output,
+ msg=output,
+ )
+ return
+
+ number_of_tokenizers = sum("tokenizer" in file for file in map(str, Path(tmpdir).rglob("*.xml")))
+ self.assertEqual(self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], number_of_tokenizers, output)
if number_of_tokenizers == 1:
self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
- elif number_of_tokenizers == 0 and task not in ("image-classification", "audio-classification"):
- self.assertTrue(("OpenVINO Tokenizer export for" in output and "is not supported." in output), output)
@parameterized.expand(SUPPORTED_ARCHITECTURES)
def test_exporters_cli_fp16(self, task: str, model_type: str):