Convert Tokenizers By Default

optimum/commands/export/openvino.py

@@ -115,10 +115,15 @@ def parse_args_openvino(parser: "ArgumentParser"):
             "OpenVINO native inference code that expects kv-cache inputs and outputs in the model."
         ),
     )
+    optional_group.add_argument(
+        "--disable-convert-tokenizer",
+        action="store_true",
+        help="Do not add converted tokenizer and detokenizer OpenVINO models.",
+    )
     optional_group.add_argument(
         "--convert-tokenizer",
         action="store_true",
-        help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers",
+        help="[Deprecated] Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
     )
 
 
@@ -189,6 +194,9 @@ def run(self):
                 quantization_config["group_size"] = 128 if "128" in self.args.weight_format else 64
             ov_config = OVConfig(quantization_config=quantization_config)
 
+        if self.args.convert_tokenizer:
+            logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")
+
         # TODO : add input shapes
         main_export(
             model_name_or_path=self.args.model,
@@ -200,6 +208,6 @@ def run(self):
             pad_token_id=self.args.pad_token_id,
             ov_config=ov_config,
             stateful=not self.args.disable_stateful,
-            convert_tokenizer=self.args.convert_tokenizer,
+            convert_tokenizer=not self.args.disable_convert_tokenizer,
             # **input_shapes,
         )

optimum/exporters/openvino/__main__.py

@@ -22,11 +22,10 @@
 from optimum.exporters import TasksManager
 from optimum.exporters.onnx.base import OnnxConfig
 from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED
+from optimum.exporters.openvino.convert import export_from_model, export_tokenizer
+from optimum.intel.utils.import_utils import is_transformers_version
 from optimum.utils.save_utils import maybe_load_preprocessors
 
-from ...intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
-from .convert import export_from_model, export_tokenizer
-
 
 if TYPE_CHECKING:
     from optimum.intel.openvino.configuration import OVConfig
@@ -179,12 +178,6 @@ def main_export(
                 f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}"
             )
 
-    if convert_tokenizer and not is_openvino_tokenizers_available():
-        logger.warning(
-            "`convert_tokenizer` requires openvino-tokenizers, please install it with `pip install optimum-intel[openvino-tokenizers]`"
-        )
-        convert_tokenizer = False
-
     do_gptq_patching = False
     custom_architecture = False
     loading_kwargs = {}

optimum/intel/utils/import_utils.py

@@ -94,21 +94,43 @@
         _openvino_tokenizers_available = False
 
 if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
-    _compatible_openvino_version = next(
-        (
-            requirement.split("==")[-1]
-            for requirement in importlib_metadata.requires("openvino-tokenizers")
-            if requirement.startswith("openvino==")
-        ),
-        "",
-    )
-    _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version
+    _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
+    _ov_version = importlib_metadata.version("openvino")
+    _is_ov_dev_version = "dev" in _ov_version
+    if _is_ovt_dev_version:
+        _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
+        _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
+        _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
+    else:
+        _compatible_ov_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
+        _compatible_ovt_version = _ov_version + ".0"
+
+    _openvino_tokenizers_available = _ov_version == _compatible_ov_version
+
     if not _openvino_tokenizers_available:
+        _update_ov_command = (
+            f"pip install {'--pre' if _is_ovt_dev_version else ''} -U openvino=={_compatible_ov_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ovt_dev_version
+                else ""
+            )
+        ).strip()
+        _update_ovt_command = (
+            f"pip install {'--pre' if _is_ov_dev_version else ''} -U openvino-tokenizers=={_compatible_ovt_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ov_dev_version
+                else ""
+            )
+        ).strip()
         logger.warning(
             "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
-            f"Installed OpenVINO version: {ov_major_version},"
-            f"OpenVINO Tokenizers requires {_compatible_openvino_version}. "
-            f"OpenVINO Tokenizers models will not be added during export."
+            f"Installed OpenVINO version: {_ov_version}, "
+            f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
+            "OpenVINO Tokenizers models will not be added during export. "
+            f"Update OpenVINO with \n{_update_ov_command}\n"
+            f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
         )
 
 _nncf_available = importlib.util.find_spec("nncf") is not None

setup.py

@@ -41,8 +41,7 @@
 
 EXTRAS_REQUIRE = {
     "neural-compressor": ["neural-compressor>=2.2.0", "onnxruntime<1.15.0", "accelerate"],
-    "openvino": ["openvino>=2023.3", "nncf>=2.8.1"],
-    "openvino-tokenizers": ["openvino-tokenizers[transformers]"],
+    "openvino": ["openvino>=2023.3", "nncf>=2.8.1", "openvino-tokenizers[transformers]"],
     "nncf": ["nncf>=2.8.1"],
     "ipex": ["intel-extension-for-pytorch"],
     "diffusers": ["diffusers"],

tests/openvino/test_exporters_cli.py

@@ -122,7 +122,7 @@ def test_exporters_cli(self, task: str, model_type: str):
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
-                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}",
+                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()