Check OV Compatibility

Author: apaniukov

optimum/exporters/openvino/__main__.py

@@ -26,7 +26,12 @@
 from optimum.utils import DEFAULT_DUMMY_SHAPES
 from optimum.utils.save_utils import maybe_load_preprocessors, maybe_save_preprocessors
 
-from ...intel.utils.import_utils import is_nncf_available, is_optimum_version, is_transformers_version
+from ...intel.utils.import_utils import (
+    is_nncf_available,
+    is_openvino_tokenizers_available,
+    is_optimum_version,
+    is_transformers_version,
+)
 from .convert import export_models, export_tokenizer
 from .stateful import ensure_export_task_support_stateful
 
@@ -339,7 +344,7 @@ class StoreAttr(object):
             generation_config.save_pretrained(output)
         maybe_save_preprocessors(model_name_or_path, output)
 
-        if tokenizer is not None:
+        if tokenizer is not None and is_openvino_tokenizers_available():
             try:
                 export_tokenizer(tokenizer, output)
             except Exception as exception:
@@ -375,12 +380,12 @@ class StoreAttr(object):
             feature_extractor.save_pretrained(output.joinpath("feature_extractor"))
 
         tokenizer = getattr(model, "tokenizer", None)
-        if tokenizer is not None:
+        if tokenizer is not None and is_openvino_tokenizers_available():
             tokenizer.save_pretrained(output.joinpath("tokenizer"))
             export_tokenizer(tokenizer, output)
 
         tokenizer_2 = getattr(model, "tokenizer_2", None)
-        if tokenizer_2 is not None:
+        if tokenizer_2 is not None and is_openvino_tokenizers_available():
             tokenizer_2.save_pretrained(output.joinpath("tokenizer_2"))
             export_tokenizer(tokenizer, output, suffix="_2")
 

optimum/intel/utils/import_utils.py

@@ -13,6 +13,7 @@
 #  limitations under the License.
 
 import importlib.util
+import logging
 import operator as op
 import sys
 from collections import OrderedDict
@@ -27,6 +28,8 @@
     import importlib.metadata as importlib_metadata
 
 
+logger = logging.getLogger(__name__)
+
 STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt}
 
 _optimum_version = importlib_metadata.version("optimum")
@@ -75,13 +78,38 @@
         version = get_version()
         # avoid invalid format
         if "-" in version:
-            major_version, dev_info = version.split("-", 1)
+            ov_major_version, dev_info = version.split("-", 1)
             commit_id = dev_info.split("-")[0]
-            version = f"{major_version}-{commit_id}"
+            version = f"{ov_major_version}-{commit_id}"
         _openvino_version = version
     except ImportError:
         _openvino_available = False
 
+_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available
+_openvino_tokenizers_version = "N/A"
+if _openvino_tokenizers_available:
+    try:
+        _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
+    except importlib_metadata.PackageNotFoundError:
+        _openvino_tokenizers_available = False
+
+if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
+    _compatible_openvino_version = next(
+        (
+            requirement.split("==")[-1]
+            for requirement in importlib_metadata.requires("openvino-tokenizers")
+            if requirement.startswith("openvino==")
+        ),
+        "",
+    )
+    _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version
+    if not _openvino_tokenizers_available:
+        logger.warning(
+            "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
+            f"Installed OpenVINO version: {ov_major_version},"
+            f"OpenVINO Tokenizers requires {_compatible_openvino_version}. "
+            f"OpenVINO Tokenizers models will not be added during export."
+        )
 
 _nncf_available = importlib.util.find_spec("nncf") is not None
 _nncf_version = "N/A"
@@ -91,16 +119,6 @@
     except importlib_metadata.PackageNotFoundError:
         _nncf_available = False
 
-
-_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None
-_openvino_tokenizers_version = "N/A"
-if _openvino_tokenizers_available:
-    try:
-        _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
-    except importlib_metadata.PackageNotFoundError:
-        _openvino_tokenizers_available = False
-
-
 _diffusers_available = importlib.util.find_spec("diffusers") is not None
 _diffusers_version = "N/A"
 if _diffusers_available: