From 6d72fe0527b4f71f7f4dd6451ccdfaaad117f902 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 29 Feb 2024 12:49:45 +0000
Subject: [PATCH 01/19] Convert Tokenizers By Default

---
 optimum/commands/export/openvino.py  | 6 +++---
 tests/openvino/test_exporters_cli.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
index 255e2a7e13..b53c20bf4f 100644
--- a/optimum/commands/export/openvino.py
+++ b/optimum/commands/export/openvino.py
@@ -116,9 +116,9 @@ def parse_args_openvino(parser: "ArgumentParser"):
         ),
     )
     optional_group.add_argument(
-        "--convert-tokenizer",
+        "--disable-convert-tokenizer",
         action="store_true",
-        help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers",
+        help="Do not add converted tokenizer and detokenizer OpenVINO models.",
     )
 
 
@@ -201,6 +201,6 @@ def run(self):
             pad_token_id=self.args.pad_token_id,
             ov_config=ov_config,
             stateful=not self.args.disable_stateful,
-            convert_tokenizer=self.args.convert_tokenizer,
+            convert_tokenizer=not self.args.disable_convert_tokenizer,
             # **input_shapes,
         )
diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index 46c6e3c69a..822a47a6f6 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -122,7 +122,7 @@ def test_exporters_cli(self, task: str, model_type: str):
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
-                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}",
+                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()

From ffcae81522d5a1b0cb4c9a1e5d5022a49bc9b02f Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 29 Feb 2024 13:40:41 +0000
Subject: [PATCH 02/19] Add Warning to Deprecated Option

---
 optimum/commands/export/openvino.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
index b53c20bf4f..cea7ab6c4f 100644
--- a/optimum/commands/export/openvino.py
+++ b/optimum/commands/export/openvino.py
@@ -120,6 +120,11 @@ def parse_args_openvino(parser: "ArgumentParser"):
         action="store_true",
         help="Do not add converted tokenizer and detokenizer OpenVINO models.",
     )
+    optional_group.add_argument(
+        "--convert-tokenizer",
+        action="store_true",
+        help="[Deprecated]  Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
+    )
 
 
 class OVExportCommand(BaseOptimumCLICommand):
@@ -190,6 +195,9 @@ def run(self):
                 quantization_config["group_size"] = 128 if "128" in weight_format else 64
             ov_config = OVConfig(quantization_config=quantization_config)
 
+        if self.args.convert_tokenizer:
+            logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")
+
         # TODO : add input shapes
         main_export(
             model_name_or_path=self.args.model,

From eadb2106264c336a990596b198934cc39adf4370 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Fri, 8 Mar 2024 15:41:34 +0000
Subject: [PATCH 03/19] Update OV Tokenizers Availability Check

---
 optimum/commands/export/openvino.py    |  2 +-
 optimum/exporters/openvino/__main__.py |  6 ----
 optimum/intel/utils/import_utils.py    | 46 +++++++++++++++++++-------
 3 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
index cea7ab6c4f..010c47218a 100644
--- a/optimum/commands/export/openvino.py
+++ b/optimum/commands/export/openvino.py
@@ -123,7 +123,7 @@ def parse_args_openvino(parser: "ArgumentParser"):
     optional_group.add_argument(
         "--convert-tokenizer",
         action="store_true",
-        help="[Deprecated]  Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
+        help="[Deprecated] Add converted tokenizer and detokenizer with OpenVINO Tokenizers.",
     )
 
 
diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index 24b65f9032..c153ce1da9 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -192,12 +192,6 @@ def main_export(
                 f"The task could not be automatically inferred as this is available only for models hosted on the Hugging Face Hub. Please provide the argument --task with the relevant task from {', '.join(TasksManager.get_all_tasks())}. Detailed error: {e}"
             )
 
-    if convert_tokenizer and not is_openvino_tokenizers_available():
-        logger.warning(
-            "`convert_tokenizer` requires openvino-tokenizers, please install it with `pip install optimum-intel[openvino-tokenizers]`"
-        )
-        convert_tokenizer = False
-
     do_gptq_patching = False
     custom_architecture = False
     loading_kwargs = {}
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index f1fb486c4f..9e0fd3f4c0 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -94,21 +94,43 @@
         _openvino_tokenizers_available = False
 
 if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
-    _compatible_openvino_version = next(
-        (
-            requirement.split("==")[-1]
-            for requirement in importlib_metadata.requires("openvino-tokenizers")
-            if requirement.startswith("openvino==")
-        ),
-        "",
-    )
-    _openvino_tokenizers_available = _compatible_openvino_version == ov_major_version
+    _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
+    _ov_version = importlib_metadata.version("openvino")
+    _is_ov_dev_version = "dev" in _ov_version
+    if _is_ovt_dev_version:
+        _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
+        _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
+        _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
+    else:
+        _compatible_ov_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
+        _compatible_ovt_version = _ov_version + ".0"
+
+    _openvino_tokenizers_available = _ov_version == _compatible_ov_version
+
     if not _openvino_tokenizers_available:
+        _update_ov_command = (
+            f"pip install {'--pre' if _is_ovt_dev_version else ''} -U openvino=={_compatible_ov_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ovt_dev_version
+                else ""
+            )
+        ).strip()
+        _update_ovt_command = (
+            f"pip install {'--pre' if _is_ov_dev_version else ''} -U openvino-tokenizers=={_compatible_ovt_version} "
+            + (
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if _is_ov_dev_version
+                else ""
+            )
+        ).strip()
         logger.warning(
             "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
-            f"Installed OpenVINO version: {ov_major_version},"
-            f"OpenVINO Tokenizers requires {_compatible_openvino_version}. "
-            f"OpenVINO Tokenizers models will not be added during export."
+            f"Installed OpenVINO version: {_ov_version}, "
+            f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
+            "OpenVINO Tokenizers models will not be added during export. "
+            f"Update OpenVINO with \n{_update_ov_command}\n"
+            f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
         )
 
 _nncf_available = importlib.util.find_spec("nncf") is not None

From 8a22002be6c417eff7a1e072e84d725eabfd29c9 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Wed, 13 Mar 2024 13:21:42 +0000
Subject: [PATCH 04/19] Move openvino-tokenizers to openvino dependencies

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 1701af990c..be4b633437 100644
--- a/setup.py
+++ b/setup.py
@@ -48,8 +48,8 @@
         "openvino>=2023.3",
         "onnx",
         "onnxruntime",
+        "openvino-tokenizers[transformers]"
     ],
-    "openvino-tokenizers": ["openvino-tokenizers[transformers]"],
     "nncf": ["nncf>=2.8.1"],
     "ipex": ["intel-extension-for-pytorch", "onnx"],
     "diffusers": ["diffusers"],

From 40f227dff9de1c1994d442786cd183773194b745 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Wed, 13 Mar 2024 13:23:33 +0000
Subject: [PATCH 05/19] Make Style

---
 optimum/exporters/openvino/__main__.py | 1 -
 setup.py                               | 7 +------
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index c153ce1da9..c3d84df043 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -24,7 +24,6 @@
 from optimum.utils.save_utils import maybe_load_preprocessors
 
 from ...intel.utils.import_utils import (
-    is_openvino_tokenizers_available,
     is_optimum_version,
     is_transformers_version,
 )
diff --git a/setup.py b/setup.py
index be4b633437..93c370d7c4 100644
--- a/setup.py
+++ b/setup.py
@@ -44,12 +44,7 @@
         "onnx",
         "onnxruntime<1.15.0",
     ],
-    "openvino": [
-        "openvino>=2023.3",
-        "onnx",
-        "onnxruntime",
-        "openvino-tokenizers[transformers]"
-    ],
+    "openvino": ["openvino>=2023.3", "onnx", "onnxruntime", "openvino-tokenizers[transformers]"],
     "nncf": ["nncf>=2.8.1"],
     "ipex": ["intel-extension-for-pytorch", "onnx"],
     "diffusers": ["diffusers"],

From 52e24b51a4aab7ba2e02bbaf08768d3f34d0c851 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Wed, 13 Mar 2024 13:33:45 +0000
Subject: [PATCH 06/19] Change Imports to Absolute

---
 optimum/exporters/openvino/__main__.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index e1f735c9a9..98b7a47254 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -22,13 +22,11 @@
 from optimum.exporters import TasksManager
 from optimum.exporters.onnx.base import OnnxConfig
 from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED
+from optimum.exporters.openvino.convert import export_from_model, export_tokenizer
+from optimum.intel.utils.import_utils import is_transformers_version
 from optimum.utils.save_utils import maybe_load_preprocessors
 
 
-from ...intel.utils.import_utils import is_transformers_version
-from .convert import export_from_model, export_tokenizer
-
-
 if TYPE_CHECKING:
     from optimum.intel.openvino.configuration import OVConfig
 

From fd1887fd07039f62e74031884a761a0589c3bcee Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 14 Mar 2024 16:07:26 +0000
Subject: [PATCH 07/19] Check openvino-nightly compatibility

Add check for installed openvino-nightly package as well.
Improve compatibility messages.
Check is OpenVINO Tokenizers is available only if tokenizer is exported.
---
 optimum/exporters/openvino/__main__.py |   4 +-
 optimum/exporters/openvino/convert.py  |   8 --
 optimum/intel/utils/import_utils.py    | 128 +++++++++++++++----------
 3 files changed, 81 insertions(+), 59 deletions(-)

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index 98b7a47254..0d3c9dfb08 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -23,7 +23,7 @@
 from optimum.exporters.onnx.base import OnnxConfig
 from optimum.exporters.onnx.constants import SDPA_ARCHS_ONNX_EXPORT_NOT_SUPPORTED
 from optimum.exporters.openvino.convert import export_from_model, export_tokenizer
-from optimum.intel.utils.import_utils import is_transformers_version
+from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
 from optimum.utils.save_utils import maybe_load_preprocessors
 
 
@@ -319,7 +319,7 @@ class StoreAttr(object):
         **kwargs_shapes,
     )
 
-    if convert_tokenizer:
+    if convert_tokenizer and is_openvino_tokenizers_available():
         if library_name != "diffusers":
             tokenizer = next(
                 (preprocessor for preprocessor in preprocessors if isinstance(preprocessor, PreTrainedTokenizerBase)),
diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py
index 5353912d48..c32566d604 100644
--- a/optimum/exporters/openvino/convert.py
+++ b/optimum/exporters/openvino/convert.py
@@ -20,7 +20,6 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
 
-from transformers import T5Tokenizer, T5TokenizerFast
 from transformers.utils import is_tf_available, is_torch_available
 
 from openvino.runtime import PartialShape, save_model
@@ -55,9 +54,6 @@
     from optimum.exporters.onnx.__main__ import _get_submodels_and_onnx_configs
 
 
-UNSUPPORTED_TOKENIZER_CLASSES = (T5Tokenizer, T5TokenizerFast)
-
-
 logger = logging.getLogger(__name__)
 
 if is_torch_available():
@@ -662,10 +658,6 @@ def export_tokenizer(
 ):
     from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME  # avoid circular imports
 
-    if isinstance(tokenizer, UNSUPPORTED_TOKENIZER_CLASSES):
-        logger.info(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
-        return
-
     try:
         from openvino_tokenizers import convert_tokenizer
     except ModuleNotFoundError:
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 3599104c75..5c2693699c 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-
+import functools
 import importlib.util
 import logging
 import operator as op
@@ -85,54 +85,6 @@
     except ImportError:
         _openvino_available = False
 
-_openvino_tokenizers_available = importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available
-_openvino_tokenizers_version = "N/A"
-if _openvino_tokenizers_available:
-    try:
-        _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
-    except importlib_metadata.PackageNotFoundError:
-        _openvino_tokenizers_available = False
-
-if _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
-    _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
-    _ov_version = importlib_metadata.version("openvino")
-    _is_ov_dev_version = "dev" in _ov_version
-    if _is_ovt_dev_version:
-        _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
-        _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
-        _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
-    else:
-        _compatible_ov_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
-        _compatible_ovt_version = _ov_version + ".0"
-
-    _openvino_tokenizers_available = _ov_version == _compatible_ov_version
-
-    if not _openvino_tokenizers_available:
-        _update_ov_command = (
-            f"pip install {'--pre' if _is_ovt_dev_version else ''} -U openvino=={_compatible_ov_version} "
-            + (
-                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
-                if _is_ovt_dev_version
-                else ""
-            )
-        ).strip()
-        _update_ovt_command = (
-            f"pip install {'--pre' if _is_ov_dev_version else ''} -U openvino-tokenizers=={_compatible_ovt_version} "
-            + (
-                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
-                if _is_ov_dev_version
-                else ""
-            )
-        ).strip()
-        logger.warning(
-            "OpenVINO Tokenizer version is not compatible with OpenVINO version. "
-            f"Installed OpenVINO version: {_ov_version}, "
-            f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
-            "OpenVINO Tokenizers models will not be added during export. "
-            f"Update OpenVINO with \n{_update_ov_command}\n"
-            f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
-        )
-
 _nncf_available = importlib.util.find_spec("nncf") is not None
 _nncf_version = "N/A"
 if _nncf_available:
@@ -204,7 +156,85 @@ def is_openvino_available():
     return _openvino_available
 
 
+def _create_pip_update_command(package_name: str, compatible_version: str, is_dev_version: bool = False) -> str:
+    return " ".join(
+        filter(
+            None,
+            (
+                "pip install",
+                "--pre" if is_dev_version else "",
+                "-U",
+                f"{package_name}=={compatible_version}",
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+                if is_dev_version
+                else "",
+            ),
+        )
+    )
+
+
+@functools.lru_cache(1)
 def is_openvino_tokenizers_available():
+    _openvino_tokenizers_available = (
+        importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available
+    )
+    _openvino_tokenizers_version = "N/A"
+    if _openvino_tokenizers_available:
+        try:
+            _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
+        except importlib_metadata.PackageNotFoundError:
+            return False
+
+    if _openvino_available and _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
+        _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
+        try:
+            _ov_version = importlib_metadata.version("openvino")
+        except importlib_metadata.PackageNotFoundError:
+            try:
+                _ov_version = importlib_metadata.version("openvino-nightly")
+            except importlib_metadata.PackageNotFoundError:
+                _ov_version = _openvino_version
+
+        _is_ov_dev_version = "dev" in _ov_version
+        if _is_ovt_dev_version:
+            _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
+            _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
+            _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
+        else:
+            _compatible_openvino_major_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
+            _compatible_ov_version = _compatible_openvino_major_version
+            if _is_ov_dev_version:
+                _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
+            else:
+                _compatible_ovt_version = _ov_version + ".0"
+
+        _update_ov_command = _create_pip_update_command(
+            "openvino", _compatible_ov_version, is_dev_version=_is_ovt_dev_version
+        )
+        _update_ovt_command = _create_pip_update_command(
+            "openvino-tokenizers", _compatible_ovt_version, is_dev_version=_is_ov_dev_version
+        )
+
+        _openvino_tokenizers_available = _ov_version.startswith(_compatible_openvino_major_version)
+
+        if not _openvino_tokenizers_available:
+            logger.warning(
+                "OpenVINO Tokenizers version is not compatible with OpenVINO version. "
+                f"Installed OpenVINO version: {_ov_version}, "
+                f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
+                "OpenVINO Tokenizers models will not be added during export. "
+                f"Update OpenVINO with \n{_update_ov_command}\n"
+                f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
+            )
+        elif _ov_version != _compatible_ov_version:
+            logger.info(
+                "OpenVINO Tokenizers version is not aligned with OpenVINO version. "
+                f"Installed OpenVINO version: {_ov_version}, "
+                f"OpenVINO Tokenizers version: {_compatible_ov_version}. "
+                "If you didn't get OpenVINO Tokenizer after model conversion "
+                f"try to align libraries versions with one of two commands:\n{_update_ov_command}\n"
+                f"Or\n{_update_ovt_command}"
+            )
     return _openvino_tokenizers_available
 
 

From 92d42f4b8149e17453290705e43368c00f7251c2 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 14 Mar 2024 16:09:08 +0000
Subject: [PATCH 08/19] Change model skip explanation

---
 tests/openvino/test_exporters_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index 822a47a6f6..e82ca8c0be 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -65,7 +65,7 @@ class OVCLIExportTestCase(unittest.TestCase):
     )
     EXPECTED_NUMBER_OF_TOKENIZER_MODELS = {
         "gpt2": 2,
-        "t5": 0,  # failed internal sentencepiece check - no <s> token in the vocab
+        "t5": 0,  # no .model file in the repository
         "albert": 0,  # not supported yet
         "distilbert": 1,  # no detokenizer
         "roberta": 2,

From 86a1ecabe676e56aef1acd9eb5950649ae29fcd0 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Fri, 22 Mar 2024 13:28:26 +0000
Subject: [PATCH 09/19] Update OV Tokenizers Availability Check

---
 optimum/intel/utils/import_utils.py | 97 ++++++++++++-----------------
 1 file changed, 39 insertions(+), 58 deletions(-)

diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 5c2693699c..30d4b8305f 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -175,67 +175,48 @@ def _create_pip_update_command(package_name: str, compatible_version: str, is_de
 
 @functools.lru_cache(1)
 def is_openvino_tokenizers_available():
-    _openvino_tokenizers_available = (
-        importlib.util.find_spec("openvino_tokenizers") is not None and _openvino_available
-    )
-    _openvino_tokenizers_version = "N/A"
-    if _openvino_tokenizers_available:
-        try:
-            _openvino_tokenizers_version = importlib_metadata.version("openvino_tokenizers")
-        except importlib_metadata.PackageNotFoundError:
-            return False
-
-    if _openvino_available and _openvino_tokenizers_available and _openvino_tokenizers_version != "N/A":
-        _is_ovt_dev_version = "dev" in _openvino_tokenizers_version
-        try:
-            _ov_version = importlib_metadata.version("openvino")
-        except importlib_metadata.PackageNotFoundError:
-            try:
-                _ov_version = importlib_metadata.version("openvino-nightly")
-            except importlib_metadata.PackageNotFoundError:
-                _ov_version = _openvino_version
-
-        _is_ov_dev_version = "dev" in _ov_version
-        if _is_ovt_dev_version:
-            _compatible_openvino_major_version, _, _dev_date = _openvino_tokenizers_version.rsplit(".", 2)
-            _compatible_ov_version = _compatible_openvino_major_version + "." + _dev_date
-            _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
-        else:
-            _compatible_openvino_major_version = _openvino_tokenizers_version.rsplit(".", 1)[0]
-            _compatible_ov_version = _compatible_openvino_major_version
-            if _is_ov_dev_version:
-                _compatible_ovt_version = _ov_version.replace("dev", "0.dev")
-            else:
-                _compatible_ovt_version = _ov_version + ".0"
-
-        _update_ov_command = _create_pip_update_command(
-            "openvino", _compatible_ov_version, is_dev_version=_is_ovt_dev_version
+    if not is_openvino_available():
+        return False
+
+    try:
+        import openvino_tokenizers
+    except ImportError:
+        logger.info(
+            "OpenVINO Tokenizers is not available, tokenizer won't be converted. To automatically "
+            "convert tokenizer during OpenVINO model export install OpenVINO Tokenizers.\n"
+            "For PyPI distribution of OpenVINO:\n"
+            "pip install openvino-tokenizers[transformers]\n"
+            "For pre-release OpenVINO version:\n"
+            "pip install --pre -U openvino openvino-tokenizers"
+            "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
+            "For archive OpenVINO distribution:\n"
+            "pip install --no-deps openvino-tokenizers\n"
+            "For detailed installation instructions go to "
+            "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file#installation"
         )
-        _update_ovt_command = _create_pip_update_command(
-            "openvino-tokenizers", _compatible_ovt_version, is_dev_version=_is_ov_dev_version
+        return False
+
+    try:
+        openvino_tokenizers._get_factory()
+    except RuntimeError as e:
+        tokenizers_version = openvino_tokenizers.__version__
+        if tokenizers_version == "0.0.0.0":
+            tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+        logger.warning(
+            "OpenVINO and OpenVINO Tokenizers versions are not binary compatible.\n"
+            f"OpenVINO version:            {_openvino_version}\n"
+            f"OpenVINO Tokenizers version: {tokenizers_version}\n"
+            "First 3 numbers should be the same. Update OpenVINO Tokenizers to compatible version. "
+            "It is recommended to use the same day builds for pre-release version.\n"
+            "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: "
+            "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
+            "#build-and-install-from-source\n"
+            f"Error: {e}\n"
+            "Tokenizer won't be converted."
         )
+        return False
 
-        _openvino_tokenizers_available = _ov_version.startswith(_compatible_openvino_major_version)
-
-        if not _openvino_tokenizers_available:
-            logger.warning(
-                "OpenVINO Tokenizers version is not compatible with OpenVINO version. "
-                f"Installed OpenVINO version: {_ov_version}, "
-                f"OpenVINO Tokenizers requires {_compatible_ov_version}. "
-                "OpenVINO Tokenizers models will not be added during export. "
-                f"Update OpenVINO with \n{_update_ov_command}\n"
-                f"Or update OpenVINO Tokenizers with \n{_update_ovt_command}"
-            )
-        elif _ov_version != _compatible_ov_version:
-            logger.info(
-                "OpenVINO Tokenizers version is not aligned with OpenVINO version. "
-                f"Installed OpenVINO version: {_ov_version}, "
-                f"OpenVINO Tokenizers version: {_compatible_ov_version}. "
-                "If you didn't get OpenVINO Tokenizer after model conversion "
-                f"try to align libraries versions with one of two commands:\n{_update_ov_command}\n"
-                f"Or\n{_update_ovt_command}"
-            )
-    return _openvino_tokenizers_available
+    return True
 
 
 def is_nncf_available():

From 9041fe5baf9321434dc830a9175259284ee68bfa Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Mon, 25 Mar 2024 17:52:14 +0000
Subject: [PATCH 10/19] Add Check for OpenVINO Nightly and Archive

---
 optimum/intel/utils/import_utils.py | 65 +++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 12 deletions(-)

diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 30d4b8305f..417ffaaa13 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -181,6 +181,7 @@ def is_openvino_tokenizers_available():
     try:
         import openvino_tokenizers
     except ImportError:
+        ov_major_version = _openvino_version.split("-")[0]
         logger.info(
             "OpenVINO Tokenizers is not available, tokenizer won't be converted. To automatically "
             "convert tokenizer during OpenVINO model export install OpenVINO Tokenizers.\n"
@@ -189,31 +190,71 @@ def is_openvino_tokenizers_available():
             "For pre-release OpenVINO version:\n"
             "pip install --pre -U openvino openvino-tokenizers"
             "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
-            "For archive OpenVINO distribution:\n"
-            "pip install --no-deps openvino-tokenizers\n"
-            "For detailed installation instructions go to "
-            "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file#installation"
+            "If you using archive OpenVINO distribution install release, nightly version or build from source:\n"
+            f"Release: pip install --no-deps openvino-tokenizers~={ov_major_version}\n"
+            f"Nightly: pip install --pre openvino-tokenizers~={ov_major_version}.0.dev --extra-index-url "
+            f"https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
+            "Build from source instructions: https://github.com/openvinotoolkit/openvino_tokenizers"
+            "?tab=readme-ov-file#build-and-install-from-source\n"
         )
         return False
 
+    try:
+        pip_metadata_version = importlib_metadata.version("openvino")
+        is_nightly = False
+    except importlib_metadata.PackageNotFoundError:
+        try:
+            pip_metadata_version = importlib_metadata.version("openvino-nightly")
+            is_nightly = True
+        except importlib_metadata.PackageNotFoundError:
+            pip_metadata_version = False
+            is_nightly = False
+
     try:
         openvino_tokenizers._get_factory()
     except RuntimeError as e:
         tokenizers_version = openvino_tokenizers.__version__
+
         if tokenizers_version == "0.0.0.0":
-            tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
-        logger.warning(
+            try:
+                tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+            except importlib_metadata.PackageNotFoundError:
+                pass
+        message = (
             "OpenVINO and OpenVINO Tokenizers versions are not binary compatible.\n"
             f"OpenVINO version:            {_openvino_version}\n"
             f"OpenVINO Tokenizers version: {tokenizers_version}\n"
             "First 3 numbers should be the same. Update OpenVINO Tokenizers to compatible version. "
-            "It is recommended to use the same day builds for pre-release version.\n"
-            "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: "
-            "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
-            "#build-and-install-from-source\n"
-            f"Error: {e}\n"
-            "Tokenizer won't be converted."
         )
+        if not pip_metadata_version:
+            message += (
+                "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: "
+                "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
+                "#build-and-install-from-source"
+            )
+        else:
+            message += "It is recommended to use the same day builds for pre-release version. "
+            if is_nightly:
+                message += (
+                    "openvino-nightly package detected - use nightly drops from PyPI Simple "
+                    "Repo (PEP 503) implementation on AWS S3 instead. "
+                )
+            message += "To update both OpenVINO and OpenVINO Tokenizers to release version perform:\n"
+            if is_nightly:
+                message += "pip uninstall -y openvino-nightly && "
+            message += (
+                "pip install --force-reinstall openvino openvino-tokenizers\n"
+                "To update both OpenVINO and OpenVINO Tokenizers to nightly version perform:\n"
+            )
+            if is_nightly:
+                message += "pip uninstall -y openvino-nightly && "
+            message += (
+                "pip install --pre -U openvino openvino-tokenizers "
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
+                f"Error: {e}\n"
+                "Tokenizer won't be converted."
+            )
+        logger.warning(message)
         return False
 
     return True

From 3dcaf9e892d853de9b60997762c42590c24f5c71 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 28 Mar 2024 18:25:00 +0000
Subject: [PATCH 11/19] Add linux distros compatibility message

---
 optimum/intel/utils/import_utils.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 008f7cd0e3..1429493584 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -246,6 +246,11 @@ def is_openvino_tokenizers_available():
                 "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
                 "#build-and-install-from-source"
             )
+            if sys.platform == "linux":
+                message += (
+                    "\nThe PyPI version of OpenVINO Tokenizers is built on CentOS and may not be compatible with other "
+                    "Linux distributions; rebuild OpenVINO Tokenizers from source."
+                )
         else:
             message += "It is recommended to use the same day builds for pre-release version. "
             if is_nightly:

From 241d265bb5a045da91d0b5b9b39a36fdc98491b4 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 28 Mar 2024 18:48:38 +0000
Subject: [PATCH 12/19] Address Review Comments

---
 optimum/intel/utils/import_utils.py | 39 +++++++++++------------------
 1 file changed, 15 insertions(+), 24 deletions(-)

diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 1429493584..41b5c98905 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -195,21 +195,10 @@ def is_openvino_tokenizers_available():
     try:
         import openvino_tokenizers
     except ImportError:
-        ov_major_version = _openvino_version.split("-")[0]
         logger.info(
-            "OpenVINO Tokenizers is not available, tokenizer won't be converted. To automatically "
-            "convert tokenizer during OpenVINO model export install OpenVINO Tokenizers.\n"
-            "For PyPI distribution of OpenVINO:\n"
-            "pip install openvino-tokenizers[transformers]\n"
-            "For pre-release OpenVINO version:\n"
-            "pip install --pre -U openvino openvino-tokenizers"
-            "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
-            "If you using archive OpenVINO distribution install release, nightly version or build from source:\n"
-            f"Release: pip install --no-deps openvino-tokenizers~={ov_major_version}\n"
-            f"Nightly: pip install --pre openvino-tokenizers~={ov_major_version}.0.dev --extra-index-url "
-            f"https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
-            "Build from source instructions: https://github.com/openvinotoolkit/openvino_tokenizers"
-            "?tab=readme-ov-file#build-and-install-from-source\n"
+            "OpenVINO Tokenizers is not available, tokenizer won't be converted. To deploy models in production "
+            "with C++ code, please follow installation instructions: "
+            "https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n"
         )
         return False
 
@@ -226,7 +215,7 @@ def is_openvino_tokenizers_available():
 
     try:
         openvino_tokenizers._get_factory()
-    except RuntimeError as e:
+    except RuntimeError:
         tokenizers_version = openvino_tokenizers.__version__
 
         if tokenizers_version == "0.0.0.0":
@@ -252,25 +241,27 @@ def is_openvino_tokenizers_available():
                     "Linux distributions; rebuild OpenVINO Tokenizers from source."
                 )
         else:
-            message += "It is recommended to use the same day builds for pre-release version. "
-            if is_nightly:
-                message += (
-                    "openvino-nightly package detected - use nightly drops from PyPI Simple "
-                    "Repo (PEP 503) implementation on AWS S3 instead. "
-                )
-            message += "To update both OpenVINO and OpenVINO Tokenizers to release version perform:\n"
+            message += (
+                "It is recommended to use the same day builds for pre-release version. "
+                "To install both OpenVINO and OpenVINO Tokenizers release version perform:\n"
+            )
             if is_nightly:
                 message += "pip uninstall -y openvino-nightly && "
             message += (
                 "pip install --force-reinstall openvino openvino-tokenizers\n"
-                "To update both OpenVINO and OpenVINO Tokenizers to nightly version perform:\n"
+            )
+            if is_nightly:
+                message += (
+                    "openvino-nightly package will be deprecated in the future - use pre-release drops instead. "
+                )
+            message += (
+                "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n"
             )
             if is_nightly:
                 message += "pip uninstall -y openvino-nightly && "
             message += (
                 "pip install --pre -U openvino openvino-tokenizers "
                 "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
-                f"Error: {e}\n"
                 "Tokenizer won't be converted."
             )
         logger.warning(message)

From 4d3df41acb42d2baa618be29b76ab11b1b89c3a8 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 28 Mar 2024 18:55:39 +0000
Subject: [PATCH 13/19] Address Review Comments

---
 optimum/intel/utils/import_utils.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 41b5c98905..0684a2fe25 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -247,16 +247,12 @@ def is_openvino_tokenizers_available():
             )
             if is_nightly:
                 message += "pip uninstall -y openvino-nightly && "
-            message += (
-                "pip install --force-reinstall openvino openvino-tokenizers\n"
-            )
+            message += "pip install --force-reinstall openvino openvino-tokenizers\n"
             if is_nightly:
                 message += (
                     "openvino-nightly package will be deprecated in the future - use pre-release drops instead. "
                 )
-            message += (
-                "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n"
-            )
+            message += "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n"
             if is_nightly:
                 message += "pip uninstall -y openvino-nightly && "
             message += (

From eb055941103db6a2101df93741afd0c6be34dabd Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 4 Apr 2024 21:02:05 +0100
Subject: [PATCH 14/19] Address Review Comments

---
 optimum/exporters/openvino/__main__.py |  2 ++
 optimum/intel/utils/import_utils.py    | 41 +++++++-------------------
 tests/openvino/test_exporters_cli.py   | 14 ++++++---
 3 files changed, 23 insertions(+), 34 deletions(-)

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index f16dcf4eea..5da6165106 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -343,6 +343,8 @@ class StoreAttr(object):
             tokenizer_2 = getattr(model, "tokenizer_2", None)
             if tokenizer_2 is not None:
                 export_tokenizer(tokenizer_2, output, suffix="_2")
+    elif convert_tokenizer and not is_openvino_tokenizers_available():
+        logger.warning("Tokenizer won't be converted.")
 
     # Unpatch modules after GPTQ export
     if do_gptq_patching:
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 0684a2fe25..fcdf932a28 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -170,33 +170,14 @@ def is_openvino_available():
     return _openvino_available
 
 
-def _create_pip_update_command(package_name: str, compatible_version: str, is_dev_version: bool = False) -> str:
-    return " ".join(
-        filter(
-            None,
-            (
-                "pip install",
-                "--pre" if is_dev_version else "",
-                "-U",
-                f"{package_name}=={compatible_version}",
-                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
-                if is_dev_version
-                else "",
-            ),
-        )
-    )
-
-
 @functools.lru_cache(1)
 def is_openvino_tokenizers_available():
     if not is_openvino_available():
         return False
 
-    try:
-        import openvino_tokenizers
-    except ImportError:
+    if importlib.util.find_spec("openvino_tokenizers") is None:
         logger.info(
-            "OpenVINO Tokenizers is not available, tokenizer won't be converted. To deploy models in production "
+            "OpenVINO Tokenizers is not available. To deploy models in production "
             "with C++ code, please follow installation instructions: "
             "https://github.com/openvinotoolkit/openvino_tokenizers?tab=readme-ov-file#installation\n"
         )
@@ -204,16 +185,17 @@ def is_openvino_tokenizers_available():
 
     try:
         pip_metadata_version = importlib_metadata.version("openvino")
-        is_nightly = False
     except importlib_metadata.PackageNotFoundError:
-        try:
-            pip_metadata_version = importlib_metadata.version("openvino-nightly")
-            is_nightly = True
-        except importlib_metadata.PackageNotFoundError:
-            pip_metadata_version = False
-            is_nightly = False
+        pip_metadata_version = False
+    try:
+        pip_metadata_version = importlib_metadata.version("openvino-nightly")
+        is_nightly = True
+    except importlib_metadata.PackageNotFoundError:
+        is_nightly = False
 
     try:
+        import openvino_tokenizers
+
         openvino_tokenizers._get_factory()
     except RuntimeError:
         tokenizers_version = openvino_tokenizers.__version__
@@ -257,8 +239,7 @@ def is_openvino_tokenizers_available():
                 message += "pip uninstall -y openvino-nightly && "
             message += (
                 "pip install --pre -U openvino openvino-tokenizers "
-                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly\n"
-                "Tokenizer won't be converted."
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
             )
         logger.warning(message)
         return False
diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index e82ca8c0be..84c8a6b90b 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -118,7 +118,6 @@ def test_exporters_cli(self, task: str, model_type: str):
         for arch in SUPPORTED_ARCHITECTURES
         if not arch[0].endswith("-with-past") and not arch[1].endswith("-refiner")
     )
-    @unittest.skipIf(not is_openvino_tokenizers_available(), reason="OpenVINO Tokenizers not available")
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
@@ -126,12 +125,19 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()
-            save_dir = Path(tmpdir)
-            number_of_tokenizers = sum("tokenizer" in file for file in map(str, save_dir.rglob("*.xml")))
+            if not is_openvino_tokenizers_available():
+                self.assertTrue(
+                    "OpenVINO Tokenizers is not available." in output
+                    or "OpenVINO and OpenVINO Tokenizers versions are not binary compatible." in output,
+                    msg=output
+                )
+                return
+
+            number_of_tokenizers = sum("tokenizer" in file for file in map(str, Path(tmpdir).rglob("*.xml")))
             self.assertEqual(
                 self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type],
                 number_of_tokenizers,
-                f"OVT: {is_openvino_tokenizers_available() }",
+                output
             )
 
             if number_of_tokenizers == 1:

From 4ed432de01a038484a155aede750d7d1613c5c4b Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 4 Apr 2024 21:10:29 +0100
Subject: [PATCH 15/19] Fix Style

---
 tests/openvino/test_exporters_cli.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index 84c8a6b90b..eabd190410 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -129,16 +129,12 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
                 self.assertTrue(
                     "OpenVINO Tokenizers is not available." in output
                     or "OpenVINO and OpenVINO Tokenizers versions are not binary compatible." in output,
-                    msg=output
+                    msg=output,
                 )
                 return
 
             number_of_tokenizers = sum("tokenizer" in file for file in map(str, Path(tmpdir).rglob("*.xml")))
-            self.assertEqual(
-                self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type],
-                number_of_tokenizers,
-                output
-            )
+            self.assertEqual(self.EXPECTED_NUMBER_OF_TOKENIZER_MODELS[model_type], number_of_tokenizers, output)
 
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)

From 37108848d732af10263d6d8273edc105b96ddcb1 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Wed, 17 Apr 2024 14:34:42 +0100
Subject: [PATCH 16/19] Change Warnings to Debug Level

---
 optimum/exporters/openvino/convert.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py
index f48a6e4867..1875cb0016 100644
--- a/optimum/exporters/openvino/convert.py
+++ b/optimum/exporters/openvino/convert.py
@@ -669,13 +669,13 @@ def export_tokenizer(
     try:
         converted = convert_tokenizer(tokenizer, with_detokenizer=True)
     except NotImplementedError:
-        logger.warning("Detokenizer is not supported, convert tokenizer only.")
+        logger.info("Detokenizer is not supported, convert tokenizer only.")
         converted = convert_tokenizer(tokenizer, with_detokenizer=False)
     except OVTypeError:
-        logger.warning(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
+        logger.debug(f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported.")
         return
     except Exception as exception:
-        logger.warning(
+        logger.debug(
             f"OpenVINO Tokenizer export for {type(tokenizer).__name__} is not supported. Exception: {exception}"
         )
         return

From 80d4c1db640926ce81aa79dc28cf1ed5dcf795bc Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 18 Apr 2024 12:09:39 +0100
Subject: [PATCH 17/19] Fix Tests Debug Message

---
 tests/openvino/test_exporters_cli.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index eabd190410..ff1f23fa56 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -138,8 +138,7 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
 
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
-            elif number_of_tokenizers == 0 and task not in ("image-classification", "audio-classification"):
-                self.assertTrue(("OpenVINO Tokenizer export for" in output and "is not supported." in output), output)
+
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
     def test_exporters_cli_fp16(self, task: str, model_type: str):

From f4b3301a603c553553d8243ba7188be64703ccc5 Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 18 Apr 2024 14:58:55 +0100
Subject: [PATCH 18/19] Fix Style

---
 tests/openvino/test_exporters_cli.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index f9526de3b3..6e1c7a56bd 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -146,7 +146,6 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
 
-
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
     def test_exporters_cli_fp16(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:

From 934ea22240b4f64368ad499cdee16ec28f4bbf3e Mon Sep 17 00:00:00 2001
From: Artur Paniukov <chgk1101@gmail.com>
Date: Thu, 18 Apr 2024 15:04:11 +0100
Subject: [PATCH 19/19] Fix Style

---
 optimum/commands/export/openvino.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/optimum/commands/export/openvino.py b/optimum/commands/export/openvino.py
index aea0322a6a..3015d7b5b5 100644
--- a/optimum/commands/export/openvino.py
+++ b/optimum/commands/export/openvino.py
@@ -270,5 +270,3 @@ def run(self):
                 library_name=library_name,
                 # **input_shapes,
             )
-            
-