Add --convert-tokenizer Option to CLI

Author: apaniukov

optimum/commands/export/openvino.py

@@ -103,6 +103,11 @@ def parse_args_openvino(parser: "ArgumentParser"):
             "OpenVINO native inference code that expects kv-cache inputs and outputs in the model."
         ),
     )
+    optional_group.add_argument(
+        "--convert-tokenizer",
+        action="store_true",
+        help="Add converted tokenizer and detokenizer with OpenVINO Tokenizers",
+    )
 
 
 class OVExportCommand(BaseOptimumCLICommand):
@@ -151,5 +156,6 @@ def run(self):
             compression_option=self.args.weight_format,
             compression_ratio=self.args.ratio,
             stateful=not self.args.disable_stateful,
+            convert_tokenizer=self.args.convert_tokenizer,
             # **input_shapes,
         )

optimum/exporters/openvino/__main__.py

@@ -71,6 +71,7 @@ def main_export(
     compression_option: Optional[str] = None,
     compression_ratio: Optional[float] = None,
     stateful: bool = True,
+    convert_tokenizer: bool = False,
     **kwargs_shapes,
 ):
     """
@@ -344,7 +345,7 @@ class StoreAttr(object):
             generation_config.save_pretrained(output)
         maybe_save_preprocessors(model_name_or_path, output)
 
-        if tokenizer is not None and is_openvino_tokenizers_available():
+        if convert_tokenizer and tokenizer is not None and is_openvino_tokenizers_available():
             try:
                 export_tokenizer(tokenizer, output)
             except Exception as exception:
@@ -380,12 +381,12 @@ class StoreAttr(object):
             feature_extractor.save_pretrained(output.joinpath("feature_extractor"))
 
         tokenizer = getattr(model, "tokenizer", None)
-        if tokenizer is not None and is_openvino_tokenizers_available():
+        if convert_tokenizer and tokenizer is not None and is_openvino_tokenizers_available():
             tokenizer.save_pretrained(output.joinpath("tokenizer"))
             export_tokenizer(tokenizer, output)
 
         tokenizer_2 = getattr(model, "tokenizer_2", None)
-        if tokenizer_2 is not None and is_openvino_tokenizers_available():
+        if convert_tokenizer and tokenizer_2 is not None and is_openvino_tokenizers_available():
             tokenizer_2.save_pretrained(output.joinpath("tokenizer_2"))
             export_tokenizer(tokenizer, output, suffix="_2")
 

tests/openvino/test_exporters_cli.py

@@ -122,7 +122,7 @@ def test_exporters_cli(self, task: str, model_type: str):
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
-                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
+                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --convert-tokenizer --task {task} {tmpdir}",
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()