Set Left Padding For Text Gen Task

Author: apaniukov

optimum/commands/export/openvino.py

@@ -315,7 +315,7 @@ def _get_default_int4_config(model_id_or_path, library_name):
             model = model_cls.from_pretrained(self.args.model, export=True, quantization_config=quantization_config)
             model.save_pretrained(self.args.output)
             if not self.args.disable_convert_tokenizer:
-                maybe_convert_tokenizers(library_name, self.args.output, model)
+                maybe_convert_tokenizers(library_name, self.args.output, model, task=task)
         elif task.startswith("text-generation") and quantize_with_dataset:
             from optimum.intel import OVModelForCausalLM
 
@@ -334,7 +334,7 @@ def _get_default_int4_config(model_id_or_path, library_name):
                 preprocessors = maybe_load_preprocessors(
                     self.args.model, trust_remote_code=self.args.trust_remote_code
                 )
-                maybe_convert_tokenizers(library_name, self.args.output, preprocessors=preprocessors)
+                maybe_convert_tokenizers(library_name, self.args.output, preprocessors=preprocessors, task=task)
         else:
             # TODO : add input shapes
             main_export(

optimum/exporters/openvino/__main__.py

@@ -387,7 +387,7 @@ class StoreAttr(object):
     )
 
     if convert_tokenizer:
-        maybe_convert_tokenizers(library_name, output, model, preprocessors)
+        maybe_convert_tokenizers(library_name, output, model, preprocessors, task=task)
 
     clear_class_registry()
     del model
@@ -399,7 +399,7 @@ class StoreAttr(object):
         GPTQQuantizer.post_init_model = orig_post_init_model
 
 
-def maybe_convert_tokenizers(library_name: str, output: Path, model=None, preprocessors=None):
+def maybe_convert_tokenizers(library_name: str, output: Path, model=None, preprocessors=None, task=None):
     """
     Tries to convert tokenizers to OV format and export them to disk.
 
@@ -412,6 +412,8 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             Model instance.
         preprocessors (`Iterable`, *optional*, defaults to None):
             Iterable possibly containing tokenizers to be converted.
+        task (`str`, *optional*, defaults to None):
+            The task to export the model for. Affects tokenizer conversion parameters.
     """
     from optimum.exporters.openvino.convert import export_tokenizer
 
@@ -420,7 +422,7 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             tokenizer = next(filter(lambda it: isinstance(it, PreTrainedTokenizerBase), preprocessors), None)
             if tokenizer:
                 try:
-                    export_tokenizer(tokenizer, output)
+                    export_tokenizer(tokenizer, output, task=task)
                 except Exception as exception:
                     logger.warning(
                         "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "
@@ -430,6 +432,6 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
             for tokenizer_name in ("tokenizer", "tokenizer_2"):
                 tokenizer = getattr(model, tokenizer_name, None)
                 if tokenizer:
-                    export_tokenizer(tokenizer, output / tokenizer_name)
+                    export_tokenizer(tokenizer, output / tokenizer_name, task=task)
     else:
         logger.warning("Tokenizer won't be converted.")

optimum/exporters/openvino/convert.py

@@ -706,6 +706,7 @@ def export_tokenizer(
     tokenizer,
     output: Union[str, Path],
     suffix: Optional[str] = "",
+    task: Optional[str] = None,
 ):
     # avoid circular imports
     from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME
@@ -722,6 +723,11 @@ def export_tokenizer(
     if output.exists():
         tokenizer = maybe_convert_tokenizer_to_fast(tokenizer, output)
 
+    if task is not None and task.startswith("text-generation"):
+        logger.info(f"Set padding side to left for `{task}` task.")
+        tokenizer.padding_side = "left"
+        tokenizer.truncation_side = "left"
+
     try:
         converted = convert_tokenizer(tokenizer, with_detokenizer=True)
     except NotImplementedError:

tests/openvino/test_exporters_cli.py

@@ -171,6 +171,10 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
 
+            if task.startswith("text-generation"):
+                self.assertTrue("Set padding side to left" in output, output)
+
+
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
     def test_exporters_cli_fp16(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir: