Cherry-Pick From Master (#112)

* Parse QWEN EOD_TOKEN_ID

(cherry picked from commit 029a341)

* Fix Parse

(cherry picked from commit 0298fea)

* Fix Test

(cherry picked from commit fd220db)

Author: apaniukov

python/openvino_tokenizers/hf_parser.py

@@ -156,8 +156,8 @@ def parse(
             ),
         ]:
             add_steps()
-        self.pipeline.eos_token_id = getattr(self.original_tokenizer, "eos_token_id", None)
 
+        self.pipeline.eos_token_id = self.pipeline.get_eos_token_id(self.original_tokenizer)
         return self.pipeline
 
     normalizers_map: Dict[
@@ -522,8 +522,9 @@ def convert_sentencepiece_model_tokenizer(
     tokenizer = Model(outputs, [input_node], TOKENIZER_NAME)
     tokenizer.validate_nodes_and_infer_types()
 
-    if hf_tokenizer.eos_token_id is not None:
-        tokenizer.set_rt_info(hf_tokenizer.eos_token_id, EOS_TOKEN_ID_NAME)
+    eos_token_id = TokenizerPipeline.get_eos_token_id(hf_tokenizer)
+    if eos_token_id is not None:
+        tokenizer.set_rt_info(eos_token_id, EOS_TOKEN_ID_NAME)
 
     if not with_detokenizer:
         return tokenizer
@@ -537,8 +538,8 @@ def convert_sentencepiece_model_tokenizer(
         clean_up_tokenization_spaces=clean_up_tokenization_spaces,
     )
 
-    if hf_tokenizer.eos_token_id is not None:
-        detokenizer.set_rt_info(hf_tokenizer.eos_token_id, EOS_TOKEN_ID_NAME)
+    if eos_token_id is not None:
+        detokenizer.set_rt_info(eos_token_id, EOS_TOKEN_ID_NAME)
 
     return tokenizer, detokenizer
 
@@ -613,9 +614,9 @@ def convert_tiktoken_model_tokenizer(
     if clean_up_tokenization_spaces:
         pipeline.add_steps(RegexDecodingStep.clean_up_tokenization_spaces())
 
+    pipeline.eos_token_id = pipeline.get_eos_token_id(hf_tokenizer)
+
     if not with_detokenizer:
         return pipeline.get_tokenizer_ov_subgraph()
 
-    pipeline.eos_token_id = hf_tokenizer.eos_token_id
-
     return pipeline.get_tokenizer_ov_subgraph(), pipeline.get_detokenizer_ov_subgraph()

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -901,6 +901,14 @@ def _(self, steps: list) -> None:
     def __getitem__(self, item: int) -> BasePipelineStep:
         return self.steps[item]
 
+    @staticmethod
+    def get_eos_token_id(hf_tokenizer) -> Optional[int]:
+        if hf_tokenizer.eos_token_id is not None:
+            return hf_tokenizer.eos_token_id
+
+        # qwen uses eod_id attrubute
+        return getattr(hf_tokenizer, "eod_id", None)
+
     def get_tokenizer_ov_subgraph(self) -> Model:
         string_inputs = [op.Parameter(Type.string, PartialShape(["?"])) for _ in range(self.number_of_inputs)]
 

tests/tokenizers_test.py

@@ -498,7 +498,7 @@ def test_eos_token_id_rt_info_bpe(hf_bpe_tokenizers):
 
 
 def test_eos_token_id_rt_info_tiktoken(hf_tiktoken_tokenizers):
-    eos_token_id = hf_tiktoken_tokenizers.eos_token_id
+    eos_token_id = hf_tiktoken_tokenizers.eos_token_id or hf_tiktoken_tokenizers.eod_id
     ov_tokenizer, ov_detokenizer = convert_tokenizer(
         hf_tiktoken_tokenizers,
         with_detokenizer=True,