add _convert_tokens_to_ids

echarlaix · echarlaix · commit 8ea90db1cdb7 · 2024-03-22T11:15:08.000+01:00
diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py
@@ -570,6 +570,10 @@ def test_pipeline(self, model_arch):
                 "trust_remote_code": True,
             }
         tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=model_arch in self.REMOTE_CODE_MODELS)
+
+        if model_arch == "qwen":
+            tokenizer._convert_tokens_to_ids = lambda x: 0
+
         model = OVModelForCausalLM.from_pretrained(
             model_id, export=True, use_cache=False, compile=False, **model_kwargs
         )

Original file line number	Diff line number	Diff line change
`@@ -570,6 +570,10 @@ def test_pipeline(self, model_arch):`
`570`	`570`	`"trust_remote_code": True,`
`571`	`571`	`}`
`572`	`572`	`tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=model_arch in self.REMOTE_CODE_MODELS)`
	`573`	`+`
	`574`	`+ if model_arch == "qwen":`
	`575`	`+ tokenizer._convert_tokens_to_ids = lambda x: 0`
	`576`	`+`
`573`	`577`	`model = OVModelForCausalLM.from_pretrained(`
`574`	`578`	`model_id, export=True, use_cache=False, compile=False, **model_kwargs`
`575`	`579`	`)`