Skip to content

Commit 7316d4e

Browse files
committed
attempt to resolve 4.49 compatibility issues and fix input processing for internvl2
1 parent 6cceb30 commit 7316d4e

File tree

2 files changed

+11
-17
lines changed

2 files changed

+11
-17
lines changed

optimum/intel/openvino/modeling_visual_language.py

+10-6
Original file line numberDiff line numberDiff line change
@@ -1426,12 +1426,18 @@ def load_image(image, input_size=448, max_num=12):
14261426
pixel_values = [transform(image) for image in images]
14271427
pixel_values = torch.stack(pixel_values)
14281428
return pixel_values
1429+
1430+
if image is not None and "<image>" not in text:
1431+
text = "<image>\n" + text
14291432

1433+
if tokenizer.chat_template is not None:
1434+
text = tokenizer.apply_chat_template([{"role": "user", "content": text}], add_generation_prompt=True, tokenize=False)
1435+
1436+
inputs = {}
1437+
14301438
if image is not None:
14311439
if config is None:
14321440
raise ValueError("Config is required.")
1433-
if "<image>" not in text:
1434-
text = "<image>\n" + text
14351441
pixel_values = load_image(image, input_size=config.vision_config.image_size)
14361442
num_patches = pixel_values.shape[0]
14371443
num_image_token = int(
@@ -1440,11 +1446,9 @@ def load_image(image, input_size=448, max_num=12):
14401446
)
14411447
image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * num_image_token * num_patches + IMG_END_TOKEN
14421448
text = text.replace("<image>", image_tokens, 1)
1443-
text_inputs = tokenizer(text, return_tensors="pt")
1444-
inputs = dict(text_inputs)
1449+
logger.warn(text)
14451450
inputs.update({"pixel_values": pixel_values})
1446-
else:
1447-
inputs = tokenizer(text, return_tensors="pt")
1451+
inputs.update(tokenizer(text, return_tensors="pt"))
14481452
return inputs
14491453

14501454
# internvl has issue with check _get_non_default_parameters, as wrkaraund overide _prepare_generation_config

tests/openvino/test_modeling.py

+1-11
Original file line numberDiff line numberDiff line change
@@ -1040,12 +1040,6 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
10401040
def test_compare_to_transformers(self, model_arch):
10411041
model_id = MODEL_NAMES[model_arch]
10421042

1043-
# TODO: add back once dtype fixed everywhere
1044-
# https://huggingface.co/katuni4ka/tiny-random-chatglm2/blob/main/modeling_chatglm.py#L720
1045-
# https://huggingface.co/katuni4ka/tiny-random-chatglm2/blob/main/modeling_chatglm.py#L759
1046-
if model_arch in {"chatglm", "glm4"} and is_transformers_version(">=", "4.49"):
1047-
self.skipTest("Incompatible modeling code")
1048-
10491043
not_stateful = []
10501044
if is_openvino_version("<", "2024.0"):
10511045
not_stateful.append("mixtral")
@@ -1126,7 +1120,7 @@ def test_compare_to_transformers(self, model_arch):
11261120
ov_outputs = ov_model.generate(**tokens, generation_config=gen_config)
11271121

11281122
# TODO: add back once https://huggingface.co/katuni4ka/tiny-random-minicpm3/discussions/1 merged (for all models) as current mdoeling incompatible with transformers >= v4.49
1129-
if model_arch in {"minicpm", "minicpm3", "arctic", "deepseek"} and is_transformers_version(">=", "4.49"):
1123+
if model_arch in {"deepseek"} and is_transformers_version(">=", "4.49"):
11301124
self.skipTest("Incompatible modeling code")
11311125

11321126
additional_inputs = {}
@@ -2240,10 +2234,6 @@ def test_compare_to_transformers(self, model_arch):
22402234
ov_outputs = ov_model.generate(**inputs, generation_config=gen_config)
22412235
set_seed(SEED)
22422236

2243-
# TODO: add back once https://huggingface.co/katuni4ka/tiny-random-minicpm3/discussions/1 merged for all models as current mdoeling incompatible with transformers >= v4.49
2244-
if model_arch in {"phi3_v", "nanollava"} and is_transformers_version(">=", "4.49"):
2245-
self.skipTest("Incompatible modeling code")
2246-
22472237
with torch.no_grad():
22482238
transformers_outputs = transformers_model.generate(**transformers_inputs, generation_config=gen_config)
22492239

0 commit comments

Comments
 (0)