Skip to content

Commit 9429c01

Browse files
committed
Add test
1 parent 7818f5a commit 9429c01

File tree

3 files changed

+145
-7
lines changed

3 files changed

+145
-7
lines changed

optimum/intel/generation/modeling.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,7 @@ def prepare_jit_inputs(model: PreTrainedModel, task: str, use_cache: bool = Fals
6666

6767
def jit_trace(model: PreTrainedModel, task: str, use_cache: bool = False):
6868
model_inputs = prepare_jit_inputs(model, task, use_cache)
69-
model.config.return_dict = "text-generation" not in task
70-
69+
model.config.return_dict = task not in {"text-generation", "audio-classification"}
7170
# check if the model_inputs is correct.
7271
model(**model_inputs)
7372

optimum/intel/ipex/modeling_base.py

+47-5
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ def __init__(
7070
self.model.to(self._device)
7171
self.model_save_dir = model_save_dir
7272

73+
self.input_names = {
74+
inputs.debugName().split(".")[0] for inputs in model.graph.inputs() if inputs.debugName() != "self"
75+
}
7376
# Registers the IPEXModelForXXX classes into the transformers AutoModel classes to avoid warnings when creating
7477
# a pipeline https://github.com/huggingface/transformers/blob/cad61b68396a1a387287a8e2e2fef78a25b79383/src/transformers/pipelines/base.py#L863
7578
AutoConfig.register(self.base_model_prefix, AutoConfig)
@@ -172,8 +175,22 @@ def _save_pretrained(self, save_directory: Union[str, Path]):
172175
output_path = os.path.join(save_directory, WEIGHTS_NAME)
173176
torch.jit.save(self.model, output_path)
174177

175-
def forward(self, *args, **kwargs):
176-
outputs = self.model(*args, **kwargs)
178+
def forward(
179+
self,
180+
input_ids: torch.Tensor,
181+
attention_mask: torch.Tensor,
182+
token_type_ids: torch.Tensor = None,
183+
**kwargs,
184+
):
185+
inputs = {
186+
"input_ids": input_ids,
187+
"attention_mask": attention_mask,
188+
}
189+
190+
if "token_type_ids" in self.input_names:
191+
inputs["token_type_ids"] = token_type_ids
192+
193+
outputs = self.model(**inputs)
177194
return ModelOutput(**outputs) if isinstance(outputs, dict) else ModelOutput(logits=outputs[0])
178195

179196
def eval(self):
@@ -212,11 +229,39 @@ class IPEXModelForImageClassification(IPEXModel):
212229
auto_model_class = AutoModelForImageClassification
213230
export_feature = "image-classification"
214231

232+
def forward(
233+
self,
234+
pixel_values: torch.Tensor,
235+
**kwargs,
236+
):
237+
inputs = {
238+
"pixel_values": pixel_values,
239+
}
240+
241+
outputs = self.model(**inputs)
242+
return ModelOutput(**outputs) if isinstance(outputs, dict) else ModelOutput(logits=outputs[0])
243+
215244

216245
class IPEXModelForAudioClassification(IPEXModel):
217246
auto_model_class = AutoModelForAudioClassification
218247
export_feature = "audio-classification"
219248

249+
def forward(
250+
self,
251+
input_values: torch.Tensor,
252+
attention_mask: torch.Tensor = None,
253+
**kwargs,
254+
):
255+
inputs = {
256+
"input_values": input_values,
257+
}
258+
259+
if "attention_mask" in self.input_names:
260+
inputs["attention_mask"] = attention_mask
261+
262+
outputs = self.model(**inputs)
263+
return ModelOutput(**outputs) if isinstance(outputs, dict) else ModelOutput(logits=outputs[0])
264+
220265

221266
class IPEXModelForQuestionAnswering(IPEXModel):
222267
auto_model_class = AutoModelForQuestionAnswering
@@ -245,9 +290,6 @@ def __init__(
245290

246291
self.normalized_config = NormalizedConfigManager.get_normalized_config_class(config.model_type)(config)
247292
self.model_dtype = kwargs.get("model_dtype", None)
248-
self.input_names = {
249-
inputs.debugName().split(".")[0] for inputs in model.graph.inputs() if inputs.debugName() != "self"
250-
}
251293
self.use_cache = "past_key_values" in self.input_names
252294

253295
if use_cache ^ self.use_cache:

tests/ipex/test_modeling.py

+97
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,13 @@
1616
import time
1717
import unittest
1818

19+
import numpy as np
20+
import requests
1921
import torch
2022
from parameterized import parameterized
23+
from PIL import Image
2124
from transformers import (
25+
AutoFeatureExtractor,
2226
AutoModelForCausalLM,
2327
AutoModelForQuestionAnswering,
2428
AutoTokenizer,
@@ -30,7 +34,9 @@
3034
from optimum.exporters.onnx import MODEL_TYPES_REQUIRING_POSITION_IDS
3135
from optimum.intel import (
3236
IPEXModel,
37+
IPEXModelForAudioClassification,
3338
IPEXModelForCausalLM,
39+
IPEXModelForImageClassification,
3440
IPEXModelForMaskedLM,
3541
IPEXModelForQuestionAnswering,
3642
IPEXModelForSequenceClassification,
@@ -42,13 +48,15 @@
4248

4349
MODEL_NAMES = {
4450
"albert": "hf-internal-testing/tiny-random-albert",
51+
"beit": "hf-internal-testing/tiny-random-BeitForImageClassification",
4552
"bert": "hf-internal-testing/tiny-random-bert",
4653
"bart": "hf-internal-testing/tiny-random-bart",
4754
"blenderbot-small": "hf-internal-testing/tiny-random-BlenderbotModel",
4855
"blenderbot": "hf-internal-testing/tiny-random-BlenderbotModel",
4956
"bloom": "hf-internal-testing/tiny-random-BloomModel",
5057
"convbert": "hf-internal-testing/tiny-random-ConvBertForSequenceClassification",
5158
"codegen": "hf-internal-testing/tiny-random-CodeGenForCausalLM",
59+
"convnext": "hf-internal-testing/tiny-random-convnext",
5260
"distilbert": "hf-internal-testing/tiny-random-distilbert",
5361
"electra": "hf-internal-testing/tiny-random-electra",
5462
"flaubert": "hf-internal-testing/tiny-random-flaubert",
@@ -57,17 +65,25 @@
5765
"gpt_neo": "hf-internal-testing/tiny-random-GPTNeoModel",
5866
"gpt_neox": "hf-internal-testing/tiny-random-GPTNeoXForCausalLM",
5967
"gptj": "hf-internal-testing/tiny-random-GPTJModel",
68+
"levit": "hf-internal-testing/tiny-random-LevitModel",
6069
"llama": "fxmarty/tiny-llama-fast-tokenizer",
6170
"opt": "hf-internal-testing/tiny-random-OPTModel",
6271
"marian": "sshleifer/tiny-marian-en-de",
6372
"mbart": "hf-internal-testing/tiny-random-mbart",
6473
"mistral": "echarlaix/tiny-random-mistral",
74+
"mobilenet_v1": "google/mobilenet_v1_0.75_192",
75+
"mobilenet_v2": "hf-internal-testing/tiny-random-MobileNetV2Model",
76+
"mobilevit": "hf-internal-testing/tiny-random-mobilevit",
6577
"mpt": "hf-internal-testing/tiny-random-MptForCausalLM",
6678
"mt5": "stas/mt5-tiny-random",
79+
"resnet": "hf-internal-testing/tiny-random-resnet",
6780
"roberta": "hf-internal-testing/tiny-random-roberta",
6881
"roformer": "hf-internal-testing/tiny-random-roformer",
6982
"squeezebert": "hf-internal-testing/tiny-random-squeezebert",
7083
"t5": "hf-internal-testing/tiny-random-t5",
84+
"unispeech": "hf-internal-testing/tiny-random-unispeech",
85+
"vit": "hf-internal-testing/tiny-random-vit",
86+
"wav2vec2": "anton-l/wav2vec2-random-tiny-classifier",
7187
"xlm": "hf-internal-testing/tiny-random-xlm",
7288
}
7389

@@ -266,3 +282,84 @@ def test_compare_with_and_without_past_key_values(self):
266282
f"With pkv latency: {with_pkv_timer.elapsed:.3f} ms, without pkv latency: {without_pkv_timer.elapsed:.3f} ms,"
267283
f" speedup: {without_pkv_timer.elapsed / with_pkv_timer.elapsed:.3f}",
268284
)
285+
286+
287+
class IPEXModelForAudioClassificationTest(unittest.TestCase):
288+
IPEX_MODEL_CLASS = IPEXModelForAudioClassification
289+
SUPPORTED_ARCHITECTURES = (
290+
"unispeech",
291+
"wav2vec2",
292+
)
293+
294+
def _generate_random_audio_data(self):
295+
np.random.seed(10)
296+
t = np.linspace(0, 5.0, int(5.0 * 22050), endpoint=False)
297+
# generate pure sine wave at 220 Hz
298+
audio_data = 0.5 * np.sin(2 * np.pi * 220 * t)
299+
return audio_data
300+
301+
@parameterized.expand(SUPPORTED_ARCHITECTURES)
302+
def test_compare_to_transformers(self, model_arch):
303+
model_id = MODEL_NAMES[model_arch]
304+
ipex_model = self.IPEX_MODEL_CLASS.from_pretrained(model_id, export=True)
305+
self.assertIsInstance(ipex_model.config, PretrainedConfig)
306+
transformers_model = self.IPEX_MODEL_CLASS.auto_model_class.from_pretrained(model_id)
307+
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
308+
inputs = preprocessor(self._generate_random_audio_data(), return_tensors="pt")
309+
with torch.no_grad():
310+
transformers_outputs = transformers_model(**inputs)
311+
outputs = ipex_model(**inputs)
312+
# Compare tensor outputs
313+
self.assertTrue(torch.allclose(outputs.logits, transformers_outputs.logits, atol=1e-3))
314+
315+
@parameterized.expand(SUPPORTED_ARCHITECTURES)
316+
def test_pipeline(self, model_arch):
317+
model_id = MODEL_NAMES[model_arch]
318+
model = self.IPEX_MODEL_CLASS.from_pretrained(model_id, export=True)
319+
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
320+
pipe = pipeline("audio-classification", model=model, feature_extractor=preprocessor)
321+
outputs = pipe([np.random.random(16000)])
322+
self.assertEqual(pipe.device, model.device)
323+
self.assertTrue(all(item["score"] > 0.0 for item in outputs[0]))
324+
325+
326+
class IPEXModelForImageClassificationIntegrationTest(unittest.TestCase):
327+
SUPPORTED_ARCHITECTURES = (
328+
"beit",
329+
# "levit",
330+
"mobilenet_v1",
331+
"mobilenet_v2",
332+
"mobilevit",
333+
"resnet",
334+
"vit",
335+
)
336+
IPEX_MODEL_CLASS = IPEXModelForImageClassification
337+
338+
@parameterized.expand(SUPPORTED_ARCHITECTURES)
339+
def test_compare_to_transformers(self, model_arch):
340+
model_id = MODEL_NAMES[model_arch]
341+
set_seed(SEED)
342+
ipex_model = self.IPEX_MODEL_CLASS.from_pretrained(model_id, export=True)
343+
self.assertIsInstance(ipex_model.config, PretrainedConfig)
344+
transformers_model = self.IPEX_MODEL_CLASS.auto_model_class.from_pretrained(model_id)
345+
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
346+
url = "http://images.cocodataset.org/val2017/000000039769.jpg"
347+
image = Image.open(requests.get(url, stream=True).raw)
348+
inputs = preprocessor(images=image, return_tensors="pt")
349+
with torch.no_grad():
350+
transformers_outputs = transformers_model(**inputs)
351+
outputs = ipex_model(**inputs)
352+
self.assertIn("logits", outputs)
353+
# Compare tensor outputs
354+
self.assertTrue(torch.allclose(outputs.logits, transformers_outputs.logits, atol=1e-4))
355+
356+
@parameterized.expand(SUPPORTED_ARCHITECTURES)
357+
def test_pipeline(self, model_arch):
358+
model_id = MODEL_NAMES[model_arch]
359+
model = self.IPEX_MODEL_CLASS.from_pretrained(model_id, export=True)
360+
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
361+
pipe = pipeline("image-classification", model=model, feature_extractor=preprocessor)
362+
outputs = pipe("http://images.cocodataset.org/val2017/000000039769.jpg")
363+
self.assertEqual(pipe.device, model.device)
364+
self.assertGreaterEqual(outputs[0]["score"], 0.0)
365+
self.assertTrue(isinstance(outputs[0]["label"], str))

0 commit comments

Comments
 (0)