add IPEX tests (#267)

Author: echarlaix

.github/workflows/test_ipex.yml

@@ -0,0 +1,37 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+name: Intel IPEX - Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8, 3.9]
+        os: [ubuntu-latest]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .[ipex,tests]
+        pip install torch==1.13.0 intel-extension-for-pytorch==1.13.0
+    - name: Test with Pytest
+      run: |
+        pytest tests/ipex/

setup.py

@@ -15,7 +15,6 @@
     "optimum>=1.7.3",
     "transformers>=4.20.0",
     "datasets>=1.4.0",
-    "torch",
     "sentencepiece",
     "scipy",
 ]

tests/ipex/test_inference.py

@@ -0,0 +1,75 @@
+#  Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import torch
+
+# TODO : add more tasks
+from transformers import (
+    AutoFeatureExtractor,
+    AutoModel,
+    AutoModelForAudioClassification,
+    AutoModelForCausalLM,
+    AutoModelForImageClassification,
+    AutoModelForMaskedLM,
+    AutoModelForQuestionAnswering,
+    AutoModelForSeq2SeqLM,
+    AutoModelForSequenceClassification,
+    AutoModelForTokenClassification,
+    AutoTokenizer,
+    pipeline,
+)
+
+from evaluate import evaluator
+from optimum.intel import inference_mode as ipex_inference_mode
+from parameterized import parameterized
+
+
+MODEL_NAMES = {
+    "bert": "hf-internal-testing/tiny-random-bert",
+    "distilbert": "hf-internal-testing/tiny-random-distilbert",
+    "roberta": "hf-internal-testing/tiny-random-roberta",
+}
+
+_TASK_TO_AUTOMODELS = {
+    "text-classification": AutoModelForSequenceClassification,
+    "token-classification": AutoModelForTokenClassification,
+}
+
+
+class IPEXIntegrationTest(unittest.TestCase):
+    SUPPORTED_ARCHITECTURES = (
+        "bert",
+        "distilbert",
+        "roberta",
+    )
+
+    @parameterized.expand(SUPPORTED_ARCHITECTURES)
+    def test_pipeline_classification_inference(self, model_arch):
+        model_id = MODEL_NAMES[model_arch]
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        inputs = "This is a sample input"
+        for task, auto_model_class in _TASK_TO_AUTOMODELS.items():
+            model = auto_model_class.from_pretrained(model_id)
+            pipe = pipeline(task, model=model, tokenizer=tokenizer)
+
+            with torch.inference_mode():
+                outputs = pipe(inputs)
+            with ipex_inference_mode(pipe) as ipex_pipe:
+                outputs_ipex = ipex_pipe(inputs)
+
+            self.assertEqual(outputs[0]["score"], outputs_ipex[0]["score"])