Transformers 4.44 support (huggingface#877)

* test

* fix ipex bloom

* fix bloom

* style

* fix

* use bloom specific modeling when export version is lower than 4.44

* fix

Author: IlyasMoutawwakil

.github/workflows/test_ipex.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.9]
-        transformers-version: ["4.39.0", "4.43.*"]
+        transformers-version: ["4.39.0", "4.44.*"]
         ipex-version: ["2.2.0", "2.3.*"]
         include:
           - python-version: 3.8

.github/workflows/test_openvino.yml

@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.12"]
-        transformers-version: ["4.36.0", "4.43.*"]
+        transformers-version: ["4.36.0", "4.44.*"]
         os: [ubuntu-latest]
 
     runs-on: ${{ matrix.os }}

.github/workflows/test_openvino_basic.yml

@@ -24,7 +24,7 @@ jobs:
         # This also ensures that the test fails if dependencies break for Python 3.7
         python-version: ["3.8", "3.12"]
         os: ["ubuntu-22.04", "windows-latest"]
-        transformers-version: ["4.43.*"]
+        transformers-version: ["4.44.*"]
         include:
           - python-version: "3.12"
             os: "ubuntu-22.04"

optimum/exporters/ipex/model_patcher.py

@@ -40,7 +40,7 @@
 
 # Please also update in the setup.py and .github/workflows/test_ipex.yml if you change the transformers version
 _TRANSFORMERS_MIN_VERSION = "4.39.0"
-_TRANSFORMERS_MAX_VERSION = "4.43.99"
+_TRANSFORMERS_MAX_VERSION = "4.44.99"
 
 _IPEX_EXPORTED_GENERATION_TASKS = ("text-generation",)
 

optimum/exporters/openvino/stateful.py

@@ -21,7 +21,7 @@
 import openvino as ov
 from openvino.runtime import opset13
 from optimum.exporters import TasksManager
-from optimum.intel.utils.import_utils import _openvino_version, is_openvino_version
+from optimum.intel.utils.import_utils import _openvino_version, is_openvino_version, is_transformers_version
 
 
 def model_has_state(ov_model: ov.Model):
@@ -216,7 +216,9 @@ def patch_stateful(config: PretrainedConfig, ov_model: ov.Model):
     batch_dim = 1 if config.model_type == "chatglm" and not hasattr(config, "rope_ratio") else 0
 
     fuse_cache_reorder(ov_model, not_kv_inputs, key_value_input_names, batch_dim)
-    num_attention_heads = config.num_attention_heads if config.model_type == "bloom" else 1
+    num_attention_heads = (
+        config.num_attention_heads if (config.model_type == "bloom" and is_transformers_version("<", "4.44")) else 1
+    )
     make_stateful(
         ov_model, not_kv_inputs, key_value_input_names, key_value_output_names, batch_dim, num_attention_heads, None
     )

optimum/intel/ipex/modeling_base.py

@@ -564,7 +564,7 @@ def _prepare_past_key_values(self, input_ids):
                 ]
             )
             return past_key_values
-        elif model_type == "bloom":
+        elif model_type == "bloom" and is_transformers_version("<", "4.44"):
             shape_key = (batch_size * num_attention_heads, d_k, 0)
             shape_value = (batch_size * num_attention_heads, 0, d_k)
             key = torch.empty(size=shape_key, dtype=self.model_dtype, device=self._device)

optimum/intel/openvino/modeling_base.py

@@ -140,7 +140,7 @@ def dtype(self) -> Optional[torch.dtype]:
     def load_model(
         file_name: Union[str, Path],
         quantization_config: Union[OVWeightQuantizationConfig, Dict] = None,
-    ):
+    ) -> openvino.runtime.Model:
         """
         Loads the model.
 

optimum/intel/openvino/modeling_decoder.py

@@ -25,6 +25,7 @@
 from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
 from openvino.preprocess import PrePostProcessor
 from openvino.runtime import Core, Tensor, Type
+from packaging.version import Version
 from transformers import AutoModelForCausalLM, PretrainedConfig
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
 from transformers.generation import GenerationMixin
@@ -38,7 +39,7 @@
 
 from ...exporters.openvino import ensure_stateful_is_available, main_export, patch_stateful
 from ...exporters.openvino.stateful import model_has_state
-from ..utils.import_utils import is_nncf_available, is_transformers_version
+from ..utils.import_utils import compare_versions, is_nncf_available, is_transformers_version
 from ..utils.modeling_utils import MULTI_QUERY_ATTN_MODELS
 from .configuration import (
     OVConfig,
@@ -51,8 +52,8 @@
 
 
 if TYPE_CHECKING:
+    from transformers.generation.streamers import BaseStreamer
     from transformers.modeling_utils import PreTrainedModel
-    from transformers.streamers import BaseStreamer
 
 
 logger = logging.getLogger(__name__)
@@ -404,7 +405,10 @@ def prepare_inputs(
         **kwargs,
     ) -> Dict:
         batch_size = input_ids.shape[0]
-        if self.config.model_type == "bloom":
+        model_transformers_version = Version(
+            self.model.rt_info["optimum"]["transformers_version"].value if "optimum" in self.model.rt_info else "0.0.0"
+        )
+        if self.config.model_type == "bloom" and compare_versions(model_transformers_version, "<", "4.44"):
             batch_size *= self.config.num_attention_heads
 
         inputs = {}
@@ -619,7 +623,10 @@ def _deduplicate_inputs(self, model_inputs: Dict):
                     shape = input_tensor.shape if isinstance(input_tensor, Tensor) else list(input_tensor.shape)
                     dtype = input_tensor.element_type if isinstance(input_tensor, Tensor) else Type(input_tensor.dtype)
                     upd_batch_size = indicies.shape[0]
-                    if self.config.model_type == "bloom":
+                    export_transformers_version = Version(self.model.rt_info["optimum"]["transformers_version"].value)
+                    if self.config.model_type == "bloom" and compare_versions(
+                        export_transformers_version, "<", "4.44"
+                    ):
                         upd_batch_size *= self.config.num_attention_heads
                     shape[
                         (
@@ -631,10 +638,11 @@ def _deduplicate_inputs(self, model_inputs: Dict):
                     upd_model_inputs[input_name] = Tensor(dtype, shape)
         upd_model_inputs["input_ids"] = unique_input_ids
         if "beam_idx" in model_inputs:
+            export_transformers_version = Version(self.model.rt_info["optimum"]["transformers_version"].value)
             beam_range = (
-                unique_input_ids.shape[0]
-                if self.config.model_type != "bloom"
-                else unique_input_ids.shape[0] * self.config.num_attention_heads
+                unique_input_ids.shape[0] * self.config.num_attention_heads
+                if (self.config.model_type == "bloom" and compare_versions(export_transformers_version, "<", "4.44"))
+                else unique_input_ids.shape[0]
             )
             beam_idx = np.arange(beam_range, dtype=int)
             upd_model_inputs["beam_idx"] = beam_idx
@@ -781,7 +789,10 @@ def _from_pretrained(
         model = cls.load_model(model_cache_path)
 
         model_type = config.model_type.replace("_", "-")
-        if model_type == "bloom":
+        export_transformers_version = Version(
+            model.rt_info["optimum"]["transformers_version"].value if "optimum" in model.rt_info else "0.0.0"
+        )
+        if model_type == "bloom" and compare_versions(export_transformers_version, "<", "4.44"):
             init_cls = OVBloomForCausalLM
         elif model_type == "gpt-bigcode":
             init_cls = OVGPTBigCodeForCausalLM

setup.py

@@ -28,7 +28,7 @@
 
 INSTALL_REQUIRE = [
     "torch>=1.11",
-    "transformers>=4.36.0,<4.44.0",
+    "transformers>=4.36,<4.45",
     "optimum@git+https://github.com/huggingface/optimum.git",
     "datasets>=1.4.0",
     "sentencepiece",
@@ -59,10 +59,10 @@
 QUALITY_REQUIRE = ["black~=23.1", "ruff==0.4.4"]
 
 EXTRAS_REQUIRE = {
-    "neural-compressor": ["neural-compressor>=2.2.0,<3.0", "accelerate", "transformers<4.43.0"],
+    "neural-compressor": ["neural-compressor>=2.2.0,<3.0", "accelerate", "transformers<4.43"],
     "openvino": ["openvino>=2023.3", "nncf>=2.11.0", "openvino-tokenizers[transformers]"],
     "nncf": ["nncf>=2.11.0"],
-    "ipex": ["intel-extension-for-pytorch", "transformers>=4.39.0,<4.44.0"],
+    "ipex": ["intel-extension-for-pytorch", "transformers>=4.39,<4.45"],
     "diffusers": ["diffusers"],
     "quality": QUALITY_REQUIRE,
     "tests": TESTS_REQUIRE,