replied to comments

Author: alexsu52

optimum/intel/openvino/quantization.py

@@ -43,6 +43,7 @@
 from ...exporters.openvino.model_patcher import patch_model_with_bettertransformer
 from ...exporters.openvino.stateful import ensure_export_task_support_stateful, ensure_stateful_is_available
 from ..utils.constant import _TASK_ALIASES
+from ..utils.modeling_utils import get_model_device
 from .configuration import OVConfig
 from .modeling_base import OVBaseModel
 from .modeling_decoder import OVBaseDecoderModel
@@ -414,7 +415,7 @@ def _quantize_torchmodel(
                 model = patch_model_with_bettertransformer(model)
 
             dummy_inputs = onnx_config.generate_dummy_inputs(framework="pt")
-            device = model.device
+            device = get_model_device(model)
             dummy_inputs = tree_map(
                 lambda value: value.to(device) if isinstance(value, torch.Tensor) else value, dummy_inputs
             )

optimum/intel/utils/modeling_utils.py

@@ -148,3 +148,24 @@ def patch_decoder_attention_mask(model: "PreTrainedModel"):
     elif model.config.model_type in {"blenderbot-small", "blenderbot", "opt", "pegasus", "bart"}:
         model.model.decoder._prepare_decoder_attention_mask = _prepare_decoder_attention_mask
     return model
+
+
+def get_model_device(model: torch.nn.Module) -> torch.device:
+    """
+    Determines the device on which a PyTorch model is currently residing.
+
+    Args:
+        model: The PyTorch model to query.
+
+    Returns:
+        torch.device: The device where the model's parameters are located.
+
+    Raises:
+        StopIteration: If the model has no parameters.
+    """
+    try:
+        device = next(model.parameters()).device
+    except StopIteration:
+        # The model had no parameters at all, doesn't matter which device to choose
+        device = torch.device("cpu")
+    return device