We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 8273de7 commit 8364b87Copy full SHA for 8364b87
optimum/intel/openvino/modeling_decoder.py
@@ -386,11 +386,8 @@ def prepare_inputs(
386
inputs = {}
387
if not self.stateful:
388
if past_key_values is not None:
389
- if (
390
- self.config.model_type not in MULTI_QUERY_ATTN_MODELS
391
- or self.config.model_type == "falcon"
392
- and self.config.new_decoder_architecture
393
- ):
+ is_falcon_multi_query = self.config.model_type == "falcon" and self.config.new_decoder_architecture
+ if self.config.model_type not in MULTI_QUERY_ATTN_MODELS or is_falcon_multi_query:
394
if self._pkv_precision == Type.bf16:
395
# numpy does not support bf16, pretending f16, should change to bf16
396
past_key_values = tuple(
0 commit comments