Merge branch 'block_size' into qwen

Author: jiqing-feng

optimum/exporters/ipex/cache_utils.py

@@ -1,14 +1,11 @@
+import os
 from typing import List, Optional, Tuple
 
 import torch
 from intel_extension_for_pytorch.llm.modules import PagedAttention
 from transformers import Cache, PretrainedConfig
 
 
-# May need to tune based on sequence length and different models but default to 16 currently.
-BLOCK_SIZE = 16
-
-
 class IPEXPagedCache(Cache):
     """
     A PagedCache that grows dynamically as more tokens are generated. everytime it grows block-size memory, vendor could set the pageCache memory layout.
@@ -48,7 +45,8 @@ def __init__(
         self.batch_size = batch_size
         # Used in `generate` to keep tally of how many tokens the cache has seen
         self._seen_tokens = torch.zeros([batch_size], dtype=torch.int32, device=device)
-        self.block_size = BLOCK_SIZE
+        default_block_size = 16 if device.type == "cpu" else 64
+        self.block_size = int(os.environ.get("OI_PAGED_ATTN_BLOCK_SIZE", str(default_block_size)))
         self.num_blocks = (max_cache_len // self.block_size + (max_cache_len % self.block_size != 0)) * batch_size
         self.block_tables = -1 * torch.ones([self.num_blocks], dtype=torch.int32, device=device).reshape(
             batch_size, -1

optimum/exporters/ipex/modeling_utils.py

@@ -773,17 +773,19 @@ def attention_interface(
                 is_causal=True,
             )
             self.use_sdpa = True
-        elif self.has_flash_attn(query) and past_len == 0:
+        elif self.has_flash_attn(query):
             attn_output = torch.empty_like(query)
             seq_len_tensor = torch.cat((input_lens.new_tensor([0]), input_lens.cumsum(-1).int()))
+            query_len_tensor = seq_len_tensor if past_len == 0 else torch.arange(seq_len_tensor.shape[0]).int()
+            query_max_len = input_lens.max() if past_len == 0 else 1
             PagedAttention.flash_attn_varlen_func(
                 attn_output,
                 query.contiguous() if query.device.type == "xpu" else query,
                 key_cache.contiguous() if key_cache.device.type == "xpu" else key_cache,
                 value_cache.contiguous() if value_cache.device.type == "xpu" else value_cache,
+                query_len_tensor,
                 seq_len_tensor,
-                seq_len_tensor,
-                input_lens.max(),
+                query_max_len,
                 input_lens.max(),
                 1.0 / math.sqrt(self.head_dim),
                 True,