Skip to content

Commit 31accd2

Browse files
committed
set different default value for block size based on device
Signed-off-by: jiqing-feng <jiqing.feng@intel.com>
1 parent 5b93036 commit 31accd2

File tree

1 file changed

+2
-5
lines changed

1 file changed

+2
-5
lines changed

optimum/exporters/ipex/cache_utils.py

+2-5
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@
66
from transformers import Cache, PretrainedConfig
77

88

9-
# Recommend 16 on CPU and 64 on XPU.
10-
BLOCK_SIZE = int(os.environ.get("OI_PAGED_ATTN_BLOCK_SIZE", 16))
11-
12-
139
class IPEXPagedCache(Cache):
1410
"""
1511
A PagedCache that grows dynamically as more tokens are generated. everytime it grows block-size memory, vendor could set the pageCache memory layout.
@@ -49,7 +45,8 @@ def __init__(
4945
self.batch_size = batch_size
5046
# Used in `generate` to keep tally of how many tokens the cache has seen
5147
self._seen_tokens = torch.zeros([batch_size], dtype=torch.int32, device=device)
52-
self.block_size = BLOCK_SIZE
48+
default_block_size = 16 if device.type == "cpu" else 64
49+
self.block_size = int(os.environ.get("OI_PAGED_ATTN_BLOCK_SIZE", str(default_block_size)))
5350
self.num_blocks = (max_cache_len // self.block_size + (max_cache_len % self.block_size != 0)) * batch_size
5451
self.block_tables = -1 * torch.ones([self.num_blocks], dtype=torch.int32, device=device).reshape(
5552
batch_size, -1

0 commit comments

Comments
 (0)