Merge pull request #11 from ilya-lavrenov/change-block-size

Changed block_size according to latest CPU changes

Author: ilya-lavrenov

text_generation/causal_lm/cpp/continuous_batching/apps/accuracy_sample.cpp

@@ -66,7 +66,7 @@ int main(int argc, char* argv[]) try {
         .max_num_batched_tokens = 32,
         // cache params
         .num_kv_blocks = 364,
-        .block_size = 16,
+        .block_size = 32,
         // mode - vLLM or dynamic_split_fuse
         .dynamic_split_fuse = dynamic_split_fuse,
         // vLLM specific params

text_generation/causal_lm/cpp/continuous_batching/apps/throughput_benchmark.cpp

@@ -177,8 +177,8 @@ int main(int argc, char* argv[]) try {
     // Perform the first inference
     SchedulerConfig scheduler_config {
         .max_num_batched_tokens = max_batch_size,
-        .num_kv_blocks = 36800,
-        .block_size = 16,
+        .num_kv_blocks = 15000,
+        .block_size = 32,
         .dynamic_split_fuse = dynamic_split_fuse,
         .max_num_seqs = 256, // not used if dynamic_split_fuse=True
         .max_paddings = 256, // not used if dynamic_split_fuse=True

text_generation/causal_lm/cpp/continuous_batching/library/include/generation_config.hpp

@@ -6,6 +6,7 @@
 
 #include <cstdlib>
 #include <limits>
+#include <string>
 #include <functional>
 
 enum class StopCriteria {

text_generation/causal_lm/cpp/continuous_batching/library/include/scheduler_config.hpp

@@ -16,7 +16,7 @@ struct SchedulerConfig {
     std::size_t num_kv_blocks = 16;
 
     // block size for KV cache
-    std::size_t block_size = 16;
+    std::size_t block_size = 32;
 
     // whether to split prompt / generate to different scheduling phases
     bool dynamic_split_fuse = true;

text_generation/causal_lm/cpp/continuous_batching/python/tests/test_preemption.py

@@ -4,7 +4,7 @@
 
 from common import run_test_pipeline, get_models_list
 
-scheduler_params_list = [{"num_kv_blocks": 300, "block_size": 16, "dynamic_split_fuse": True, "max_num_batched_tokens": 256, "max_num_seqs": 256},
+scheduler_params_list = [{"num_kv_blocks": 300, "block_size": 32, "dynamic_split_fuse": True, "max_num_batched_tokens": 256, "max_num_seqs": 256},
                          {"num_kv_blocks": 40, "block_size": 4, "dynamic_split_fuse": True, "max_num_batched_tokens": 256, "max_num_seqs": 256}, # test preemption for dynamic_split_fuse
                          {"num_kv_blocks": 40, "block_size": 4, "dynamic_split_fuse": False, "max_num_batched_tokens": 256, "max_num_seqs": 256}] # test preemption for vllm
 @pytest.mark.parametrize("scheduler_params", scheduler_params_list)