Skip to content

Commit da00c67

Browse files
as-suvorovas-suvorov
authored
Add CB CI tests (openvinotoolkit#572)
1 parent 6d7d70d commit da00c67

File tree

8 files changed

+246
-30
lines changed

8 files changed

+246
-30
lines changed

‎.github/workflows/causal_lm_cpp.yml

+117
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ concurrency:
1414

1515
env:
1616
l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.3.0-15945-a349dc82f9a/l_openvino_toolkit_ubuntu20_2024.3.0.dev20240708_x86_64.tgz
17+
m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.3.0-15945-a349dc82f9a/m_openvino_toolkit_macos_12_6_2024.3.0.dev20240708_x86_64.tgz
1718
w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2024.3.0-15945-a349dc82f9a/w_openvino_toolkit_windows_2024.3.0.dev20240708_x86_64.zip
1819
jobs:
1920
cpp-multinomial-greedy_causal_lm-ubuntu:
@@ -584,3 +585,119 @@ jobs:
584585
timeout 30s ./samples/python/chat_sample/chat_sample.py ./TinyLlama-1.1B-Chat-v1.0/ < input.txt > ./pred2.txt
585586
diff pred2.txt ref.txt
586587
echo "Chat sample python" passed
588+
589+
cpp-continuous-batching-ubuntu:
590+
runs-on: ubuntu-20.04-8-cores
591+
steps:
592+
- uses: actions/checkout@v4
593+
with:
594+
submodules: recursive
595+
- uses: actions/setup-python@v4
596+
with:
597+
python-version: 3.8
598+
- name: Install OpenVINO
599+
run: |
600+
mkdir ./ov/
601+
curl ${{ env.l_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
602+
sudo ./ov/install_dependencies/install_openvino_dependencies.sh
603+
- name: Download, convert and build
604+
run: |
605+
source ./ov/setupvars.sh
606+
python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
607+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
608+
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
609+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
610+
cmake --build ./build/ --config Release -j
611+
- name: Run gtests
612+
run: |
613+
source ./ov/setupvars.sh
614+
./build/tests/cpp/tests_continuous_batching
615+
- name: Run accuracy_sample
616+
run: |
617+
source ./ov/setupvars.sh
618+
timeout 50s ./build/samples/cpp/continuous_batching_accuracy/continuous_batching_accuracy -m ./TinyLlama-1.1B-Chat-v1.0/ -n 5
619+
- name: Run throughput_benchmark
620+
run: |
621+
wget -q https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
622+
source ./ov/setupvars.sh
623+
timeout 200s ./build/samples/cpp/continuous_batching_benchmark/continuous_batching_benchmark -n 10 --dynamic_split_fuse -m ./TinyLlama-1.1B-Chat-v1.0/ --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json --cache_size 1
624+
625+
626+
cpp-continuous-batching-windows:
627+
runs-on: windows-latest
628+
defaults:
629+
run:
630+
shell: cmd
631+
steps:
632+
- uses: actions/checkout@v4
633+
with:
634+
submodules: recursive
635+
- uses: actions/setup-python@v4
636+
with:
637+
python-version: 3.8
638+
- name: Install OpenVINO
639+
run: |
640+
curl --output ov.zip ${{ env.w_ov_link }}
641+
unzip -d ov ov.zip
642+
dirs=(ov/*) && mv ov/*/* ov && rmdir "${dirs[@]}"
643+
shell: bash
644+
- name: Install dependencies and build
645+
run: |
646+
call .\ov\setupvars.bat
647+
python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
648+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
649+
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
650+
cmake -DCMAKE_BUILD_TYPE=Releas -S ./ -B ./build/
651+
cmake --build ./build/ --config Release -j
652+
- name: Run gtests
653+
run: |
654+
set PATH=.\build\openvino_genai\;%PATH%
655+
call .\ov\setupvars.bat
656+
.\build\tests\cpp\Release\tests_continuous_batching.exe
657+
- name: Run accuracy_sample
658+
run: |
659+
set PATH=.\build\openvino_genai\;%PATH%
660+
call .\ov\setupvars.bat
661+
.\build\samples\cpp\continuous_batching_accuracy\Release\continuous_batching_accuracy.exe -m .\TinyLlama-1.1B-Chat-v1.0\ -n 5
662+
- name: Run throughput_benchmark
663+
run: |
664+
curl -o .\ShareGPT_V3_unfiltered_cleaned_split.json -s -L "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
665+
set PATH=.\build\openvino_genai\;%PATH%
666+
call .\ov\setupvars.bat
667+
.\build\samples\cpp\continuous_batching_benchmark\Release\continuous_batching_benchmark.exe -n 2 --dynamic_split_fuse -m .\TinyLlama-1.1B-Chat-v1.0\ --dataset .\ShareGPT_V3_unfiltered_cleaned_split.json --cache_size 1
668+
669+
cpp-continuous-batching-macos:
670+
runs-on: macos-12
671+
steps:
672+
- uses: actions/checkout@v4
673+
with:
674+
submodules: recursive
675+
- uses: actions/setup-python@v4
676+
with:
677+
python-version: 3.8
678+
- name: Install OpenVINO
679+
run: |
680+
mkdir ./ov/
681+
curl ${{ env.m_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
682+
brew install coreutils scons
683+
- name: Download, convert and build
684+
run: |
685+
source ./ov/setupvars.sh
686+
python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
687+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
688+
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
689+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
690+
cmake --build ./build/ --config Release -j
691+
- name: Run gtests
692+
run: |
693+
source ./ov/setupvars.sh
694+
./build/tests/cpp/tests_continuous_batching
695+
- name: Run accuracy_sample
696+
run: |
697+
source ./ov/setupvars.sh
698+
timeout 120s ./build/samples/cpp/continuous_batching_accuracy/continuous_batching_accuracy -m ./TinyLlama-1.1B-Chat-v1.0/ -n 5
699+
- name: Run throughput_benchmark
700+
run: |
701+
wget -q https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
702+
source ./ov/setupvars.sh
703+
./build/samples/cpp/continuous_batching_benchmark/continuous_batching_benchmark -n 5 --dynamic_split_fuse -m ./TinyLlama-1.1B-Chat-v1.0/ --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json --cache_size 1

‎.github/workflows/genai_python_lib.yml

+87
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,90 @@ jobs:
8484
- run: set "PYTHONPATH=./build/" && call ./ov/setupvars.bat && python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
8585
- run: call ./ov/setupvars.bat && python -m pip install . --verbose
8686
- run: python -m pytest ./tests/python_tests/test_generate_api.py -m precommit
87+
88+
continuous_batching_python_lib_ubuntu:
89+
# A tokenizers' dependency fails to compile on ubuntu-20 n CenOS7 env.
90+
runs-on: ubuntu-22.04
91+
env:
92+
# A tokenizers' dependency fails to compile with Ninja in CenOS7 env.
93+
CMAKE_GENERATOR: Unix Makefiles
94+
CMAKE_BUILD_PARALLEL_LEVEL: null
95+
steps:
96+
- uses: actions/checkout@v4
97+
with:
98+
submodules: recursive
99+
- uses: actions/setup-python@v4
100+
with:
101+
python-version: 3.8
102+
# Install CentOS7 instead of Ubuntu to match PyPI distribution ABI.
103+
- name: Install OpenVINO
104+
run: |
105+
mkdir ./ov/
106+
curl ${{ env.l_ov_centos_link }} | tar --directory ./ov/ --strip-components 1 -xz
107+
sudo ./ov/install_dependencies/install_openvino_dependencies.sh
108+
- name: Install dependencies and build
109+
run: |
110+
source ./ov/setupvars.sh
111+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --upgrade-strategy eager
112+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
113+
cmake --build ./build/ --config Release -j
114+
- run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_sampling.py -m precommit
115+
- run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_preemption.py -m precommit
116+
- run: source ./ov/setupvars.sh && python -m pip install .
117+
- run: python -m pytest ./tests/python_tests/test_preemption.py -m precommit
118+
119+
continuous_batching_python_lib_windows:
120+
runs-on: windows-latest
121+
defaults:
122+
run:
123+
shell: cmd
124+
steps:
125+
- uses: actions/checkout@v4
126+
with:
127+
submodules: recursive
128+
- uses: actions/setup-python@v4
129+
with:
130+
python-version: 3.8
131+
132+
- name: Install OpenVINO
133+
run: |
134+
curl --output ov.zip ${{ env.w_ov_link }}
135+
unzip -d ov ov.zip
136+
dirs=(ov/*) && mv ov/*/* ov && rmdir "${dirs[@]}"
137+
shell: bash
138+
- name: Install dependencies and build
139+
run: |
140+
call .\ov\setupvars.bat
141+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --upgrade-strategy eager
142+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
143+
cmake --build ./build/ --config Release -j
144+
- run: set "PYTHONPATH=./build/" && call ./ov/setupvars.bat && python -m pytest ./tests/python_tests/test_sampling.py -m precommit
145+
- run: set "PYTHONPATH=./build/" && call ./ov/setupvars.bat && python -m pytest ./tests/python_tests/test_preemption.py -m precommit
146+
- run: call ./ov/setupvars.bat && python -m pip install . --verbose
147+
- run: python -m pytest ./tests/python_tests/test_preemption.py -m precommit
148+
149+
150+
continuous_batching_python_lib_macos:
151+
runs-on: macos-12
152+
steps:
153+
- uses: actions/checkout@v4
154+
with:
155+
submodules: recursive
156+
- uses: actions/setup-python@v4
157+
with:
158+
python-version: 3.8
159+
- name: Install OpenVINO
160+
run: |
161+
mkdir ./ov/
162+
curl ${{ env.m_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
163+
brew install coreutils scons
164+
- name: Download, convert and build
165+
run: |
166+
source ./ov/setupvars.sh
167+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --upgrade-strategy eager
168+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
169+
cmake --build ./build/ --config Release -j
170+
- run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_sampling.py -m precommit
171+
- run: source ./ov/setupvars.sh && PYTHONPATH=./build/:$PYTHONPATH python -m pytest ./tests/python_tests/test_preemption.py -m precommit
172+
- run: source ./ov/setupvars.sh && python -m pip install .
173+
- run: python -m pytest ./tests/python_tests/test_preemption.py -m precommit

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,4 @@ CMakeUserPresets.json
3434
*.?env*
3535
*.pyc
3636
__pycache__
37+
.py-build-cmake_cache

‎samples/cpp/continuous_batching_benchmark/CMakeLists.txt

-1
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,3 @@ find_package(Threads REQUIRED)
2424
set(TARGET_NAME continuous_batching_benchmark)
2525
add_executable(${TARGET_NAME} ${TARGET_NAME}.cpp)
2626
target_link_libraries(${TARGET_NAME} PRIVATE openvino::genai nlohmann_json::nlohmann_json cxxopts::cxxopts Threads::Threads)
27-
target_compile_features(${TARGET_NAME} PRIVATE cxx_std_20)

‎samples/cpp/continuous_batching_benchmark/continuous_batching_benchmark.cpp

+6-7
Original file line numberDiff line numberDiff line change
@@ -466,13 +466,12 @@ int main(int argc, char* argv[]) try {
466466
Dataset dataset = filtered_dataset(models_path, dataset_path, num_prompts, max_input_len, max_output_len);
467467

468468
// Perform the first inference
469-
ov::genai::SchedulerConfig scheduler_config {
470-
.max_num_batched_tokens = max_batch_size,
471-
.cache_size = cache_size,
472-
.block_size = 32,
473-
.dynamic_split_fuse = dynamic_split_fuse,
474-
.max_num_seqs = 256, // not used if dynamic_split_fuse=True
475-
};
469+
ov::genai::SchedulerConfig scheduler_config;
470+
scheduler_config.max_num_batched_tokens = max_batch_size,
471+
scheduler_config.cache_size = cache_size,
472+
scheduler_config.block_size = 32,
473+
scheduler_config.dynamic_split_fuse = dynamic_split_fuse,
474+
scheduler_config.max_num_seqs = 256, // not used if dynamic_split_fuse=True
476475

477476
std::cout << "Benchmarking parameters: " << std::endl;
478477
std::cout << "\tMax number of batched tokens: " << scheduler_config.max_num_batched_tokens << std::endl;

‎tests/cpp/generate_config.cpp

+11-1
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,23 @@
77

88
TEST(GenerationConfigTest, invalid_temperature) {
99
ov::genai::GenerationConfig config;
10+
config.max_new_tokens = 20;
1011
config.temperature = -0.1;
1112
config.do_sample = true;
1213
EXPECT_THROW(config.validate(), ov::Exception);
1314
}
1415

1516
TEST(GenerationConfigTest, valid_temperature) {
1617
ov::genai::GenerationConfig config;
18+
config.max_new_tokens = 20;
1719
config.do_sample = true;
1820
config.temperature = 0.1;
1921
EXPECT_NO_THROW(config.validate());
2022
}
2123

2224
TEST(GenerationConfigTest, invalid_top_p) {
2325
ov::genai::GenerationConfig config;
26+
config.max_new_tokens = 20;
2427
config.do_sample = true;
2528
config.top_p = -0.5;
2629
EXPECT_THROW(config.validate(), ov::Exception);
@@ -30,13 +33,15 @@ TEST(GenerationConfigTest, invalid_top_p) {
3033

3134
TEST(GenerationConfigTest, valid_top_p) {
3235
ov::genai::GenerationConfig config;
36+
config.max_new_tokens = 20;
3337
config.do_sample = true;
3438
config.top_p = 0.1;
3539
EXPECT_NO_THROW(config.validate());
3640
}
3741

3842
TEST(GenerationConfigTest, invalid_repeatition_penalty) {
3943
ov::genai::GenerationConfig config;
44+
config.max_new_tokens = 20;
4045
config.do_sample = true;
4146
config.repetition_penalty = -3.0;
4247
EXPECT_THROW(config.validate(), ov::Exception);
@@ -46,15 +51,17 @@ TEST(GenerationConfigTest, invalid_repeatition_penalty) {
4651

4752
TEST(GenerationConfigTest, valid_repeatition_penalty) {
4853
ov::genai::GenerationConfig config;
54+
config.max_new_tokens = 20;
4955
config.do_sample = true;
5056
config.repetition_penalty = 1.8;
5157
EXPECT_NO_THROW(config.validate());
52-
config.repetition_penalty = 0.0;
58+
config.repetition_penalty = 0.1;
5359
EXPECT_NO_THROW(config.validate());
5460
}
5561

5662
TEST(GenerationConfigTest, invalid_presence_penalty) {
5763
ov::genai::GenerationConfig config;
64+
config.max_new_tokens = 20;
5865
config.do_sample = true;
5966
config.presence_penalty = 3.0;
6067
EXPECT_THROW(config.validate(), ov::Exception);
@@ -64,6 +71,7 @@ TEST(GenerationConfigTest, invalid_presence_penalty) {
6471

6572
TEST(GenerationConfigTest, valid_presence_penalty) {
6673
ov::genai::GenerationConfig config;
74+
config.max_new_tokens = 20;
6775
config.do_sample = true;
6876
config.presence_penalty = 1.8;
6977
EXPECT_NO_THROW(config.validate());
@@ -73,6 +81,7 @@ TEST(GenerationConfigTest, valid_presence_penalty) {
7381

7482
TEST(GenerationConfigTest, invalid_frequency_penalty) {
7583
ov::genai::GenerationConfig config;
84+
config.max_new_tokens = 20;
7685
config.do_sample = true;
7786
config.frequency_penalty = 3.0;
7887
EXPECT_THROW(config.validate(), ov::Exception);
@@ -82,6 +91,7 @@ TEST(GenerationConfigTest, invalid_frequency_penalty) {
8291

8392
TEST(GenerationConfigTest, valid_frequency_penalty) {
8493
ov::genai::GenerationConfig config;
94+
config.max_new_tokens = 20;
8595
config.do_sample = true;
8696
config.frequency_penalty = 1.8;
8797
EXPECT_NO_THROW(config.validate());

‎tests/python_tests/test_preemption.py

+7-6
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
# Copyright (C) 2018-2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
import sys
45
import pytest
5-
from dataclasses import dataclass
6-
from typing import List
76

8-
from openvino_genai.py_continuous_batching import GenerationConfig
7+
from openvino_genai import GenerationConfig
98
from common import get_model_and_tokenizer, save_ov_model_from_optimum, generate_and_compare_with_reference_text, \
109
DEFAULT_SCHEDULER_CONFIG, get_scheduler_config, run_test_pipeline, get_models_list, get_beam_search, get_greedy, \
1110
get_multinomial_all_parameters, get_multinomial_temperature_and_num_return_sequence, \
@@ -20,11 +19,11 @@ def get_greedy_seq_len_300() -> GenerationConfig:
2019

2120
def get_beam_search_seq_len_300() -> GenerationConfig:
2221
generation_config = GenerationConfig()
23-
generation_config.num_groups = 3
24-
generation_config.group_size = 2
22+
generation_config.num_beam_groups = 3
23+
generation_config.num_beams = 6
2524
generation_config.max_new_tokens = 300
2625
generation_config.num_return_sequences = 3
27-
generation_config.num_return_sequences = generation_config.num_groups * generation_config.group_size
26+
generation_config.num_return_sequences = generation_config.num_beams
2827
return generation_config
2928

3029
scheduler_params_list = [({"num_kv_blocks": 2, "block_size": 32, "dynamic_split_fuse": True, "max_num_batched_tokens": 256, "max_num_seqs": 256}, get_greedy()),
@@ -56,6 +55,7 @@ def test_preemption(tmp_path, params):
5655
# todo: Anastasiia Pnevskaya: fix the test because it is hanging according max_new_tokens = std::numeric_limits<std::size_t>::max()
5756
@pytest.mark.parametrize("dynamic_split_fuse", [True, False])
5857
@pytest.mark.precommit
58+
@pytest.mark.xfail(raises=AssertionError, reason="assert ref_text == ov_text fails in CI.", condition=sys.platform in ["win32", "darwin"], strict=True)
5959
def test_preemption_with_multinomial(tmp_path, dynamic_split_fuse):
6060
generation_configs = multinomial_params.generation_config
6161
for config in generation_configs:
@@ -99,6 +99,7 @@ def test_preemption_with_multinomial(tmp_path, dynamic_split_fuse):
9999

100100
@pytest.mark.parametrize("dynamic_split_fuse", [True, False])
101101
@pytest.mark.precommit
102+
@pytest.mark.xfail(reason="assert ref_text == ov_text fails", condition=sys.platform in ["win32", "darwin"])
102103
def test_preemption_with_multinomial_n_seq(tmp_path, dynamic_split_fuse):
103104
generation_configs = multinomial_params_n_seq.generation_config
104105
for config in generation_configs:

0 commit comments

Comments
 (0)