Skip to content

Commit 747c5d2

Browse files
Wovchenayatarkanilya-lavrenovTolyaTalamanovwgzintel
authored
Merge releases/2024/5 into master (openvinotoolkit#1168)
Co-authored-by: yatarkan <yaroslav.tarkan@intel.com> Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com> Co-authored-by: TolyaTalamanov <anatoliy.talamanov@intel.com> Co-authored-by: wgzintel <guozhong.wang@intel.com> Co-authored-by: Sergey Lyalin <sergey.lyalin@gmail.com>
1 parent 809d93e commit 747c5d2

File tree

12 files changed

+420
-147
lines changed

12 files changed

+420
-147
lines changed

‎.github/actions/build_app/action.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: 'Build App'
2+
inputs:
3+
ov_dir:
4+
description: 'Directory where OpenVINO is installed'
5+
default: './ov'
6+
required: false
7+
build_dir:
8+
description: 'Directory where the app is built'
9+
default: './build'
10+
required: false
11+
build_target:
12+
description: 'Target to build'
13+
default: ''
14+
required: false
15+
runs:
16+
using: "composite"
17+
steps:
18+
- name: Build app
19+
shell: bash
20+
run: |
21+
source ${{ inputs.ov_dir }}/setupvars.sh
22+
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ inputs.build_dir }}
23+
cmake --build ${{ inputs.build_dir }} --config Release ${{ inputs.build_target && format('--target {0}', inputs.build_target) || '' }} -j

‎.github/actions/install_openvino/action.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: 'Install OpenVINO'
2+
inputs:
3+
ov_link:
4+
description: 'URL to download OpenVINO'
5+
required: true
6+
ov_dir:
7+
description: 'Directory to install OpenVINO'
8+
default: './ov'
9+
required: false
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: 'Install OpenVINO'
14+
shell: bash
15+
run: |
16+
mkdir ${{ inputs.ov_dir }}
17+
curl ${{ inputs.ov_link }} | tar --directory ${{ inputs.ov_dir }} --strip-components 1 -xz
18+
sudo ${{ inputs.ov_dir }}/install_dependencies/install_openvino_dependencies.sh

‎.github/actions/install_python_deps/action.yml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
name: 'Install Python Dependencies'
2+
inputs:
3+
ov_dir:
4+
description: 'Directory where OpenVINO is installed'
5+
default: './ov'
6+
required: false
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: Install Python dependencies
11+
shell: bash
12+
run: |
13+
source ${{ inputs.ov_dir }}/setupvars.sh
14+
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
15+
python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt

‎.github/scripts/generate_reference_llava.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import argparse
2+
from pathlib import Path
3+
from optimum.intel.openvino import OVModelForVisualCausalLM
4+
from transformers import AutoProcessor
5+
from PIL import Image
6+
7+
IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png", ".bmp"]
8+
9+
10+
def main(model_path: str, images_path: str):
11+
print(f"Selected model: {model_path}\n")
12+
13+
if Path(images_path).is_file():
14+
image_files = [Path(images_path)]
15+
else:
16+
image_files = sorted(
17+
[f for f in Path(images_path).glob("*") if f.is_file() and f.suffix.lower() in IMAGE_EXTENSIONS],
18+
key=lambda x: x.name
19+
)
20+
21+
if not image_files:
22+
raise FileNotFoundError(f"No images found in '{images_path}' directory. Supported formats: {IMAGE_EXTENSIONS}")
23+
24+
images = []
25+
for file in image_files:
26+
images.append(
27+
Image.open(file).convert("RGB")
28+
)
29+
30+
print("Images:", image_files)
31+
32+
model = OVModelForVisualCausalLM.from_pretrained(model_path, trust_remote_code=True)
33+
processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
34+
35+
conversation = [{
36+
"role": "user",
37+
"content": [
38+
*[{"type": "image"} for _ in images],
39+
{"type": "text", "text": "Describe the images."},
40+
],
41+
}]
42+
43+
prompt = processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
44+
print(prompt)
45+
inputs = processor(text=[prompt], images=images, return_tensors="pt")
46+
result = model.generate(**inputs, max_new_tokens=100, do_sample=False)
47+
decoded = processor.tokenizer.batch_decode(result[:, inputs["input_ids"].shape[1]:], skip_special_tokens=True)[0]
48+
print(decoded)
49+
with open("ref.txt", "w") as f:
50+
f.write(f"question:\n{decoded}\n----------\nquestion:\n")
51+
52+
53+
if __name__ == "__main__":
54+
parser = argparse.ArgumentParser()
55+
parser.add_argument("-m", "--model_path", type=str, required=True, help="Path to the model.")
56+
parser.add_argument("-i", "--images_path", type=str, required=True, help="Path to the directory with images.")
57+
args = parser.parse_args()
58+
main(args.model_path, args.images_path)

‎.github/workflows/causal_lm_cpp.yml

+50-46
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,7 @@ jobs:
713713
echo "Chat sample python" passed
714714
715715
716-
visual_language_chat_sample-ubuntu:
716+
visual_language_chat_sample-ubuntu-minicpm_v2_6:
717717
runs-on: ubuntu-22.04-16-cores
718718
steps:
719719
- uses: actions/checkout@v4
@@ -722,21 +722,13 @@ jobs:
722722
- uses: actions/setup-python@v4
723723
with:
724724
python-version: 3.11
725-
- name: Install OpenVINO
726-
run: |
727-
mkdir ./ov/
728-
curl ${{ env.l_u22_ov_link }} | tar --directory ./ov/ --strip-components 1 -xz
729-
sudo ./ov/install_dependencies/install_openvino_dependencies.sh
730-
- name: Build app
731-
run: |
732-
source ./ov/setupvars.sh
733-
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
734-
cmake --build ./build/ --config Release --target visual_language_chat py_openvino_genai -j
735-
- name: Install dependencies
736-
run: |
737-
source ./ov/setupvars.sh
738-
python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly
739-
python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt opencv-python
725+
- uses: ./.github/actions/install_openvino
726+
with:
727+
ov_link: ${{ env.l_u22_ov_link }}
728+
- uses: ./.github/actions/build_app
729+
with:
730+
build_target: 'visual_language_chat py_openvino_genai'
731+
- uses: ./.github/actions/install_python_deps
740732
- name: Download and convert tiny-random-minicpmv-2_6 model and an image
741733
run: |
742734
python -m pip install git+https://github.com/eaidova/optimum-intel.git@ea/minicpmv
@@ -764,13 +756,6 @@ jobs:
764756
&& ./build/samples/cpp/visual_language_chat/visual_language_chat ./tiny-random-minicpmv-2_6/ ./images/
765757
<<< $'Describe the images?' | tee cpp.txt
766758
timeout-minutes: 2
767-
- name: Encode cpp.txt with Python encoding instead of terminal one
768-
shell: python
769-
run: |
770-
with open("cpp.txt", "rb") as f:
771-
content = f.read().decode("utf-8", "replace")
772-
with open("cpp.txt", "wb") as f:
773-
f.write(content.encode("utf-8"))
774759
- name: Run visual_language_chat Python sample - tiny-random-minicpmv-2_6
775760
run: >
776761
set -o pipefail
@@ -779,6 +764,13 @@ jobs:
779764
<<< $'Describe the images?' | tee py.txt
780765
env:
781766
PYTHONPATH: "./build/"
767+
- name: Encode cpp.txt with Python encoding instead of terminal one
768+
shell: python
769+
run: |
770+
with open("cpp.txt", "rb") as f:
771+
content = f.read().decode("utf-8", "replace")
772+
with open("cpp.txt", "wb") as f:
773+
f.write(content.encode("utf-8"))
782774
- run: diff cpp.txt py.txt
783775
- name: Run visual_language_chat C++ sample with 2 prompts - tiny-random-minicpmv-2_6
784776
run: >
@@ -803,39 +795,51 @@ jobs:
803795
with open("cpp2.txt", "wb") as f:
804796
f.write(content.encode("utf-8"))
805797
- run: diff cpp2.txt py2.txt
806-
- name: Download and convert LLaVa 1.5 model and an image
807-
run: |
808-
source ./ov/setupvars.sh
809-
optimum-cli export openvino --model llava-hf/llava-1.5-7b-hf ./llava_1_5_7b_ov/
810-
wget https://llava-vl.github.io/static/images/monalisa.jpg
811-
- name: Run visual_language_chat C++ sample - LLaVa 1.5
812-
run: >
813-
source ./ov/setupvars.sh
814-
&& ./build/samples/cpp/visual_language_chat/visual_language_chat ./llava_1_5_7b_ov/ monalisa.jpg
815-
<<< $'Who drew this painting?\nWhen did the painter live?'
816-
timeout-minutes: 4
817-
- name: Download and convert LLaVa-Next model
818-
run: |
819-
source ./ov/setupvars.sh
820-
optimum-cli export openvino --model llava-hf/llava-v1.6-mistral-7b-hf ./llava_v1_6_mistral_7b_ov/
821-
- name: Run visual_language_chat C++ sample - LLaVa-Next
822-
run: >
823-
source ./ov/setupvars.sh
824-
&& ./build/samples/cpp/visual_language_chat/visual_language_chat ./llava_v1_6_mistral_7b_ov/ monalisa.jpg
825-
<<< $'Who drew this painting?\nWhen did the painter live?'
826-
timeout-minutes: 4
798+
799+
visual_language_chat_sample-ubuntu-llava_1_5:
800+
uses: ./.github/workflows/job_vlm_sample_llava.yml
801+
with:
802+
model_id: llava-hf/llava-1.5-7b-hf
803+
model_dir: llava_1_5_7b_ov
804+
805+
visual_language_chat_sample-ubuntu-llava_next:
806+
uses: ./.github/workflows/job_vlm_sample_llava.yml
807+
with:
808+
model_id: llava-hf/llava-v1.6-mistral-7b-hf
809+
model_dir: llava_v1_6_mistral_7b_ov
810+
811+
visual_language_chat_sample-ubuntu-internvl2:
812+
runs-on: ubuntu-22.04-16-cores
813+
steps:
814+
- uses: actions/checkout@v4
815+
with:
816+
submodules: recursive
817+
- uses: actions/setup-python@v4
818+
with:
819+
python-version: 3.11
820+
- uses: ./.github/actions/install_openvino
821+
with:
822+
ov_link: ${{ env.l_u22_ov_link }}
823+
- uses: ./.github/actions/build_app
824+
with:
825+
build_target: 'visual_language_chat py_openvino_genai'
826+
- uses: ./.github/actions/install_python_deps
827827
- name: Download and convert InternVL2 model
828828
run: |
829+
# Lowering transformers version, workaround for https://huggingface.co/OpenGVLab/InternVL2-1B/discussions/7
830+
python -m pip install -U "transformers<4.45.0"
829831
source ./ov/setupvars.sh
830832
optimum-cli export openvino --model OpenGVLab/InternVL2-4B ./internvl2_4b_ov/ --trust-remote-code
833+
- name: Download images
834+
run: |
835+
wget https://llava-vl.github.io/static/images/monalisa.jpg
831836
- name: Run visual_language_chat C++ sample - InternVL2
832837
run: >
833838
source ./ov/setupvars.sh
834839
&& ./build/samples/cpp/visual_language_chat/visual_language_chat ./internvl2_4b_ov/ monalisa.jpg
835840
<<< $'Who drew this painting?\nWhen did the painter live?'
836841
timeout-minutes: 4
837842

838-
839843
cpp-continuous-batching-ubuntu:
840844
runs-on: ubuntu-20.04-8-cores
841845
defaults:
@@ -975,7 +979,7 @@ jobs:
975979
cpp-greedy_causal_lm-Qwen-7B-Chat, cpp-beam_search_causal_lm-Qwen1_5-7B-Chat, cpp-beam_search_causal_lm-Phi-2,
976980
cpp-beam_search_causal_lm-notus-7b-v1, cpp-speculative_decoding_lm-ubuntu, cpp-prompt_lookup_decoding_lm-ubuntu,
977981
cpp-Phi-1_5, cpp-greedy_causal_lm-redpajama-3b-chat, cpp-chat_sample-ubuntu, cpp-continuous-batching-ubuntu,
978-
visual_language_chat_sample-ubuntu,
982+
visual_language_chat_sample-ubuntu-minicpm_v2_6, visual_language_chat_sample-ubuntu-llava_1_5, visual_language_chat_sample-ubuntu-llava_next, visual_language_chat_sample-ubuntu-internvl2,
979983
cpp-continuous-batching-windows, cpp-continuous-batching-macos]
980984
if: ${{ always() }}
981985
runs-on: ubuntu-latest

‎.github/workflows/job_vlm_sample_llava.yml

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: visual_language_chat sample - LLaVA
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
model_id:
7+
required: true
8+
type: string
9+
model_dir:
10+
required: true
11+
type: string
12+
13+
env:
14+
l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz
15+
jobs:
16+
visual_language_chat_sample-ubuntu-llava:
17+
runs-on: ubuntu-22.04-16-cores
18+
steps:
19+
- uses: actions/checkout@v4
20+
with:
21+
submodules: recursive
22+
- uses: actions/setup-python@v4
23+
with:
24+
python-version: 3.11
25+
- uses: ./.github/actions/install_openvino
26+
with:
27+
ov_link: ${{ env.l_u22_ov_link }}
28+
- uses: ./.github/actions/build_app
29+
with:
30+
build_target: 'visual_language_chat py_openvino_genai'
31+
- uses: ./.github/actions/install_python_deps
32+
- name: Download and convert model
33+
run: |
34+
source ./ov/setupvars.sh
35+
optimum-cli export openvino --model ${{ inputs.model_id }} ./${{ inputs.model_dir }}
36+
- name: Download images
37+
run: |
38+
wget https://llava-vl.github.io/static/images/monalisa.jpg
39+
- name: Run visual_language_chat C++ sample
40+
run: >
41+
source ./ov/setupvars.sh
42+
&& ./build/samples/cpp/visual_language_chat/visual_language_chat ./${{ inputs.model_dir }} monalisa.jpg
43+
<<< $'Who drew this painting?\nWhen did the painter live?'
44+
timeout-minutes: 4

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
!__init__.py
66
!__main__.py
77

8+
# don't skip GitHub Actions files and directories
9+
!.github/**
10+
811
# developer tools
912
*.idea
1013
.vscode

‎src/cpp/src/llm_pipeline_static.cpp

+30-4
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,26 @@
2020

2121
namespace {
2222

23+
uint32_t align_to(uint32_t value, uint32_t alignment) {
24+
return (value + alignment - 1) & ~(alignment - 1);
25+
}
26+
27+
enum class GenerateHint {
28+
FAST_COMPILE,
29+
BEST_PERF
30+
};
31+
32+
GenerateHint str_to_hint(const std::string& str) {
33+
if (str == "FAST_COMPILE") {
34+
return GenerateHint::FAST_COMPILE;
35+
}
36+
if (str == "BEST_PERF") {
37+
return GenerateHint::BEST_PERF;
38+
}
39+
OPENVINO_THROW("Unsupported \"GENERATE_HINT\" provided: " +
40+
str + ". Please select either \"FAST_COMPILE\" or \"BEST_PERF\".");
41+
}
42+
2343
std::shared_ptr<ov::Model> cvt_kvcache_to_fp16(const std::shared_ptr<ov::Model>& model) {
2444
ov::preprocess::PrePostProcessor ppp(model);
2545

@@ -275,8 +295,12 @@ ov::AnyMap get_default_prefill_config(const std::shared_ptr<ov::Model>& model,
275295
}
276296

277297
ov::AnyMap get_default_generate_config(const std::shared_ptr<ov::Model>& model,
278-
const std::optional<NPUDesc>& npudesc) {
298+
const std::optional<NPUDesc>& npudesc,
299+
const GenerateHint hint) {
279300
auto config = get_default_common_config(model);
301+
if (hint == GenerateHint::BEST_PERF) {
302+
config.emplace("NPUW_ONLINE_PIPELINE", "NONE");
303+
}
280304
// NB: Unconditionally set for generation model
281305
config.emplace("NPUW_DQ", "YES");
282306
if (npudesc.has_value() && npudesc->arch == "4000") {
@@ -404,8 +428,8 @@ void StaticLLMPipeline::setupAndCompileModels(
404428
m_prefill_model = m_kvcache_model->clone();
405429
m_prefill_model->set_friendly_name(m_kvcache_model->get_friendly_name() + "_prefill");
406430
// (7) Reshape both models to static shape
407-
const uint32_t kMaxPromptLen = pop_int_and_cast(properties, "MAX_PROMPT_LEN").value_or(1024u);
408-
const uint32_t kMinResponseLen = pop_int_and_cast(properties, "MIN_RESPONSE_LEN").value_or(128u);
431+
const uint32_t kMaxPromptLen = align_to(pop_int_and_cast(properties, "MAX_PROMPT_LEN").value_or(1024u), 64u);
432+
const uint32_t kMinResponseLen = align_to(pop_int_and_cast(properties, "MIN_RESPONSE_LEN").value_or(128u), 64u);
409433
KVAxesPosition axes = get_kv_axes(get_model_type_from_json(models_path / "config.json"));
410434
m_kvcache_desc = KVCacheDesc { kMaxPromptLen, kMaxPromptLen + kMinResponseLen, 0u, axes.seq_len };
411435
reshape_to_static(m_prefill_model, m_kvcache_desc.max_prompt_size, m_kvcache_desc.max_prompt_size, axes);
@@ -414,8 +438,10 @@ void StaticLLMPipeline::setupAndCompileModels(
414438
auto prefill_config = pop_or_default(
415439
properties, "PREFILL_CONFIG", get_default_prefill_config(m_prefill_model, npudesc)
416440
);
441+
// NB: GENERATE_HINT is only applicable for default generate config!
442+
auto generate_hint = str_to_hint(pop_or_default<std::string>(properties, "GENERATE_HINT", "FAST_COMPILE"));
417443
auto generate_config = pop_or_default(
418-
properties, "GENERATE_CONFIG", get_default_generate_config(m_kvcache_model, npudesc)
444+
properties, "GENERATE_CONFIG", get_default_generate_config(m_kvcache_model, npudesc, generate_hint)
419445
);
420446
merge_config_with(prefill_config, properties);
421447
merge_config_with(generate_config, properties);

0 commit comments

Comments
 (0)