Skip to content

Commit 17537d7

Browse files
andreyanufrandreyanufr
committed
Merge remote-tracking branch 'upstream/master' into aanuf/fix_gpu_generation_for_text_eval
2 parents 546555f + 5d986b7 commit 17537d7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+646
-334
lines changed

‎.github/workflows/linux.yml

+33-1
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,7 @@ jobs:
435435
cmd: 'tests/python_tests/samples'
436436
runner: 'aks-linux-8-cores-64gb'
437437

438-
needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ]
438+
needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_build_nodejs ]
439439
timeout-minutes: 60
440440
defaults:
441441
run:
@@ -465,6 +465,14 @@ jobs:
465465
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_archive_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}"
466466
path: ${{ env.INSTALL_DIR }}
467467
merge-multiple: true
468+
469+
- name: Download GenAI JS Bildings Artifacts
470+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
471+
if: ${{ 'LLM' == matrix.test.name }}
472+
with:
473+
name: genai_nodejs_bindings
474+
path: ${{ env.SRC_DIR }}/src/js/bin
475+
merge-multiple: true
468476

469477
- name: Extract Artifacts
470478
run: |
@@ -484,11 +492,35 @@ jobs:
484492
requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt"
485493
local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels
486494

495+
- name: Setup Node
496+
uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
497+
if: ${{ 'LLM' == matrix.test.name }}
498+
with:
499+
node-version: 21
500+
501+
- name: Install GenAI NPM package
502+
working-directory: ${{ env.SRC_DIR }}/src/js
503+
if: ${{ 'LLM' == matrix.test.name }}
504+
run: |
505+
npm install --verbose
506+
rm -rf node_modules/openvino-node/bin
507+
cp -R ${{ env.INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin
508+
cd ${{ env.SRC_DIR }}/samples/js/text_generation
509+
510+
- name: Install NPM dependencies for samples
511+
working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation
512+
if: ${{ 'LLM' == matrix.test.name }}
513+
run: |
514+
npm install --verbose
515+
rm -rf node_modules/openvino-node/bin
516+
cp -R ${{ env.INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin
517+
487518
- name: Test Samples (Python and C++)
488519
run: python -m pytest -vs ${{ env.SRC_DIR }}/${{ matrix.test.cmd }} -m "${{ matrix.test.marker }}"
489520
env:
490521
LD_LIBRARY_PATH: "${{ env.INSTALL_DIR }}/runtime/lib/intel64:${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb/lib:$LD_LIBRARY_PATH" # Required for C++ samples
491522
SAMPLES_PY_DIR: "${{ env.INSTALL_DIR }}/samples/python"
523+
SAMPLES_JS_DIR: "${{ env.SRC_DIR }}/samples/js"
492524
SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
493525
SAMPLES_C_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
494526

‎.github/workflows/mac.yml

+36-1
Original file line numberDiff line numberDiff line change
@@ -435,14 +435,15 @@ jobs:
435435
- name: 'Whisper'
436436
marker: 'whisper'
437437
cmd: 'tests/python_tests/samples'
438-
needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples ]
438+
needs: [ openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_build_nodejs ]
439439
timeout-minutes: 45
440440
defaults:
441441
run:
442442
shell: bash
443443
runs-on: macos-13
444444
env:
445445
INSTALL_DIR: ${{ github.workspace }}/ov
446+
LATEST_OV_DIR: ${{ github.workspace }}/ov_latest
446447
SRC_DIR: ${{ github.workspace }}/src
447448
BUILD_DIR: ${{ github.workspace }}/build
448449

@@ -460,6 +461,20 @@ jobs:
460461
path: ${{ env.INSTALL_DIR }}
461462
merge-multiple: true
462463

464+
- name: Download OpenVINO JS Bildings Artifacts
465+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
466+
with:
467+
name: ${{ needs.openvino_download.outputs.ov_latest_artifact_name }}
468+
path: ${{ env.LATEST_OV_DIR }}
469+
merge-multiple: true
470+
471+
- name: Download GenAI JS Bildings Artifacts
472+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
473+
with:
474+
name: genai_nodejs_bindings
475+
path: ${{ env.SRC_DIR }}/src/js/bin
476+
merge-multiple: true
477+
463478
- name: Setup Python ${{ env.PYTHON_VERSION }}
464479
uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
465480
with:
@@ -481,12 +496,32 @@ jobs:
481496
- name: Fix C++ samples permissions
482497
run: chmod +x ${{ env.INSTALL_DIR }}/samples_bin/*
483498

499+
- name: Setup NodeJS
500+
uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
501+
with:
502+
node-version: 21
503+
504+
- name: Install GenAI NPM package
505+
working-directory: ${{ env.SRC_DIR }}/src/js
506+
run: |
507+
npm install --verbose
508+
rm -rf node_modules/openvino-node/bin
509+
cp -R ${{ env.LATEST_OV_DIR }}/openvino_js_package node_modules/openvino-node/bin
510+
511+
- name: Install NPM dependencies for samples
512+
working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation
513+
run: |
514+
npm install --verbose
515+
rm -rf node_modules/openvino-node/bin
516+
cp -R ${{ env.LATEST_OV_DIR }}/openvino_js_package node_modules/openvino-node/bin
517+
484518
- name: Test Samples (Python and C++)
485519
run: |
486520
source ${{ env.INSTALL_DIR }}/setupvars.sh
487521
python -m pytest -vs ${{ env.SRC_DIR }}/${{ matrix.test.cmd }} -m "${{ matrix.test.marker }}"
488522
env:
489523
SAMPLES_PY_DIR: "${{ env.INSTALL_DIR }}/samples/python"
524+
SAMPLES_JS_DIR: "${{ env.SRC_DIR }}/samples/js"
490525
SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
491526
SAMPLES_C_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
492527

‎.github/workflows/windows.yml

+29-2
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ jobs:
473473
marker: 'image_generation'
474474
cmd: 'tests/python_tests/samples'
475475
runner: 'aks-win-8-cores-16gb'
476-
needs: [ openvino_download, genai_build_cpack, genai_build_wheel, genai_build_samples ]
476+
needs: [ openvino_download, genai_build_cpack, genai_build_wheel, genai_build_samples, genai_build_nodejs ]
477477
timeout-minutes: 60
478478
defaults:
479479
run:
@@ -499,7 +499,14 @@ jobs:
499499
pattern: "{${{ needs.openvino_download.outputs.ov_artifact_name }},genai_cpack_${{ matrix.build-type }},genai_samples_${{ matrix.build-type }},genai_wheels}"
500500
path: ${{ env.INSTALL_DIR }}
501501
merge-multiple: true
502-
502+
503+
- name: Download GenAI JS Bildings Artifacts
504+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
505+
with:
506+
name: genai_nodejs_bindings
507+
path: ${{ env.SRC_DIR }}/src/js/bin
508+
merge-multiple: true
509+
503510
- name: Setup Python ${{ env.PYTHON_VERSION }}
504511
uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
505512
with:
@@ -517,11 +524,31 @@ jobs:
517524
requirements_files: "${{ env.SRC_DIR }}/samples/requirements.txt"
518525
local_wheel_dir: ${{ env.INSTALL_DIR }}/wheels
519526

527+
- name: Setup NodeJS
528+
uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
529+
with:
530+
node-version: 21
531+
532+
- name: Install GenAI NPM package
533+
working-directory: ${{ env.SRC_DIR }}/src/js
534+
run: |
535+
npm install --verbose
536+
Remove-Item -Recurse -Force node_modules/openvino-node/bin
537+
Copy-Item -Recurse ${{ env.INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin
538+
539+
- name: Install NPM dependencies for samples
540+
working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation
541+
run: |
542+
npm install --verbose
543+
Remove-Item -Recurse -Force node_modules/openvino-node/bin
544+
Copy-Item -Recurse ${{ env.INSTALL_DIR }}/openvino_js_package node_modules/openvino-node/bin
545+
520546
- name: Test Samples (Python and C++)
521547
run: python -m pytest -vs ${{ env.SRC_DIR }}/${{ matrix.test.cmd }} -m "${{ matrix.test.marker }}"
522548
env:
523549
PATH: "${{ env.INSTALL_DIR }}/runtime/bin/intel64/${{ matrix.build-type }};${{ env.INSTALL_DIR }}/runtime/3rdparty/tbb/bin;%PATH%" # Required for C++ samples
524550
SAMPLES_PY_DIR: "${{ env.INSTALL_DIR }}/samples/python"
551+
SAMPLES_JS_DIR: "${{ env.SRC_DIR }}/samples/js"
525552
SAMPLES_CPP_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
526553
SAMPLES_C_DIR: "${{ env.INSTALL_DIR }}/samples_bin"
527554

‎samples/js/text_generation/greedy_causal_lm.js

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import { LLMPipeline } from 'openvino-genai-node';
2+
3+
main();
4+
5+
async function main() {
6+
const modelPath = process.argv[2];
7+
const prompt = process.argv[3];
8+
9+
if (!modelPath) {
10+
console.error('Please specify path to model directory\n'
11+
+ 'Run command must be: `node chat_sample.js *path_to_model_dir* *prompt*`');
12+
process.exit(1);
13+
}
14+
if (!modelPath) {
15+
console.error('Please specify prompt\n'
16+
+ 'Run command must be: `node chat_sample.js *path_to_model_dir* *prompt*`');
17+
process.exit(1);
18+
}
19+
20+
const device = 'CPU'; // GPU can be used as well
21+
const pipe = await LLMPipeline(modelPath, device);
22+
23+
const config = { 'max_new_tokens': 100 };
24+
const result = await pipe.generate(prompt, config);
25+
26+
console.log(result);
27+
}

‎site/docs/guides/chat-scenario.mdx

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
---
2+
sidebar_position: 2
3+
title: Chat Scenario
4+
---
5+
6+
# Using OpenVINO GenAI in Chat Scenario
7+
8+
For chat applications, OpenVINO GenAI provides special optimizations to maintain conversation context and improve performance using KV-cache.
9+
10+
Refer to the [Stateful Models vs Stateless Models](/docs/concepts/stateful-vs-stateless-models) for more information about KV-cache.
11+
12+
:::tip
13+
Use `start_chat()` and `finish_chat()` to properly manage the chat session's KV-cache. This improves performance by reusing context between messages.
14+
:::
15+
16+
:::info
17+
Chat mode is supported for both `LLMPipeline` and `VLMPipeline`.
18+
:::
19+
20+
A simple chat example (with grouped beam search decoding):
21+
22+
<LanguageTabs>
23+
<TabItemPython>
24+
```python showLineNumbers
25+
import openvino_genai as ov_genai
26+
pipe = ov_genai.LLMPipeline(model_path, 'CPU')
27+
28+
config = {'max_new_tokens': 100, 'num_beam_groups': 3, 'num_beams': 15, 'diversity_penalty': 1.5}
29+
pipe.set_generation_config(config)
30+
31+
# highlight-next-line
32+
pipe.start_chat()
33+
while True:
34+
try:
35+
prompt = input('question:\n')
36+
except EOFError:
37+
break
38+
answer = pipe.generate(prompt)
39+
print('answer:\n')
40+
print(answer)
41+
print('\n----------\n')
42+
# highlight-next-line
43+
pipe.finish_chat()
44+
```
45+
</TabItemPython>
46+
<TabItemCpp>
47+
```cpp showLineNumbers
48+
#include "openvino/genai/llm_pipeline.hpp"
49+
#include <iostream>
50+
51+
int main(int argc, char* argv[]) {
52+
std::string prompt;
53+
54+
std::string model_path = argv[1];
55+
ov::genai::LLMPipeline pipe(model_path, "CPU");
56+
57+
ov::genai::GenerationConfig config;
58+
config.max_new_tokens = 100;
59+
config.num_beam_groups = 3;
60+
config.num_beams = 15;
61+
config.diversity_penalty = 1.0f;
62+
63+
// highlight-next-line
64+
pipe.start_chat();
65+
std::cout << "question:\n";
66+
while (std::getline(std::cin, prompt)) {
67+
std::cout << "answer:\n";
68+
auto answer = pipe.generate(prompt, config);
69+
std::cout << answer << std::endl;
70+
std::cout << "\n----------\n"
71+
"question:\n";
72+
}
73+
// highlight-next-line
74+
pipe.finish_chat();
75+
}
76+
```
77+
</TabItemCpp>
78+
</LanguageTabs>
79+
80+
:::info
81+
For more information, refer to the [Python](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/python/text_generation/chat_sample.py) and [C++](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/text_generation/chat_sample.cpp) chat samples.
82+
:::

‎site/docs/guides/model-preparation/convert-to-openvino.mdx

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import UseCasesNote from './_use_cases_note.mdx';
88

99
# Convert Models to OpenVINO Format
1010

11-
This page explains how to convert various generative AI models from Hugging Face and ModelScope to OpenVINO IR format. Refer to the [Supported Models](../../supported-models/index.mdx) for a list of available models.
11+
This page explains how to convert various generative AI models from [Hugging Face](https://huggingface.co/) and [ModelScope](https://modelscope.cn/) to OpenVINO IR format.
12+
Refer to the [Supported Models](../../supported-models/index.mdx) for a list of available models.
1213

1314
For downloading pre-converted models, see [Download Pre-Converted OpenVINO Models](./download-openvino-models.mdx).
1415

‎site/docs/guides/model-preparation/download-openvino-models.mdx

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import UseCasesNote from './_use_cases_note.mdx';
88
# Download Pre-Converted OpenVINO Models
99

1010
OpenVINO GenAI allows to run different generative AI models (see [Supported Models](../../supported-models/index.mdx)).
11-
While you can convert models from other frameworks (see [Convert Models to OpenVINO Format](./convert-to-openvino.mdx)), using pre-converted models can save time and effort.
11+
While you can convert models from other frameworks (see [Convert Models to OpenVINO Format](./convert-to-openvino.mdx)), using pre-converted models from [Hugging Face](https://huggingface.co/) and [ModelScope](https://modelscope.cn/) can save time and effort.
1212

1313
## Download from Hugging Face
1414

0 commit comments

Comments
 (0)