Merge releases/2024/2 into master (openvinotoolkit#497)

Author: Wovchena

.github/workflows/causal_lm_cpp.yml

@@ -19,19 +19,21 @@ jobs:
         with:
           python-version: 3.8
       - run: mkdir ./ov/
-      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/linux/l_openvino_toolkit_ubuntu20_2024.2.0.dev20240524_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.dev20240529_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
       - run: sudo ./ov/install_dependencies/install_openvino_dependencies.sh
       - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
       - run: source ./ov/setupvars.sh && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
+      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt
+      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
       - run: source ./ov/setupvars.sh && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
       - run: ov/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace
         if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
       - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ov/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j && cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace
         if: ${{ 'Release' != matrix.build-type }}
-      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
-      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
-      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
-      - run: source ./ov/setupvars.sh && timeout 50s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
+      - run: source ./ov/setupvars.sh && timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
+      - run: source ./ov/setupvars.sh && timeout 25s ./ov/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0
+        if: ${{ 'Release' == matrix.build-type }}  # Python bindings can be built in Release only
 
   macos_genai_package:
     strategy:
@@ -50,19 +52,24 @@ jobs:
       - run: brew install coreutils scons
       - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
       - run: source ./ov/setupvars.sh && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
+      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
+      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt
+      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
       - run: source ./ov/setupvars.sh && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
       - run: ov/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace
         if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
-      - run: source ./ov/setupvars.sh && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
-        if: ${{ 'Release' == matrix.build-type }}
-      - run: source ./ov/setupvars.sh && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
-        if: ${{ 'Release' == matrix.build-type }}
-      - run: source ./ov/setupvars.sh && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
-        if: ${{ 'Release' == matrix.build-type }}
-      - run: source ./ov/setupvars.sh && timeout 50s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
-        if: ${{ 'Release' == matrix.build-type }}
+      - run: >
+          source ./ov/setupvars.sh
+          && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ov/samples/cpp/ -B ./samples\ build/
+          && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j
+          && cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace
+        if: ${{ 'Release' != matrix.build-type }}
+      - run: source ./ov/setupvars.sh && timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ ""
+      - run: source ./ov/setupvars.sh && timeout 25s ./ov/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0
+        if: ${{ 'Release' == matrix.build-type }}  # Python bindings can be built in Release only
 
   windows_genai_package:
+    if: false
     strategy:
       matrix:
         build-type: [Release, Debug]
@@ -79,20 +86,24 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/windows/w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64.zip
+      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/windows/w_openvino_toolkit_windows_2024.2.0.dev20240529_x86_64.zip
       - run: unzip ov.zip
       # Shorten the next setupvars calls.
-      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64
+      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240529_x86_64
       - run: call ov\setupvars.bat && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/
       - run: call ov\setupvars.bat && cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j
-      - run: call ov\setupvars.bat && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
-      - run: call ov\samples\cpp\build_samples_msvc.bat -i "${{ github.workspace }}/samples_install"
-        if: ${{ false && 'Release' == matrix.build-type }}  # build_samples enforces Release build
       - run: call ov\setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release
-        if: ${{ false && 'Release' == matrix.build-type }}
-      - run: call ov\setupvars.bat && python -m pip install --upgrade-strategy eager -r ./samples/cpp/requirements.txt
-        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt
       - run: call ov\setupvars.bat && optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
-        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ov
+      - run: call ov\samples\cpp\build_samples_msvc.bat -i "${{ github.workspace }}/samples_install"
+        if: ${{ 'Release' == matrix.build-type }}  # build_samples enforces Release build
+      - run: >
+          call ov\setupvars.bat
+          && cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ov/samples/cpp/ -B "samples build"
+          && cmake --build "samples build" --config ${{ matrix.build-type }} -j
+          && cmake --install "samples build" --config ${{ matrix.build-type }} --component samples_bin --prefix samples_install
+        if: ${{ 'Release' != matrix.build-type }}
       - run: call ov\setupvars.bat && "${{ github.workspace }}/samples_install/samples_bin/greedy_causal_lm" .\TinyLlama-1.1B-Chat-v1.0\ ""
-        if: ${{ false && 'Release' == matrix.build-type }}
+      - run: call ov\setupvars.bat && ./ov/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0
+        if: ${{ 'Release' == matrix.build-type }}  # Python bindings can be built in Release only

.github/workflows/genai_package.yml

@@ -20,7 +20,7 @@ jobs:
           python-version: 3.8
       - run: mkdir ./ov/
       # Install CentOS7 instead of Ubuntu to match PyPI distribution ABI.
-      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/linux/l_openvino_toolkit_centos7_2024.2.0.dev20240524_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
+      - run: curl https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/linux/l_openvino_toolkit_centos7_2024.2.0.dev20240529_x86_64.tgz | tar --directory ./ov/ --strip-components 1 -xz
       - run: sudo ./ov/install_dependencies/install_openvino_dependencies.sh
       - run: source ./ov/setupvars.sh && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
       - run: source ./ov/setupvars.sh && cmake --build ./build/ --config Release -j
@@ -72,10 +72,10 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc1/windows/w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64.zip
+      - run: curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/pre-release/2024.2.0rc2/windows/w_openvino_toolkit_windows_2024.2.0.dev20240529_x86_64.zip
       - run: unzip ov.zip
       # Shorten the next setupvars calls.
-      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240524_x86_64
+      - run: mklink /D ov w_openvino_toolkit_windows_2024.2.0.dev20240529_x86_64
       - run: call ./ov/setupvars.bat && cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
       - run: call ./ov/setupvars.bat && cmake --build ./build/ --config Release -j
       - run: call ./ov/setupvars.bat && python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./requirements-build.txt -r ./tests/python_tests/requirements.txt --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/pre-release --upgrade-strategy eager

.github/workflows/genai_python_lib.yml

@@ -54,7 +54,7 @@ jobs:
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_lcm_cpp
-          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 models/lcm_dreamshaper_v7/FP16
+          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 models/lcm_dreamshaper_v7/FP16
 
       - name: Build app
         working-directory: ${{ env.working_directory }}
@@ -99,7 +99,7 @@ jobs:
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_lcm_cpp
-          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 models/lcm_dreamshaper_v7/FP16
+          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 models/lcm_dreamshaper_v7/FP16
   
       - name: Build app
         working-directory: ${{ env.working_directory }}

.github/workflows/lcm_dreamshaper_cpp.yml

@@ -53,7 +53,7 @@ jobs:
         working-directory: ${{ env.working_directory }}
         run: |
           conda activate openvino_sd_cpp
-          optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
+          optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion models/stable_diffusion_v1_5_ov/FP16
 
       - name: Build app
         working-directory: ${{ env.working_directory }}
@@ -96,7 +96,7 @@ jobs:
           working-directory: ${{ env.working_directory }}
           run: |
             conda activate openvino_sd_cpp
-            optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
+            optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion models/stable_diffusion_v1_5_ov/FP16
   
         - name: Build app
           working-directory: ${{ env.working_directory }}

.github/workflows/stable_diffusion_1_5_cpp.yml

@@ -35,10 +35,17 @@ install(DIRECTORY
         ./samples/cpp/multinomial_causal_lm
         # Don't install prompt_lookup_decoding_lm and speculative_decoding_lm because they don't use openvino_genai library and arent verifyed yet.
     DESTINATION samples/cpp/ COMPONENT cpp_samples_genai)
-install(FILES ./samples/cpp/requirements.txt DESTINATION samples/cpp/ COMPONENT cpp_samples_genai)
+install(DIRECTORY
+        ./samples/python/beam_search_causal_lm
+        ./samples/python/chat_sample
+        ./samples/python/greedy_causal_lm
+        ./samples/python/multinomial_causal_lm
+    DESTINATION samples/python/ COMPONENT cpp_samples_genai
+    USE_SOURCE_PERMISSIONS)
+install(FILES ./samples/requirements.txt DESTINATION ./samples/ COMPONENT cpp_samples_genai)
 install(FILES LICENSE DESTINATION licensing COMPONENT licensing_genai RENAME LICENSE-GENAI)
 install(FILES third-party-programs.txt DESTINATION licensing COMPONENT licensing_genai RENAME third-party-programs-genai.txt)
-if(MSVC AND NOT DEFINED CPACK_GENERATOR)
+if(WIN32 AND NOT DEFINED CPACK_GENERATOR)
     set(CPACK_GENERATOR "ZIP")
 endif()
 include(CPack)

CMakeLists.txt

@@ -0,0 +1,21 @@
+#!groovy
+
+properties([
+    parameters([
+        booleanParam(defaultValue: false,
+                     description: 'Cancel the rest of parallel stages if one of them fails and return status immediately',
+                     name: 'failFast'),
+        booleanParam(defaultValue: true,
+                     description: 'Whether to propagate commit status to GitHub',
+                     name: 'propagateStatus'),
+        booleanParam(defaultValue: false,
+               description: 'If true, forces running pre-commit scope',
+               name: 'forceRunPrecommitScope'),
+        string(defaultValue: '',
+               description: 'Pipeline shared library version (branch/tag/commit). Determined automatically if empty',
+               name: 'library_version')
+    ])
+])
+loadOpenVinoLibrary {
+    entrypoint(this)
+}

Jenkinsfile

@@ -290,6 +290,7 @@ def run_text_generation_benchmark(model_path, framework, device, args, num_iters
     iter_data_list = []
     warmup_md5 = {}
     input_text_list = utils.model_utils.get_prompts(args)
+    text_gen_fn = run_text_generation if not use_genai else run_text_generation_genai
     if args['prompt_index'] is None:
         prompt_idx_list = [prompt_idx for prompt_idx, input_text in enumerate(input_text_list)]
         text_list = input_text_list

llm_bench/python/benchmark.py

@@ -7,7 +7,11 @@ find_package(OpenVINOGenAI REQUIRED PATHS
 )
 add_executable(beam_search_causal_lm beam_search_causal_lm.cpp)
 target_link_libraries(beam_search_causal_lm PRIVATE openvino::genai)
-target_compile_features(beam_search_causal_lm PRIVATE cxx_std_17)
+set_target_properties(beam_search_causal_lm PROPERTIES
+    COMPILE_PDB_NAME beam_search_causal_lm
+    # Ensure out of box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+target_compile_features(beam_search_causal_lm PRIVATE cxx_std_11)
 install(TARGETS beam_search_causal_lm
     RUNTIME DESTINATION samples_bin/
     COMPONENT samples_bin

samples/cpp/beam_search_causal_lm/CMakeLists.txt

@@ -1,13 +1,13 @@
 # Text generation C++ sample that supports most popular models like LLaMA 2
 
-This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application don't have many configuration options to encourage the reader to explore and modify the source code. It's only possible to change the device for inference to a differnt one, GPU for example, from the command line interface. The sample fearures `ov::genai::LLMPipeline` and configures it to use multiple beam grops. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
+This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. It's only possible to change the device for inference to a differnt one, GPU for example, from the command line interface. The sample fearures `ov::genai::LLMPipeline` and configures it to use multiple beam grops. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
 
 ## Download and convert the model and tokenizers
 
 The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version.
 
 ```sh
-python3 -m pip install --upgrade-strategy eager -r ../../requirements.txt
+pip install --upgrade-strategy eager -r ../../requirements.txt
 optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
 ```
 

samples/cpp/beam_search_causal_lm/README.md

@@ -8,22 +8,21 @@ int main(int argc, char* argv[]) try {
         throw std::runtime_error(std::string{"Usage: "} + argv[0] + " <MODEL_DIR> '<PROMPT 1>' ['<PROMPT 2>' ...]");
     }
     auto prompts = std::vector<std::string>(argv + 2, argv + argc);
-    
     std::string model_path = argv[1];
-    std::string device = "CPU";  // GPU can be used as well
 
+    std::string device = "CPU";  // GPU can be used as well
     ov::genai::LLMPipeline pipe(model_path, device);
-    ov::genai::GenerationConfig config = pipe.get_generation_config();
+
+    ov::genai::GenerationConfig config;
     config.max_new_tokens = 20;
     config.num_beam_groups = 3;
     config.num_beams = 15;
-    config.num_return_sequences = config.num_beams * prompts.size();
+    config.num_return_sequences = config.num_beams;
 
+    // Since the streamer is set, the results will
+    // be printed each time a new token is generated.
     auto beams = pipe.generate(prompts, config);
-    for (int i = 0; i < beams.scores.size(); i++)
-        std::cout << beams.scores[i] << ": " << beams.texts[i] << '\n';
-
-    return 0;
+    std::cout << beams << '\n';
 } catch (const std::exception& error) {
     std::cerr << error.what() << '\n';
     return EXIT_FAILURE;

samples/cpp/beam_search_causal_lm/beam_search_causal_lm.cpp

@@ -7,7 +7,11 @@ find_package(OpenVINOGenAI REQUIRED PATHS
 )
 add_executable(chat_sample chat_sample.cpp)
 target_link_libraries(chat_sample PRIVATE openvino::genai)
-target_compile_features(chat_sample PRIVATE cxx_std_17)
+set_target_properties(chat_sample PROPERTIES
+    COMPILE_PDB_NAME chat_sample
+    # Ensure out of box LC_RPATH on macOS with SIP
+    INSTALL_RPATH_USE_LINK_PATH ON)
+target_compile_features(chat_sample PRIVATE cxx_std_11)
 install(TARGETS chat_sample
     RUNTIME DESTINATION samples_bin/
     COMPONENT samples_bin