[Port 24.1] Move from image_generation python conversion scripts to optimum-cli (LCM Dreamshaper v7 model) (#402)

Author: yatarkan

.github/workflows/lcm_dreamshaper_cpp.yml

@@ -1,80 +1,118 @@
 name: lcm_dreamshaper
+
 on:
   pull_request:
     paths:
       - image_generation/lcm_dreamshaper_v7/cpp/**
       - image_generation/common/**
       - .github/workflows/lcm_dreamshaper_cpp.yml
       - thirdparty/openvino_tokenizers
+
+env:
+  working_directory: "./image_generation/lcm_dreamshaper_v7/cpp/"
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
+
 jobs:
   lcm_dreamshaper_v7_cpp-linux:
     runs-on: ubuntu-20.04
+    defaults:
+      run:
+        # Do not ignore bash profile files. From:
+        # https://github.com/marketplace/actions/setup-miniconda#important
+        shell: bash -l {0}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: actions/setup-python@v4
+
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v3
         with:
-          python-version: 3.8
-      - name: Initialize OpenVINO
+          miniconda-version: "latest"
+          activate-environment: openvino_lcm_cpp
+          python-version: "3.10"
+
+      - name: Install OpenVINO and other conda dependencies
         run: |
-          mkdir openvino
-          curl https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64.tgz | tar --directory ./openvino/ --strip-components 1 -xz
-          sudo ./openvino/install_dependencies/install_openvino_dependencies.sh
-      - name: Download / convert a model / tokenizer
+          conda activate openvino_lcm_cpp
+          conda update -c conda-forge --all
+          conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
+          conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+      
+      - name: Install python dependencies
+        working-directory: ${{ env.working_directory }}
         run: |
-          source ./openvino/setupvars.sh
-          cd ./image_generation/lcm_dreamshaper_v7/cpp/scripts/
-          python -m pip install -U pip
-          python -m pip install -r ./requirements.txt
-          python -m pip install ../../../../thirdparty/openvino_tokenizers/
-          python convert_model.py -lcm "SimianLuo/LCM_Dreamshaper_v7" -t "FP16"
+          conda activate openvino_lcm_cpp
+          python -m pip install -r requirements.txt
+          python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+          
+      - name: Download and convert model and tokenizer
+        working-directory: ${{ env.working_directory }}
+        run: |
+          conda activate openvino_lcm_cpp
+          export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
+          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
+          convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
+
       - name: Build app
+        working-directory: ${{ env.working_directory }}
         run: |
-          source ./openvino/setupvars.sh
-          cd ./image_generation/lcm_dreamshaper_v7/cpp/
+          conda activate openvino_lcm_cpp
           cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
           cmake --build ./build/ --config Release --parallel
+      
       - name: Run app
+        working-directory: ${{ env.working_directory }}
         run: |
-          source ./openvino/setupvars.sh
-          cd ./image_generation/lcm_dreamshaper_v7/cpp/build/
-          ./lcm_dreamshaper
+          ./build/lcm_dreamshaper
+
   lcm_dreamshaper_v7_cpp-windows:
-      runs-on: windows-latest
-      steps:
-        - uses: actions/checkout@v4
-          with:
-            submodules: recursive
-        - uses: actions/setup-python@v4
-          with:
-            python-version: 3.8
-        - name: Initialize OpenVINO
-          shell: cmd
-          run: |
-            curl --output ov.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/windows/w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64.zip
-            unzip ov.zip
-        - name: Download / convert a model / tokenizer
-          shell: cmd
-          run: |
-            call w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64/setupvars.bat
-            cd ./image_generation/lcm_dreamshaper_v7/cpp/scripts/
-            python -m pip install -r ./requirements.txt
-            python -m pip install ../../../../thirdparty/openvino_tokenizers/
-            python convert_model.py -lcm "SimianLuo/LCM_Dreamshaper_v7" -t "FP16"
-        - name: Build app
-          shell: cmd
-          run: |
-            call w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64/setupvars.bat
-            cd ./image_generation/lcm_dreamshaper_v7/cpp/
-            cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
-            cmake --build ./build/ --config Release --parallel
-        - name: Run app
-          shell: cmd
-          run: |
-            call w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64/setupvars.bat
-            cd ./image_generation/lcm_dreamshaper_v7/cpp/build/
-            call "./Release/lcm_dreamshaper.exe"
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          activate-environment: openvino_lcm_cpp
+          python-version: "3.10"
+
+      - name: Install OpenVINO and other conda dependencies
+        run: |
+          conda activate openvino_lcm_cpp
+          conda update -c conda-forge --all
+          conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
+          conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+      
+      - name: Install python dependencies
+        working-directory: ${{ env.working_directory }}
+        run: |
+          conda activate openvino_lcm_cpp
+          python -m pip install -r requirements.txt
+          python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
+
+      - name: Download and convert model and tokenizer
+        working-directory: ${{ env.working_directory }}
+        run: |
+          conda activate openvino_lcm_cpp
+          $env:MODEL_PATH='models/lcm_dreamshaper_v7/FP16'
+          optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $env:MODEL_PATH
+          convert_tokenizer $env:MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $env:MODEL_PATH/tokenizer/
+  
+      - name: Build app
+        working-directory: ${{ env.working_directory }}
+        run: |
+          conda activate openvino_lcm_cpp
+          cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
+          cmake --build ./build/ --config Release --parallel
+      
+      - name: Run app
+        working-directory: ${{ env.working_directory }}
+        run: |
+          & "./build/Release/lcm_dreamshaper.exe" -r --dynamic

.github/workflows/stable_diffusion_1_5_cpp.yml

@@ -92,8 +92,6 @@ jobs:
           run: |
             conda activate openvino_sd_cpp
             python -m pip install -r requirements.txt
-            $env:SPM_PROTOBUF_PROVIDER = 'internal'
-            $env:SPM_ABSL_PROVIDER = 'internal'
             python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
   
         - name: Download and convert model and tokenizer

image_generation/common/diffusers/src/scheduler_lcm.cpp

@@ -192,7 +192,7 @@ std::map<std::string, ov::Tensor> LCMScheduler::step(ov::Tensor noise_pred, ov::
     if (inference_step != num_inference_steps - 1) {
         std::vector<float> noise;
         if (read_torch_noise) {
-            std::string noise_file = "../scripts/torch_noise_step_" + std::to_string(inference_step) + ".txt";
+            std::string noise_file = "./latents/torch_noise_step_" + std::to_string(inference_step) + ".txt";
             noise = read_vector_from_txt(noise_file);
         } else {
             noise = randn_function(noise_pred.get_size(), seed);

image_generation/lcm_dreamshaper_v7/cpp/README.md

@@ -2,19 +2,25 @@
 The pure C++ text-to-image pipeline, driven by the OpenVINO native API for SD v1.5 Latent Consistency Model with LCM Scheduler. It includes advanced features like LoRA integration with safetensors and [OpenVINO Tokenizers](https://github.com/openvinotoolkit/openvino_tokenizers). Loading `openvino_tokenizers` to `ov::Core` enables tokenization. [The common folder](../../common/) contains schedulers for image generation and `imwrite()` for saving `bmp` images. This demo has been tested for Linux platform only. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/263-latent-consistency-models-image-generation/263-lcm-lora-controlnet.ipynb) which provides an example of image generaztion in Python.
 
 > [!NOTE]
->This tutorial assumes that the current working directory is `<openvino.genai repo>/image_generation/lcm_dreamshaper_v7/cpp/` and all paths are relative to this folder.
+> This tutorial assumes that the current working directory is `<openvino.genai repo>/image_generation/lcm_dreamshaper_v7/cpp/` and all paths are relative to this folder.
 
 ## Step 1: Prepare build environment
 
+Prerequisites:
+- Conda ([installation guide](https://conda.io/projects/conda/en/latest/user-guide/install/index.html))
+
 C++ Packages:
 * [CMake](https://cmake.org/download/): Cross-platform build tool
-* [OpenVINO](https://docs.openvino.ai/2023.2/openvino_docs_install_guides_overview.html): Model inference
+* [OpenVINO](https://docs.openvino.ai/2024/get-started/install-openvino.html): Model inference
 
 Prepare a python environment and install dependencies:
 ```shell
 conda create -n openvino_lcm_cpp python==3.10
 conda activate openvino_lcm_cpp
-conda install -c conda-forge openvino c-compiler cxx-compiler make
+conda update -c conda-forge --all
+conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
+# Ensure that Conda standard libraries are used
+conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
 ```
 
 ## Step 2: Latent Consistency Model and Tokenizer models
@@ -26,20 +32,19 @@ conda install -c conda-forge openvino c-compiler cxx-compiler make
     ```shell
     git submodule update --init
     conda activate openvino_lcm_cpp
-    python -m pip install -r scripts/requirements.txt
+    python -m pip install -r requirements.txt
     python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
     ```
 
-2. Run model conversion script to download and convert PyTorch model to OpenVINO IR via [optimum-intel](https://github.com/huggingface/optimum-intel). Please, use the script `scripts/convert_model.py` to convert the model:
-
+2. Download the model from Huggingface and convert it to OpenVINO IR via [optimum-intel CLI](https://github.com/huggingface/optimum-intel). Example command for downloading and exporting FP16 model:
     ```shell
-    cd scripts
-    python convert_model.py -lcm "SimianLuo/LCM_Dreamshaper_v7" -t FP16
+    export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
+    # Using optimum-cli for exporting model to OpenVINO format
+    optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
+    # Converting tokenizer
+    convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
     ```
 
-> [!NOTE]
->Only static model is currently supported for this sample.
-
 ### LoRA enabling with safetensors
 
 Refer to [python pipeline blog](https://blog.openvino.ai/blog-posts/enable-lora-weights-with-stable-diffusion-controlnet-pipeline).
@@ -65,19 +70,20 @@ Usage:
   lcm_dreamshaper [OPTION...]
 ```
 
-* `-p, --posPrompt arg` Initial positive prompt for SD  (default: cyberpunk cityscape like Tokyo New York  with tall buildings at dusk golden hour cinematic lighting)
+* `-p, --posPrompt arg` Initial positive prompt for LCM (default: a beautiful pink unicorn)
 * `-d, --device arg`    AUTO, CPU, or GPU. Doesn't apply to Tokenizer model, OpenVINO Tokenizers can be inferred on a CPU device only (default: CPU)
-* `--step arg`          Number of diffusion step ( default: 20)
+* `--step arg`          Number of diffusion step (default: 4)
 * `-s, --seed arg`      Number of random seed to generate latent (default: 42)
-* `--num arg`           Number of image output(default: 1)
+* `--num arg`           Number of image output (default: 1)
 * `--height arg`        Height of output image (default: 512)
 * `--width arg`         Width of output image (default: 512)
 * `-c, --useCache`      Use model caching
-* `-r, --readNPLatent`  Read numpy generated latents from file
-* `-m, --modelPath arg` Specify path of SD model IR (default: ../scripts/SimianLuo/LCM_Dreamshaper_v7)
-* `-t, --type arg`      Specify the type of SD model IR (FP16_static or FP16_dyn) (default: FP16_static)
-* `-l, --loraPath arg`  Specify path of lora file. (*.safetensors). (default: )
-* `-a, --alpha arg`     alpha for lora (default: 0.75)
+* `-r, --readNPLatent`  Read numpy generated latents from file, only supported for one output image
+* `-m, --modelPath arg` Specify path to LCM model IRs (default: ./models/lcm_dreamshaper_v7)
+* `-t, --type arg`      Specify the type of LCM model IRs (e.g., FP32, FP16 or INT8) (default: FP16)
+* `--dynamic`           Specify the model input shape to use dynamic shape
+* `-l, --loraPath arg`  Specify path to LoRA file (*.safetensors) (default: )
+* `-a, --alpha arg`     Specify alpha for LoRA (default: 0.75)
 * `-h, --help`          Print usage
 
 > [!NOTE]
@@ -89,15 +95,15 @@ Positive prompt: a beautiful pink unicorn
 
 Read the numpy latent input and noise for scheduler instead of C++ std lib for the alignment with Python pipeline.
 
-* Generate image with random data generated by Python `./build/lcm_dreamshaper -r`
+* Generate image with random data generated by Python: `./build/lcm_dreamshaper -r`
 
 ![image](./python_random.bmp)
 
-* Generate image with C++ lib generated latent and noise : `./build/lcm_dreamshaper`
+* Generate image with C++ lib generated latent and noise: `./build/lcm_dreamshaper`
 
 ![image](./cpp_random.bmp)
 
-* Generate image with soulcard lora and C++ generated latent and noise `./stable_diffusion -r -l path/to/soulcard.safetensors`
+* Generate image with soulcard lora and C++ generated latent and noise: `./stable_diffusion -r -l path/to/soulcard.safetensors`
 
 ![image](./lora_cpp_random.bmp)