[Image Generation] Supported force_zeros_for_empty_prompt in SDXL (openvinotoolkit#1115)

ilya-lavrenov · web-flow · commit a8145bc29a7d · 2024-10-31T20:15:38.000Z
- Supported force_zeros_for_empty_prompt 
- Supported prompt_2, negative_prompt_2
- Fixed indexes of text encoders hidden state which are used to create
UNet's encoder_hidden_state

Now implementation is fully aligned with vanilla HF implementation

CVS-156383
diff --git a/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp b/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp
@@ -22,8 +22,7 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModel {
 public:
     struct OPENVINO_GENAI_EXPORTS Config {
         size_t max_position_embeddings = 77;
-        size_t hidden_size = 512;
-        size_t num_hidden_layers = 13;
+        size_t num_hidden_layers = 12;
 
         explicit Config(const std::filesystem::path& config_path);
     };
diff --git a/src/cpp/include/openvino/genai/image_generation/clip_text_model_with_projection.hpp b/src/cpp/include/openvino/genai/image_generation/clip_text_model_with_projection.hpp
@@ -22,8 +22,7 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModelWithProjection {
 public:
     struct OPENVINO_GENAI_EXPORTS Config {
         size_t max_position_embeddings = 77;
-        size_t hidden_size = 512;
-        size_t num_hidden_layers = 33;
+        size_t num_hidden_layers = 32;
 
         explicit Config(const std::filesystem::path& config_path);
     };
diff --git a/src/cpp/include/openvino/genai/image_generation/generation_config.hpp b/src/cpp/include/openvino/genai/image_generation/generation_config.hpp
@@ -43,9 +43,8 @@ struct OPENVINO_GENAI_EXPORTS ImageGenerationConfig {
     // SD XL: prompt2 and negative_prompt2
     // FLUX: prompt2 (prompt if prompt2 is not defined explicitly)
     // SD 3: prompt2, prompt3 (with fallback to prompt) and negative_prompt2, negative_prompt3
-    std::string negative_prompt;
     std::optional<std::string> prompt_2 = std::nullopt, prompt_3 = std::nullopt;
-    std::optional<std::string> negative_prompt_2 = std::nullopt, negative_prompt_3 = std::nullopt;
+    std::optional<std::string> negative_prompt = std::nullopt, negative_prompt_2 = std::nullopt, negative_prompt_3 = std::nullopt;
 
     size_t num_images_per_prompt = 1;
 
diff --git a/src/cpp/src/image_generation/generation_config.cpp b/src/cpp/src/image_generation/generation_config.cpp
@@ -68,7 +68,7 @@ void ImageGenerationConfig::update_generation_config(const ov::AnyMap& propertie
 }
 
 void ImageGenerationConfig::validate() const {
-    OPENVINO_ASSERT(guidance_scale > 1.0f || negative_prompt.empty(), "Guidance scale <= 1.0 ignores negative prompt");
+    OPENVINO_ASSERT(guidance_scale > 1.0f || negative_prompt == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt");
     OPENVINO_ASSERT(guidance_scale > 1.0f || negative_prompt_2 == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt 2");
     OPENVINO_ASSERT(guidance_scale > 1.0f || negative_prompt_3 == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt 3");
 }
diff --git a/src/cpp/src/image_generation/models/clip_text_model.cpp b/src/cpp/src/image_generation/models/clip_text_model.cpp
@@ -20,7 +20,6 @@ CLIPTextModel::Config::Config(const std::filesystem::path& config_path) {
     using utils::read_json_param;
 
     read_json_param(data, "max_position_embeddings", max_position_embeddings);
-    read_json_param(data, "hidden_size", hidden_size);
     read_json_param(data, "num_hidden_layers", num_hidden_layers);
 }
 
diff --git a/src/cpp/src/image_generation/models/clip_text_model_with_projection.cpp b/src/cpp/src/image_generation/models/clip_text_model_with_projection.cpp
@@ -20,7 +20,6 @@ CLIPTextModelWithProjection::Config::Config(const std::filesystem::path& config_
     using utils::read_json_param;
 
     read_json_param(data, "max_position_embeddings", max_position_embeddings);
-    read_json_param(data, "hidden_size", hidden_size);
     read_json_param(data, "num_hidden_layers", num_hidden_layers);
 }
 
diff --git a/src/cpp/src/image_generation/stable_diffusion_3_pipeline.hpp b/src/cpp/src/image_generation/stable_diffusion_3_pipeline.hpp
@@ -289,7 +289,9 @@ class StableDiffusion3Pipeline : public DiffusionPipeline {
         std::string prompt_3_str =
             generation_config.prompt_3 != std::nullopt ? *generation_config.prompt_3 : positive_prompt;
 
-        std::string negative_prompt_1_str = generation_config.negative_prompt;
+        std::string negative_prompt_1_str = generation_config.negative_prompt != std::nullopt
+                                                ? *generation_config.negative_prompt
+                                                : std::string{};
         std::string negative_prompt_2_str = generation_config.negative_prompt_2 != std::nullopt
                                                 ? *generation_config.negative_prompt_2
                                                 : negative_prompt_1_str;
@@ -582,9 +584,10 @@ class StableDiffusion3Pipeline : public DiffusionPipeline {
 
             ov::Shape noise_pred_shape = noise_pred_tensor.get_shape();
             noise_pred_shape[0] /= batch_size_multiplier;
-            noisy_residual_tensor.set_shape(noise_pred_shape);
 
             if (batch_size_multiplier > 1) {
+                noisy_residual_tensor.set_shape(noise_pred_shape);
+
                 // perform guidance
                 float* noisy_residual = noisy_residual_tensor.data<float>();
                 const float* noise_pred_uncond = noise_pred_tensor.data<const float>();
@@ -657,7 +660,7 @@ class StableDiffusion3Pipeline : public DiffusionPipeline {
             generation_config.prompt_3 == std::nullopt || generation_config.negative_prompt_3 == std::nullopt,
             "T5Encoder is not currently supported, 'prompt_3' and 'negative_prompt_3' can't be used. Please, add "
             "support.");
-        OPENVINO_ASSERT(is_classifier_free_guidance || generation_config.negative_prompt.empty(),
+        OPENVINO_ASSERT(is_classifier_free_guidance || generation_config.negative_prompt == std::nullopt,
                         "Negative prompt is not used when guidance scale < 1.0");
         OPENVINO_ASSERT(is_classifier_free_guidance || generation_config.negative_prompt_2 == std::nullopt,
                         "Negative prompt 2 is not used when guidance scale < 1.0");
diff --git a/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp b/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp
@@ -220,7 +220,8 @@ class StableDiffusionPipeline : public DiffusionPipeline {
             generation_config.generator = std::make_shared<CppStdGenerator>(seed);
         }
 
-        ov::Tensor encoder_hidden_states = m_clip_text_encoder->infer(positive_prompt, generation_config.negative_prompt,
+        std::string negative_prompt = generation_config.negative_prompt != std::nullopt ? *generation_config.negative_prompt : std::string{};
+        ov::Tensor encoder_hidden_states = m_clip_text_encoder->infer(positive_prompt, negative_prompt,
             batch_size_multiplier > 1);
 
         // replicate encoder hidden state to UNet model
@@ -261,13 +262,10 @@ class StableDiffusionPipeline : public DiffusionPipeline {
 
         ov::Tensor denoised, noisy_residual_tensor(ov::element::f32, {});
         for (size_t inference_step = 0; inference_step < generation_config.num_inference_steps; inference_step++) {
+            batch_copy(latent, latent_cfg, 0, 0, generation_config.num_images_per_prompt);
             // concat the same latent twice along a batch dimension in case of CFG
             if (batch_size_multiplier > 1) {
-                batch_copy(latent, latent_cfg, 0, 0, generation_config.num_images_per_prompt);
                 batch_copy(latent, latent_cfg, 0, generation_config.num_images_per_prompt, generation_config.num_images_per_prompt);
-            } else {
-                // just assign to save memory copy
-                latent_cfg = latent;
             }
 
             m_scheduler->scale_model_input(latent_cfg, inference_step);
@@ -277,9 +275,10 @@ class StableDiffusionPipeline : public DiffusionPipeline {
 
             ov::Shape noise_pred_shape = noise_pred_tensor.get_shape();
             noise_pred_shape[0] /= batch_size_multiplier;
-            noisy_residual_tensor.set_shape(noise_pred_shape);
 
             if (batch_size_multiplier > 1) {
+                noisy_residual_tensor.set_shape(noise_pred_shape);
+
                 // perform guidance
                 float* noisy_residual = noisy_residual_tensor.data<float>();
                 const float* noise_pred_uncond = noise_pred_tensor.data<const float>();
@@ -349,9 +348,9 @@ class StableDiffusionPipeline : public DiffusionPipeline {
         OPENVINO_ASSERT(generation_config.prompt_2 == std::nullopt, "Prompt 2 is not used by ", pipeline_name);
         OPENVINO_ASSERT(generation_config.prompt_3 == std::nullopt, "Prompt 3 is not used by ", pipeline_name);
         if (is_lcm) {
-            OPENVINO_ASSERT(generation_config.negative_prompt.empty(), "Negative prompt is not used by ", pipeline_name);
+            OPENVINO_ASSERT(generation_config.negative_prompt == std::nullopt, "Negative prompt is not used by ", pipeline_name);
         } else if (!is_classifier_free_guidance) {
-            OPENVINO_ASSERT(generation_config.negative_prompt.empty(), "Negative prompt is not used when guidance scale <= 1.0");
+            OPENVINO_ASSERT(generation_config.negative_prompt == std::nullopt, "Negative prompt is not used when guidance scale <= 1.0");
         }
         OPENVINO_ASSERT(generation_config.negative_prompt_2 == std::nullopt, "Negative prompt 2 is not used by ", pipeline_name);
         OPENVINO_ASSERT(generation_config.negative_prompt_3 == std::nullopt, "Negative prompt 3 is not used by ", pipeline_name);
diff --git a/src/cpp/src/image_generation/stable_diffusion_xl_pipeline.hpp b/src/cpp/src/image_generation/stable_diffusion_xl_pipeline.hpp
diff --git a/src/python/py_image_generation_models.cpp b/src/python/py_image_generation_models.cpp

Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@ void ImageGenerationConfig::update_generation_config(const ov::AnyMap& propertie`
`68`	`68`	`}`
`69`	`69`
`70`	`70`	`void ImageGenerationConfig::validate() const {`
`71`		`- OPENVINO_ASSERT(guidance_scale > 1.0f \|\| negative_prompt.empty(), "Guidance scale <= 1.0 ignores negative prompt");`
	`71`	`+ OPENVINO_ASSERT(guidance_scale > 1.0f \|\| negative_prompt == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt");`
`72`	`72`	`OPENVINO_ASSERT(guidance_scale > 1.0f \|\| negative_prompt_2 == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt 2");`
`73`	`73`	`OPENVINO_ASSERT(guidance_scale > 1.0f \|\| negative_prompt_3 == std::nullopt, "Guidance scale <= 1.0 ignores negative prompt 3");`
`74`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,6 @@ CLIPTextModel::Config::Config(const std::filesystem::path& config_path) {`
`20`	`20`	`using utils::read_json_param;`
`21`	`21`
`22`	`22`	`read_json_param(data, "max_position_embeddings", max_position_embeddings);`
`23`		`- read_json_param(data, "hidden_size", hidden_size);`
`24`	`23`	`read_json_param(data, "num_hidden_layers", num_hidden_layers);`
`25`	`24`	`}`
`26`	`25`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,6 @@ CLIPTextModelWithProjection::Config::Config(const std::filesystem::path& config_`
`20`	`20`	`using utils::read_json_param;`
`21`	`21`
`22`	`22`	`read_json_param(data, "max_position_embeddings", max_position_embeddings);`
`23`		`- read_json_param(data, "hidden_size", hidden_size);`
`24`	`23`	`read_json_param(data, "num_hidden_layers", num_hidden_layers);`
`25`	`24`	`}`
`26`	`25`