From 62ec71506cf70f8ff97809e6bfc99ebaa3f815eb Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Thu, 2 Jan 2025 14:24:06 +0100 Subject: [PATCH 1/2] test version --- samples/cpp/image_generation/CMakeLists.txt | 18 +++++++++++ .../genai/image_generation/autoencoder_kl.hpp | 2 ++ .../image_generation/clip_text_model.hpp | 2 ++ .../unet2d_condition_model.hpp | 1 + .../models/autoencoder_kl.cpp | 31 ++++++++++++++----- .../models/clip_text_model.cpp | 19 ++++++++---- .../models/unet2d_condition_model.cpp | 3 ++ .../models/unet_inference_dynamic.hpp | 24 +++++++++++--- .../models/unet_inference_static_bs1.hpp | 17 ++++++++-- .../stable_diffusion_pipeline.hpp | 4 +++ 10 files changed, 100 insertions(+), 21 deletions(-) diff --git a/samples/cpp/image_generation/CMakeLists.txt b/samples/cpp/image_generation/CMakeLists.txt index 004b305088..c8c93618ca 100644 --- a/samples/cpp/image_generation/CMakeLists.txt +++ b/samples/cpp/image_generation/CMakeLists.txt @@ -28,6 +28,24 @@ install(TARGETS text2image COMPONENT samples_bin EXCLUDE_FROM_ALL) +# create text2image sample executable with concurrency + +add_executable(text2image_concurrency text2image_concurrency.cpp imwrite.cpp) + +target_include_directories(text2image_concurrency PRIVATE ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}") +target_link_libraries(text2image_concurrency PRIVATE openvino::genai) + +set_target_properties(text2image_concurrency PROPERTIES + COMPILE_PDB_NAME text2image_concurrency + # Ensure out of box LC_RPATH on macOS with SIP + INSTALL_RPATH_USE_LINK_PATH ON) + +install(TARGETS text2image_concurrency + RUNTIME DESTINATION samples_bin/ + COMPONENT samples_bin + EXCLUDE_FROM_ALL) + + # create LoRA sample executable add_executable(lora_text2image lora_text2image.cpp imwrite.cpp) diff --git a/src/cpp/include/openvino/genai/image_generation/autoencoder_kl.hpp b/src/cpp/include/openvino/genai/image_generation/autoencoder_kl.hpp index d48661d899..1a3ac31b06 100644 --- a/src/cpp/include/openvino/genai/image_generation/autoencoder_kl.hpp +++ b/src/cpp/include/openvino/genai/image_generation/autoencoder_kl.hpp @@ -29,6 +29,7 @@ class OPENVINO_GENAI_EXPORTS AutoencoderKL { std::vector block_out_channels = { 64 }; explicit Config(const std::filesystem::path& config_path); + Config() = default; }; explicit AutoencoderKL(const std::filesystem::path& vae_decoder_path); @@ -140,6 +141,7 @@ class OPENVINO_GENAI_EXPORTS AutoencoderKL { Config m_config; ov::InferRequest m_encoder_request, m_decoder_request; + std::shared_ptr encoder_compiled_model, decoder_compiled_model; std::shared_ptr m_encoder_model = nullptr, m_decoder_model = nullptr; }; diff --git a/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp b/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp index a3b9ebbd88..755abc220c 100644 --- a/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp +++ b/src/cpp/include/openvino/genai/image_generation/clip_text_model.hpp @@ -25,6 +25,7 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModel { size_t num_hidden_layers = 12; explicit Config(const std::filesystem::path& config_path); + Config() = default; }; explicit CLIPTextModel(const std::filesystem::path& root_dir); @@ -92,6 +93,7 @@ class OPENVINO_GENAI_EXPORTS CLIPTextModel { Config m_config; AdapterController m_adapter_controller; ov::InferRequest m_request; + std::shared_ptr compiled_model; std::shared_ptr m_model; Tokenizer m_clip_tokenizer; diff --git a/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp b/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp index 4acfd2ce9b..aebc1e66ee 100644 --- a/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp +++ b/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp @@ -28,6 +28,7 @@ class OPENVINO_GENAI_EXPORTS UNet2DConditionModel { int time_cond_proj_dim = -1; explicit Config(const std::filesystem::path& config_path); + //Config() = default; }; explicit UNet2DConditionModel(const std::filesystem::path& root_dir); diff --git a/src/cpp/src/image_generation/models/autoencoder_kl.cpp b/src/cpp/src/image_generation/models/autoencoder_kl.cpp index d3dd7324ee..7c109f4a73 100644 --- a/src/cpp/src/image_generation/models/autoencoder_kl.cpp +++ b/src/cpp/src/image_generation/models/autoencoder_kl.cpp @@ -179,7 +179,20 @@ AutoencoderKL::AutoencoderKL(const std::string& vae_encoder_model, } } -AutoencoderKL::AutoencoderKL(const AutoencoderKL&) = default; +AutoencoderKL::AutoencoderKL(const AutoencoderKL& original_model){ + std::cout << "bla1" << std::endl; + encoder_compiled_model = original_model.encoder_compiled_model; + std::cout << "bla2" << std::endl; + decoder_compiled_model = original_model.decoder_compiled_model; + std::cout << "bla3" << std::endl; + m_decoder_request = original_model.decoder_compiled_model->create_infer_request(); + if (m_encoder_model){ + std::cout << "bla4" << std::endl; + m_decoder_request = decoder_compiled_model->create_infer_request(); + } + m_encoder_model = original_model.m_encoder_model; + m_decoder_model = original_model.m_decoder_model; +} AutoencoderKL& AutoencoderKL::reshape(int batch_size, int height, int width) { OPENVINO_ASSERT(m_decoder_model, "Model has been already compiled. Cannot reshape already compiled model"); @@ -211,16 +224,20 @@ AutoencoderKL& AutoencoderKL::compile(const std::string& device, const ov::AnyMa ov::Core core = utils::singleton_core(); if (m_encoder_model) { - ov::CompiledModel encoder_compiled_model = core.compile_model(m_encoder_model, device, properties); - ov::genai::utils::print_compiled_model_properties(encoder_compiled_model, "Auto encoder KL encoder model"); - m_encoder_request = encoder_compiled_model.create_infer_request(); + std::cout << "copiling model" << std::endl; + encoder_compiled_model = std::make_shared(core.compile_model(m_encoder_model, device, properties)); + std::cout << "compiled encoder model" << std::endl; + ov::genai::utils::print_compiled_model_properties(*encoder_compiled_model, "Auto encoder KL encoder model"); + m_encoder_request = encoder_compiled_model->create_infer_request(); + std::cout << "infer request encoder model" << std::endl; // release the original model m_encoder_model.reset(); } - ov::CompiledModel decoder_compiled_model = core.compile_model(m_decoder_model, device, properties); - ov::genai::utils::print_compiled_model_properties(decoder_compiled_model, "Auto encoder KL decoder model"); - m_decoder_request = decoder_compiled_model.create_infer_request(); + decoder_compiled_model = std::make_shared(core.compile_model(m_decoder_model, device, properties)); + std::cout << "compiled decoder model" << std::endl; + ov::genai::utils::print_compiled_model_properties(*decoder_compiled_model, "Auto encoder KL decoder model"); + m_decoder_request = decoder_compiled_model->create_infer_request(); // release the original model m_decoder_model.reset(); diff --git a/src/cpp/src/image_generation/models/clip_text_model.cpp b/src/cpp/src/image_generation/models/clip_text_model.cpp index efbc840d4f..691055f551 100644 --- a/src/cpp/src/image_generation/models/clip_text_model.cpp +++ b/src/cpp/src/image_generation/models/clip_text_model.cpp @@ -67,7 +67,15 @@ CLIPTextModel::CLIPTextModel(const std::string& model, compile(device, properties); } -CLIPTextModel::CLIPTextModel(const CLIPTextModel&) = default; +CLIPTextModel::CLIPTextModel(const CLIPTextModel& origin_model) { + m_config = origin_model.m_config; + m_adapter_controller = origin_model.m_adapter_controller; + compiled_model = origin_model.compiled_model; + m_request = compiled_model->create_infer_request(); + m_model = origin_model.m_model; + m_clip_tokenizer = origin_model.m_clip_tokenizer; +} + const CLIPTextModel::Config& CLIPTextModel::get_config() const { return m_config; @@ -88,17 +96,16 @@ CLIPTextModel& CLIPTextModel::reshape(int batch_size) { CLIPTextModel& CLIPTextModel::compile(const std::string& device, const ov::AnyMap& properties) { OPENVINO_ASSERT(m_model, "Model has been already compiled. Cannot re-compile already compiled model"); ov::Core core = utils::singleton_core(); - ov::CompiledModel compiled_model; std::optional adapters; if (auto filtered_properties = extract_adapters_from_properties(properties, &adapters)) { adapters->set_tensor_name_prefix(adapters->get_tensor_name_prefix().value_or("lora_te")); m_adapter_controller = AdapterController(m_model, *adapters, device); - compiled_model = core.compile_model(m_model, device, *filtered_properties); + compiled_model = std::make_shared(core.compile_model(m_model, device, *filtered_properties)); } else { - compiled_model = core.compile_model(m_model, device, properties); + compiled_model = std::make_shared(core.compile_model(m_model, device, properties)); } - ov::genai::utils::print_compiled_model_properties(compiled_model, "Clip Text model"); - m_request = compiled_model.create_infer_request(); + ov::genai::utils::print_compiled_model_properties(*compiled_model, "Clip Text model"); + m_request = compiled_model->create_infer_request(); // release the original model m_model.reset(); diff --git a/src/cpp/src/image_generation/models/unet2d_condition_model.cpp b/src/cpp/src/image_generation/models/unet2d_condition_model.cpp index ca65c9d9d6..9960731a6e 100644 --- a/src/cpp/src/image_generation/models/unet2d_condition_model.cpp +++ b/src/cpp/src/image_generation/models/unet2d_condition_model.cpp @@ -84,6 +84,7 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c if (device == "NPU") { m_impl = std::make_shared(); } else { + std::cout << "impl UNetInferenceDynamic" << std::endl; m_impl = std::make_shared(); } @@ -93,6 +94,7 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c m_adapter_controller = AdapterController(m_model, *adapters, device); m_impl->compile(m_model, device, *filtered_properties); } else { + std::cout << "impl compile" << std::endl; m_impl->compile(m_model, device, properties); } @@ -104,6 +106,7 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c void UNet2DConditionModel::set_hidden_states(const std::string& tensor_name, ov::Tensor encoder_hidden_states) { OPENVINO_ASSERT(m_impl, "UNet model must be compiled first"); + std::cout << "UNet2DConditionModel::set_hidden_states " << typeid(m_impl).name() << std::endl; m_impl->set_hidden_states(tensor_name, encoder_hidden_states); } diff --git a/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp b/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp index 6dc285f76d..d06a521626 100644 --- a/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp +++ b/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp @@ -19,13 +19,15 @@ class UNet2DConditionModel::UNetInferenceDynamic : public UNet2DConditionModel:: { ov::Core core = utils::singleton_core(); - ov::CompiledModel compiled_model = core.compile_model(model, device, properties); - ov::genai::utils::print_compiled_model_properties(compiled_model, "UNet 2D Condition dynamic model"); - m_request = compiled_model.create_infer_request(); + compiled_model = std::make_shared(core.compile_model(model, device, properties)); + ov::genai::utils::print_compiled_model_properties(*compiled_model, "UNet 2D Condition dynamic model"); + std::cout << "unet dynamic compile" << std::endl; + m_request = compiled_model->create_infer_request(); } virtual void set_hidden_states(const std::string& tensor_name, ov::Tensor encoder_hidden_states) override { + std::cout << "unet dynamic set_hidden_states" << std::endl; OPENVINO_ASSERT(m_request, "UNet model must be compiled first"); m_request.set_tensor(tensor_name, encoder_hidden_states); } @@ -38,19 +40,31 @@ class UNet2DConditionModel::UNetInferenceDynamic : public UNet2DConditionModel:: virtual ov::Tensor infer(ov::Tensor sample, ov::Tensor timestep) override { + std::cout << "unet dynamic infer" << std::endl; OPENVINO_ASSERT(m_request, "UNet model must be compiled first. Cannot infer non-compiled model"); - + std::cout << sample.get_shape().to_string() << std::endl; m_request.set_tensor("sample", sample); + std::cout << timestep.get_shape().to_string() << std::endl; m_request.set_tensor("timestep", timestep); - + ov::CompiledModel test = m_request.get_compiled_model(); + std::cout << "compiled model ok" << test.inputs().at(0).get_any_name() << std::endl; m_request.infer(); + std::cout << "unet dynamic inference complated" << std::endl; return m_request.get_output_tensor(); } + UNetInferenceDynamic(const UNetInferenceDynamic & origin_model){ + OPENVINO_ASSERT(origin_model.compiled_model, "Source model must be compiled first"); + compiled_model = origin_model.compiled_model; + m_request = compiled_model->create_infer_request(); + } + UNetInferenceDynamic() = default; + private: ov::InferRequest m_request; + std::shared_ptr compiled_model; }; diff --git a/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp b/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp index 7aa6f6301c..9525ae480c 100644 --- a/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp +++ b/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp @@ -13,6 +13,15 @@ namespace genai { // Static Batch-Size 1 variant of UNetInference class UNet2DConditionModel::UNetInferenceStaticBS1 : public UNet2DConditionModel::UNetInference { public: + UNetInferenceStaticBS1() = default; + UNetInferenceStaticBS1(const UNetInferenceStaticBS1 & origin_model){ + OPENVINO_ASSERT(origin_model.compiled_model, "Source model must be compiled first"); + compiled_model = origin_model.compiled_model; + size_t m_native_batch_size = origin_model.m_native_batch_size; + for (int i = 0; i < m_native_batch_size; i++) { + m_requests[i] = compiled_model->create_infer_request(); + } + } virtual void compile(std::shared_ptr model, const std::string& device, const ov::AnyMap& properties) override { @@ -39,12 +48,12 @@ class UNet2DConditionModel::UNetInferenceStaticBS1 : public UNet2DConditionModel UNetInference::reshape(model, 1); ov::Core core = utils::singleton_core(); - ov::CompiledModel compiled_model = core.compile_model(model, device, properties); - ov::genai::utils::print_compiled_model_properties(compiled_model, "UNet 2D Condition batch-1 model"); + compiled_model = std::make_shared(core.compile_model(model, device, properties)); + ov::genai::utils::print_compiled_model_properties(*compiled_model, "UNet 2D Condition batch-1 model"); for (int i = 0; i < m_native_batch_size; i++) { - m_requests[i] = compiled_model.create_infer_request(); + m_requests[i] = compiled_model->create_infer_request(); } } @@ -138,8 +147,10 @@ class UNet2DConditionModel::UNetInferenceStaticBS1 : public UNet2DConditionModel } private: + std::shared_ptr compiled_model; std::vector m_requests; size_t m_native_batch_size = 0; + }; } // namespace genai diff --git a/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp b/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp index 3801c855fd..64b8c3fbdf 100644 --- a/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp +++ b/src/cpp/src/image_generation/stable_diffusion_pipeline.hpp @@ -138,9 +138,13 @@ class StableDiffusionPipeline : public DiffusionPipeline { const UNet2DConditionModel& unet, const AutoencoderKL& vae) : StableDiffusionPipeline(pipeline_type) { + std::cout << "test" << std::endl; m_clip_text_encoder = std::make_shared(clip_text_model); + std::cout << "test1" << std::endl; m_unet = std::make_shared(unet); + std::cout << "test2" << std::endl; m_vae = std::make_shared(vae); + std::cout << "test3" << std::endl; const bool is_lcm = m_unet->get_config().time_cond_proj_dim > 0; const char * const pipeline_name = is_lcm ? "LatentConsistencyModelPipeline" : "StableDiffusionPipeline"; From 9a1863b746167dd0b9b19acff079e5f806a067a0 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 3 Jan 2025 16:00:09 +0100 Subject: [PATCH 2/2] initial working stable-diffusion pipeline --- .../unet2d_condition_model.hpp | 3 ++- .../models/autoencoder_kl.cpp | 2 ++ .../models/unet2d_condition_model.cpp | 17 +++++++++++++---- .../models/unet_inference.hpp | 3 +++ .../models/unet_inference_dynamic.hpp | 19 ++++++++----------- .../models/unet_inference_static_bs1.hpp | 13 +++++++++---- src/cpp/src/utils.cpp | 6 ++++++ 7 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp b/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp index aebc1e66ee..557882a3d1 100644 --- a/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp +++ b/src/cpp/include/openvino/genai/image_generation/unet2d_condition_model.hpp @@ -28,7 +28,7 @@ class OPENVINO_GENAI_EXPORTS UNet2DConditionModel { int time_cond_proj_dim = -1; explicit Config(const std::filesystem::path& config_path); - //Config() = default; + Config() = default; }; explicit UNet2DConditionModel(const std::filesystem::path& root_dir); @@ -96,6 +96,7 @@ class OPENVINO_GENAI_EXPORTS UNet2DConditionModel { return guidance_scale > 1.0f && m_config.time_cond_proj_dim < 0; } + private: class UNetInference; std::shared_ptr m_impl; diff --git a/src/cpp/src/image_generation/models/autoencoder_kl.cpp b/src/cpp/src/image_generation/models/autoencoder_kl.cpp index 7c109f4a73..0b18a71257 100644 --- a/src/cpp/src/image_generation/models/autoencoder_kl.cpp +++ b/src/cpp/src/image_generation/models/autoencoder_kl.cpp @@ -179,6 +179,7 @@ AutoencoderKL::AutoencoderKL(const std::string& vae_encoder_model, } } +//AutoencoderKL::AutoencoderKL(const AutoencoderKL&) = default; AutoencoderKL::AutoencoderKL(const AutoencoderKL& original_model){ std::cout << "bla1" << std::endl; encoder_compiled_model = original_model.encoder_compiled_model; @@ -192,6 +193,7 @@ AutoencoderKL::AutoencoderKL(const AutoencoderKL& original_model){ } m_encoder_model = original_model.m_encoder_model; m_decoder_model = original_model.m_decoder_model; + m_config = original_model.m_config; } AutoencoderKL& AutoencoderKL::reshape(int batch_size, int height, int width) { diff --git a/src/cpp/src/image_generation/models/unet2d_condition_model.cpp b/src/cpp/src/image_generation/models/unet2d_condition_model.cpp index 9960731a6e..3b087c4c40 100644 --- a/src/cpp/src/image_generation/models/unet2d_condition_model.cpp +++ b/src/cpp/src/image_generation/models/unet2d_condition_model.cpp @@ -32,6 +32,7 @@ UNet2DConditionModel::UNet2DConditionModel(const std::filesystem::path& root_dir m_config(root_dir / "config.json") { ov::Core core = utils::singleton_core(); m_model = core.read_model((root_dir / "openvino_model.xml").string()); + //ov::save_model(m_model, "static_unet_model.xml"); m_vae_scale_factor = get_vae_scale_factor(root_dir.parent_path() / "vae_decoder" / "config.json"); } @@ -61,7 +62,18 @@ UNet2DConditionModel::UNet2DConditionModel(const std::string& model, compile(device, properties); } -UNet2DConditionModel::UNet2DConditionModel(const UNet2DConditionModel&) = default; +UNet2DConditionModel::UNet2DConditionModel(const UNet2DConditionModel& original_model) { + m_config = original_model.m_config; + m_adapter_controller = original_model.m_adapter_controller; + m_model = original_model.m_model; + m_vae_scale_factor = original_model.m_vae_scale_factor; + if (typeid(m_impl) == typeid(UNet2DConditionModel::UNetInferenceStaticBS1)) { + m_impl = std::make_shared(original_model.m_impl->get_compiled_model()); + } else { + m_impl = std::make_shared(original_model.m_impl->get_compiled_model()); + } + +} const UNet2DConditionModel::Config& UNet2DConditionModel::get_config() const { return m_config; @@ -84,7 +96,6 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c if (device == "NPU") { m_impl = std::make_shared(); } else { - std::cout << "impl UNetInferenceDynamic" << std::endl; m_impl = std::make_shared(); } @@ -94,7 +105,6 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c m_adapter_controller = AdapterController(m_model, *adapters, device); m_impl->compile(m_model, device, *filtered_properties); } else { - std::cout << "impl compile" << std::endl; m_impl->compile(m_model, device, properties); } @@ -106,7 +116,6 @@ UNet2DConditionModel& UNet2DConditionModel::compile(const std::string& device, c void UNet2DConditionModel::set_hidden_states(const std::string& tensor_name, ov::Tensor encoder_hidden_states) { OPENVINO_ASSERT(m_impl, "UNet model must be compiled first"); - std::cout << "UNet2DConditionModel::set_hidden_states " << typeid(m_impl).name() << std::endl; m_impl->set_hidden_states(tensor_name, encoder_hidden_states); } diff --git a/src/cpp/src/image_generation/models/unet_inference.hpp b/src/cpp/src/image_generation/models/unet_inference.hpp index ae928aac30..cf7dc03d6b 100644 --- a/src/cpp/src/image_generation/models/unet_inference.hpp +++ b/src/cpp/src/image_generation/models/unet_inference.hpp @@ -15,6 +15,7 @@ class UNet2DConditionModel::UNetInference { virtual void set_hidden_states(const std::string& tensor_name, ov::Tensor encoder_hidden_states) = 0; virtual void set_adapters(AdapterController& adapter_controller, const AdapterConfig& adapters) = 0; virtual ov::Tensor infer(ov::Tensor sample, ov::Tensor timestep) = 0; + virtual std::shared_ptr get_compiled_model() = 0; // utility function to resize model given optional dimensions. static void reshape(std::shared_ptr model, @@ -62,6 +63,8 @@ class UNet2DConditionModel::UNetInference { model->reshape(name_to_shape); } + UNetInference(const UNetInference & ); + UNetInference() = default; }; } // namespace genai diff --git a/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp b/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp index d06a521626..3cc4950710 100644 --- a/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp +++ b/src/cpp/src/image_generation/models/unet_inference_dynamic.hpp @@ -21,13 +21,11 @@ class UNet2DConditionModel::UNetInferenceDynamic : public UNet2DConditionModel:: compiled_model = std::make_shared(core.compile_model(model, device, properties)); ov::genai::utils::print_compiled_model_properties(*compiled_model, "UNet 2D Condition dynamic model"); - std::cout << "unet dynamic compile" << std::endl; m_request = compiled_model->create_infer_request(); } virtual void set_hidden_states(const std::string& tensor_name, ov::Tensor encoder_hidden_states) override { - std::cout << "unet dynamic set_hidden_states" << std::endl; OPENVINO_ASSERT(m_request, "UNet model must be compiled first"); m_request.set_tensor(tensor_name, encoder_hidden_states); } @@ -40,27 +38,26 @@ class UNet2DConditionModel::UNetInferenceDynamic : public UNet2DConditionModel:: virtual ov::Tensor infer(ov::Tensor sample, ov::Tensor timestep) override { - std::cout << "unet dynamic infer" << std::endl; OPENVINO_ASSERT(m_request, "UNet model must be compiled first. Cannot infer non-compiled model"); - std::cout << sample.get_shape().to_string() << std::endl; m_request.set_tensor("sample", sample); - std::cout << timestep.get_shape().to_string() << std::endl; m_request.set_tensor("timestep", timestep); ov::CompiledModel test = m_request.get_compiled_model(); - std::cout << "compiled model ok" << test.inputs().at(0).get_any_name() << std::endl; + ov::genai::utils::print_compiled_model_properties(test, "UNet 2D Condition TEST"); m_request.infer(); - std::cout << "unet dynamic inference complated" << std::endl; - return m_request.get_output_tensor(); } - UNetInferenceDynamic(const UNetInferenceDynamic & origin_model){ - OPENVINO_ASSERT(origin_model.compiled_model, "Source model must be compiled first"); - compiled_model = origin_model.compiled_model; + UNetInferenceDynamic(std::shared_ptr origin_compiled_model){ + compiled_model = origin_compiled_model; m_request = compiled_model->create_infer_request(); } + UNetInferenceDynamic() = default; + std::shared_ptr get_compiled_model(){ + return compiled_model; + } + private: ov::InferRequest m_request; diff --git a/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp b/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp index 9525ae480c..ef6f4f8fff 100644 --- a/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp +++ b/src/cpp/src/image_generation/models/unet_inference_static_bs1.hpp @@ -14,14 +14,16 @@ namespace genai { class UNet2DConditionModel::UNetInferenceStaticBS1 : public UNet2DConditionModel::UNetInference { public: UNetInferenceStaticBS1() = default; - UNetInferenceStaticBS1(const UNetInferenceStaticBS1 & origin_model){ - OPENVINO_ASSERT(origin_model.compiled_model, "Source model must be compiled first"); - compiled_model = origin_model.compiled_model; - size_t m_native_batch_size = origin_model.m_native_batch_size; + + UNetInferenceStaticBS1(const std::shared_ptr & origin_compiled_model){ + OPENVINO_ASSERT(origin_compiled_model, "Source model must be compiled first"); + compiled_model = origin_compiled_model; + m_native_batch_size = compiled_model->input("sample").get_shape()[0]; for (int i = 0; i < m_native_batch_size; i++) { m_requests[i] = compiled_model->create_infer_request(); } } + virtual void compile(std::shared_ptr model, const std::string& device, const ov::AnyMap& properties) override { @@ -145,6 +147,9 @@ class UNet2DConditionModel::UNetInferenceStaticBS1 : public UNet2DConditionModel return out_sample; } + std::shared_ptr get_compiled_model(){ + return compiled_model; + } private: std::shared_ptr compiled_model; diff --git a/src/cpp/src/utils.cpp b/src/cpp/src/utils.cpp index 9fa14b7f9f..18438359c7 100644 --- a/src/cpp/src/utils.cpp +++ b/src/cpp/src/utils.cpp @@ -407,6 +407,12 @@ void print_compiled_model_properties(ov::CompiledModel& compiled_Model, const ch std::cout << " " << cfg << ": " << prop.as() << std::endl; } } + for (const auto& input : compiled_Model.inputs()) { + std::cout << "Input name: " << input.get_any_name() << ", shape: " << input.get_partial_shape().to_string() << std::endl; + } + for (const auto& out : compiled_Model.outputs()) { + std::cout << "Output name: " << out.get_any_name() << ", shape: " << out.get_partial_shape().to_string() << std::endl; + } ov::Core core; std::vector exeTargets;