diff --git a/assets/qwen/qwen-image-2.1-alpha-in1.png b/assets/qwen/qwen-image-2.1-alpha-in1.png new file mode 100644 index 00000000..e65fcf07 Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-in1.png differ diff --git a/assets/qwen/qwen-image-2.1-alpha-out1.png b/assets/qwen/qwen-image-2.1-alpha-out1.png new file mode 100644 index 00000000..6841b7f7 Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out1.png differ diff --git a/assets/qwen/qwen-image-2.1-alpha-out2.png b/assets/qwen/qwen-image-2.1-alpha-out2.png new file mode 100644 index 00000000..b3db6c89 Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out2.png differ diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md index 08a50875..31008395 100644 --- a/docs/qwen_image_2.1.md +++ b/docs/qwen_image_2.1.md @@ -39,3 +39,21 @@ Pass the reference image with `-r` and describe the edit in `-p`. Vision weights ``` For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`. + +### Alpha channel + +This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results: + +> `This is an RGBA image with transparency. . The image has alpha channel and the background is transparent.` + +Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency. + +Here are some examples ran with Q6_K quantization: +| Input | Prompt | Output | +| --- | --- | --- | +| ![Qwen Image 2.1 alpha input example 1](../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../assets/qwen/qwen-image-2.1-alpha-out1.png) | +| ![Qwen Image 2.1 alpha input example 2](../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../assets/qwen/qwen-image-2.1-alpha-out2.png) | + +### Other features + +Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model. diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index eb96d9b5..1a0bb28e 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -357,7 +357,8 @@ bool load_images_from_dir(const std::string dir, LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str()); int width = 0; int height = 0; - uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height); + int loaded_channel = 0; + uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, expected_width, expected_height); if (image_buffer == nullptr) { LOG_ERROR("load image from '%s' failed", path.c_str()); return false; @@ -365,7 +366,7 @@ bool load_images_from_dir(const std::string dir, images.emplace_back(sd_image_t{(uint32_t)width, (uint32_t)height, - 3, + (uint32_t)loaded_channel, image_buffer}); if (max_image_num > 0 && static_cast(images.size()) >= max_image_num) { @@ -781,7 +782,8 @@ int main(int argc, const char* argv[]) { }; if (gen_params.init_image_path.size() > 0) { - if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) { + const bool native_init = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; + if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image, true, native_init ? 0 : 3)) { return 1; } } @@ -795,8 +797,8 @@ int main(int argc, const char* argv[]) { if (gen_params.ref_image_paths.size() > 0) { gen_params.ref_images.clear(); for (auto& path : gen_params.ref_image_paths) { - SDImageOwner ref_image({0, 0, 3, nullptr}); - if (!load_image_and_update_size(path, ref_image, false)) { + SDImageOwner ref_image({0, 0, 0, nullptr}); + if (!load_image_and_update_size(path, ref_image, false, 0)) { return 1; } gen_params.ref_images.push_back(std::move(ref_image)); diff --git a/examples/common/common.cpp b/examples/common/common.cpp index cc97f669..25f78d3b 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1848,20 +1848,22 @@ bool decode_base64_image(const std::string& encoded_input, return false; } - int decoded_width = 0; - int decoded_height = 0; - uint8_t* raw_data = load_image_from_memory(reinterpret_cast(image_bytes.data()), - static_cast(image_bytes.size()), - decoded_width, - decoded_height, - expected_width, - expected_height, - target_channels); + int decoded_width = 0; + int decoded_height = 0; + int resolved_channel = target_channels; + uint8_t* raw_data = load_image_from_memory(reinterpret_cast(image_bytes.data()), + static_cast(image_bytes.size()), + decoded_width, + decoded_height, + resolved_channel, + expected_width, + expected_height, + target_channels); if (raw_data == nullptr) { return false; } - out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)target_channels, raw_data}); + out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)resolved_channel, raw_data}); return true; } @@ -2215,7 +2217,7 @@ bool SDGenerationParams::from_json_str( LOG_ERROR("invalid lora"); return false; } - if (!parse_image_json_field(j, "init_image", 3, width, height, init_image)) { + if (!parse_image_json_field(j, "init_image", 0, width, height, init_image)) { LOG_ERROR("invalid init_image"); return false; } @@ -2225,7 +2227,7 @@ bool SDGenerationParams::from_json_str( } if (!parse_image_array_json_field(j, "ref_images", - 3, + 0, auto_resize_ref_image ? width : 0, auto_resize_ref_image ? height : 0, ref_images)) { diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index 345b6f15..ba0e8c5b 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -261,6 +261,10 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data, height = features.height; source_channel_count = features.has_alpha ? 4 : 3; + if (expected_channel == 0) { + expected_channel = source_channel_count; + } + const size_t pixel_count = static_cast(width) * static_cast(height); if (expected_channel == 1) { @@ -481,7 +485,8 @@ uint8_t* load_image_common(bool from_memory, int& height, int expected_width, int expected_height, - int expected_channel) { + int expected_channel, + int& out_channel) { const char* image_path; FreeUniquePtr image_buffer; int source_channel_count = 0; @@ -538,6 +543,32 @@ uint8_t* load_image_common(bool from_memory, LOG_ERROR("load image from '%s' failed", image_path); return nullptr; } + if (expected_channel == 0) { + expected_channel = source_channel_count == 2 ? 4 : (source_channel_count == 1 ? 3 : source_channel_count); + if (expected_channel != source_channel_count) { + FreeUniquePtr promoted((uint8_t*)malloc((size_t)width * height * expected_channel)); + if (promoted == nullptr) { + LOG_ERROR("error: allocate memory for channel promotion, image_path = %s", image_path); + return nullptr; + } + const size_t pixel_count = (size_t)width * (size_t)height; + for (size_t i = 0; i < pixel_count; ++i) { + if (source_channel_count == 1) { + promoted.get()[i * 3 + 0] = image_buffer.get()[i]; + promoted.get()[i * 3 + 1] = image_buffer.get()[i]; + promoted.get()[i * 3 + 2] = image_buffer.get()[i]; + } else { + promoted.get()[i * 4 + 0] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 1] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 2] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 3] = image_buffer.get()[i * 2 + 1]; + } + } + image_buffer = std::move(promoted); + source_channel_count = expected_channel; + } + } + // stb reports the source channel count even when it converts the output. if (source_channel_count < expected_channel) { fprintf(stderr, "the number of channels for the input image must be >= %d," @@ -597,7 +628,7 @@ uint8_t* load_image_common(bool from_memory, } stbir_resize(image_buffer.get(), width, height, 0, resized_image_buffer.get(), expected_width, expected_height, 0, STBIR_TYPE_UINT8, - expected_channel, STBIR_ALPHA_CHANNEL_NONE, 0, + expected_channel, expected_channel == 4 ? 3 : STBIR_ALPHA_CHANNEL_NONE, 0, STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_FILTER_BOX, STBIR_FILTER_BOX, STBIR_COLORSPACE_SRGB, nullptr); @@ -605,6 +636,7 @@ uint8_t* load_image_common(bool from_memory, height = expected_height; image_buffer = std::move(resized_image_buffer); } + out_channel = expected_channel; return image_buffer.release(); } @@ -777,10 +809,11 @@ bool write_image_to_file(const std::string& path, uint8_t* load_image_from_file(const char* image_path, int& width, int& height, + int& out_channel, int expected_width, int expected_height, int expected_channel) { - return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel); + return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel); } bool load_sd_image_from_file(sd_image_t* image, @@ -790,13 +823,14 @@ bool load_sd_image_from_file(sd_image_t* image, int expected_channel) { int width; int height; - image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel); + int resolved_channel = expected_channel; + image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, resolved_channel); if (image->data == nullptr) { return false; } image->width = width; image->height = height; - image->channel = expected_channel; + image->channel = resolved_channel; return true; } @@ -804,10 +838,11 @@ uint8_t* load_image_from_memory(const char* image_bytes, int len, int& width, int& height, + int& out_channel, int expected_width, int expected_height, int expected_channel) { - return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel); + return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel, out_channel); } static void append_avi_metadata(std::vector& data, const std::string& parameters) { diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 6adbe987..b2b56ecc 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -32,9 +32,12 @@ bool write_image_to_file(const std::string& path, const std::string& parameters = "", int quality = 90); +// expected_channel == 0 preserves native channels (grayscale -> RGB, gray+alpha -> RGBA). +// out_channel receives the output channel count. uint8_t* load_image_from_file(const char* image_path, int& width, int& height, + int& out_channel, int expected_width = 0, int expected_height = 0, int expected_channel = 3); @@ -49,6 +52,7 @@ uint8_t* load_image_from_memory(const char* image_bytes, int len, int& width, int& height, + int& out_channel, int expected_width = 0, int expected_height = 0, int expected_channel = 3); diff --git a/examples/server/api.md b/examples/server/api.md index 3ec9a1da..ffcc96e2 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -735,12 +735,15 @@ Any image field accepts: Channel expectations: -- `init_image`: 3 channels -- `ref_images[]`: 3 channels +- `init_image`: native channels (3 or 4); alpha is preserved and applied per model +- `ref_images[]`: native channels (3 or 4); alpha is preserved and applied per model - `control_image`: 3 channels - `ip_adapter_image`: 3 channels - `mask_image`: 1 channel +Models that support RGBA (e.g. Qwen-Image 2.1) use the alpha channel of `init_image` +and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every model. + If omitted or null: - single-image fields map to an empty `sd_image_t` diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index ccde2bae..de36091c 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -158,19 +158,20 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.batch_count = n; for (auto& bytes : images_bytes) { - int img_w = 0; - int img_h = 0; - uint8_t* raw_pixels = load_image_from_memory( - reinterpret_cast(bytes.data()), - static_cast(bytes.size()), - img_w, img_h, - 0, 0, 3); + int img_w = 0; + int img_h = 0; + int resolved_channel = 0; + uint8_t* raw_pixels = load_image_from_memory( + reinterpret_cast(bytes.data()), + static_cast(bytes.size()), + img_w, img_h, resolved_channel, + 0, 0, 0); if (raw_pixels == nullptr) { continue; } const bool is_first_ref_image = request.gen_params.ref_images.empty(); - SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels}); + SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, (uint32_t)resolved_channel, raw_pixels}); request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height); if (is_first_ref_image) { @@ -181,15 +182,16 @@ static bool build_openai_edit_request(const httplib::Request& req, init_h = request.gen_params.height; } - int init_img_w = 0; - int init_img_h = 0; - uint8_t* init_pixels = load_image_from_memory( - reinterpret_cast(bytes.data()), - static_cast(bytes.size()), - init_img_w, init_img_h, - init_w, init_h, 3); + int init_img_w = 0; + int init_img_h = 0; + int init_resolved_channel = 0; + uint8_t* init_pixels = load_image_from_memory( + reinterpret_cast(bytes.data()), + static_cast(bytes.size()), + init_img_w, init_img_h, init_resolved_channel, + init_w, init_h, 0); if (init_pixels != nullptr) { - request.gen_params.init_image.reset({(uint32_t)init_img_w, (uint32_t)init_img_h, 3, init_pixels}); + request.gen_params.init_image.reset({(uint32_t)init_img_w, (uint32_t)init_img_h, (uint32_t)init_resolved_channel, init_pixels}); } } @@ -203,13 +205,14 @@ static bool build_openai_edit_request(const httplib::Request& req, expected_width = request.gen_params.width; expected_height = request.gen_params.height; } - int mask_w = 0; - int mask_h = 0; + int mask_w = 0; + int mask_h = 0; + int mask_channel = 0; uint8_t* mask_raw = load_image_from_memory( reinterpret_cast(mask_bytes.data()), static_cast(mask_bytes.size()), - mask_w, mask_h, + mask_w, mask_h, mask_channel, expected_width, expected_height, 1); request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw}); const sd_image_t& mask_image = request.gen_params.mask_image.get(); diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index f6bd0949..1966a08f 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -199,7 +199,7 @@ static bool build_sdapi_img_gen_request(const json& j, if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) { if (decode_base64_image(j["init_images"][0].get(), - 3, + 0, expected_width, expected_height, request.gen_params.init_image)) { @@ -243,7 +243,7 @@ static bool build_sdapi_img_gen_request(const json& j, } SDImageOwner image_owner; if (decode_base64_image(extra_image.get(), - 3, + 0, request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0, diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 503afa53..3127ecb6 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -470,11 +470,15 @@ namespace sd::pipeline { sd::Tensor end_image; if (sd_vid_gen_params->init_image.data) { - start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height); + start_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height), + sd->get_image_channels()); } if (sd_vid_gen_params->end_image.data) { - end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height); + end_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height), + sd->get_image_channels()); } if (sd_version_is_minimax_h3(sd->version)) { @@ -1416,7 +1420,9 @@ namespace sd::pipeline { sd::Tensor video_mask = make_ltxav_video_denoise_mask(video_latent, 1.f); if (sd_vid_gen_params->init_image.data != nullptr) { - sd::Tensor start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height); + sd::Tensor start_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height), + sd->get_image_channels()); if (!apply_ltxav_condition_image_by_latent_index(sd, start_image, &video_latent, @@ -1429,7 +1435,9 @@ namespace sd::pipeline { } if (sd_vid_gen_params->end_image.data != nullptr) { - sd::Tensor end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height); + sd::Tensor end_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height), + sd->get_image_channels()); sd::Tensor end_image_latent = encode_ltxav_condition_image(sd, end_image, "end"); if (end_image_latent.empty()) { return false;