fix(server): route OpenAI image edits by reference conditioning support (#2167)

examples/server/api.md

@@ -155,6 +155,10 @@ image establishes the generation dimensions if `size` is omitted. Input
 geometry follows `image_preprocess`: references preserve their dimensions by
 default, while init and mask use the generation canvas preset.
 
+When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection.
+
+Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection.
+
 Reference encoding then follows model presets and `ref_image_args`. To skip
 input geometry for references and disable resizing before VAE encoding, include
 this in `prompt`:

examples/server/openai_edit_policy.h

@@ -0,0 +1,44 @@
+#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+
+#include "common/resource_owners.hpp"
+
+enum class OpenAIEditInitImageAction {
+    KeepExisting,
+    UseUploaded,
+    Clear,
+};
+
+inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning,
+                                                    bool has_ref_images,
+                                                    bool has_mask,
+                                                    bool has_explicit_strength_override) {
+    return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override;
+}
+
+inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy(
+    SDImageOwner& init_image,
+    SDImageOwner& uploaded_image,
+    bool uploaded_image_was_first_reference,
+    bool supports_ref_image_conditioning,
+    bool has_ref_images,
+    bool has_mask,
+    bool has_extra_init_image_override,
+    bool has_explicit_strength_override) {
+    if (!uploaded_image_was_first_reference || has_extra_init_image_override) {
+        return OpenAIEditInitImageAction::KeepExisting;
+    }
+
+    if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning,
+                                                has_ref_images,
+                                                has_mask,
+                                                has_explicit_strength_override)) {
+        init_image = std::move(uploaded_image);
+        return OpenAIEditInitImageAction::UseUploaded;
+    }
+
+    init_image.reset();
+    return OpenAIEditInitImageAction::Clear;
+}
+
+#endif  // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__

examples/server/routes_openai.cpp

@@ -7,6 +7,7 @@
 #include "common/common.h"
 #include "common/media_io.h"
 #include "common/resource_owners.hpp"
+#include "openai_edit_policy.h"
 
 static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
     std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
@@ -157,7 +158,24 @@ static bool build_openai_edit_request(const httplib::Request& req,
     request.gen_params.height      = height;
     request.gen_params.batch_count = n;
 
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    std::string sd_cpp_extra_args_str          = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    const json extra_args                      = sd_cpp_extra_args_str.empty()
+                                                     ? json::object()
+                                                     : json::parse(sd_cpp_extra_args_str, nullptr, false);
+    const bool has_extra_init_image_override   = extra_args.is_object() && extra_args.contains("init_image");
+    const bool has_extra_ref_images_override   = extra_args.is_object() && extra_args.contains("ref_images");
+    const bool has_extra_strength_override     = extra_args.is_object() &&
+                                                 extra_args.contains("strength") &&
+                                                 extra_args["strength"].is_number();
+    const bool has_extra_mask_image            = extra_args.is_object() &&
+                                                 extra_args.contains("mask_image") &&
+                                                 !extra_args["mask_image"].is_null();
+    const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx);
+    const bool may_need_uploaded_init_image    = !has_extra_init_image_override &&
+                                                 (!supports_ref_image_conditioning || !mask_bytes.empty() ||
+                                                  has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override);
+    SDImageOwner uploaded_image_for_init;
+    bool uploaded_image_was_first_reference = false;
     for (auto& bytes : images_bytes) {
         int img_w            = 0;
         int img_h            = 0;
@@ -177,10 +195,9 @@ static bool build_openai_edit_request(const httplib::Request& req,
         request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
 
         if (is_first_ref_image) {
-            request.gen_params.init_image = image_owner;
-            if (request.gen_params.init_image.get().data == nullptr) {
-                error_message = "could not allocate init image";
-                return false;
+            uploaded_image_was_first_reference = true;
+            if (may_need_uploaded_init_image) {
+                uploaded_image_for_init = image_owner;
             }
         }
 
@@ -214,6 +231,21 @@ static bool build_openai_edit_request(const httplib::Request& req,
         return false;
     }
 
+    const auto init_image_action = apply_openai_edit_init_image_policy(
+        request.gen_params.init_image,
+        uploaded_image_for_init,
+        uploaded_image_was_first_reference,
+        supports_ref_image_conditioning,
+        !request.gen_params.ref_images.empty(),
+        request.gen_params.mask_image.get().data != nullptr,
+        has_extra_init_image_override,
+        has_extra_strength_override);
+    if (init_image_action == OpenAIEditInitImageAction::UseUploaded &&
+        request.gen_params.init_image.get().data == nullptr) {
+        error_message = "could not allocate init image";
+        return false;
+    }
+
     // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
     if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
         error_message = "invalid params";

include/stable-diffusion.h

@@ -486,6 +486,8 @@ SD_API int32_t sd_get_num_physical_cores();
 SD_API const char* sd_get_system_info();
 SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
 SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
+/** Returns whether the loaded model consumes ref_images as conditioning inputs. */
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx);
 
 // ControlNet hot-swap APIs are not safe to call while generation is in flight.
 SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);

src/model.h

@@ -365,6 +365,29 @@ static inline bool sd_version_is_control(SDVersion version) {
     return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2;
 }
 
+// These model families consume uploaded reference images as generation
+// conditioning. Inpaint and control variants keep their init-image semantics.
+static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) {
+    if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) {
+        return false;
+    }
+
+    return sd_version_is_unet_edit(version) ||
+           sd_version_is_flux(version) ||
+           sd_version_is_flux3(version) ||
+           sd_version_is_flux2(version) ||
+           sd_version_is_qwen_image(version) ||
+           sd_version_is_longcat(version) ||
+           sd_version_is_z_image(version) ||
+           sd_version_is_boogu_image(version) ||
+           sd_version_is_llada_image(version) ||
+           sd_version_is_krea2(version) ||
+           sd_version_is_mage_flow(version) ||
+           sd_version_is_anima(version) ||
+           sd_version_is_sefi_image(version) ||
+           sd_version_is_pid(version);
+}
+
 static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) {
     return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version);
 }

src/stable-diffusion.cpp

@@ -704,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
     return sd_version_supports_video_generation(sd_ctx->sd->version);
 }
 
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) {
+    if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+        return false;
+    }
+    return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version);
+}
+
 SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
     if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
         return false;