feat: add configurable image input preprocessing (#2028)

examples/server/api.md

@@ -148,17 +148,17 @@ Native extension fields:
 
 - any `sdcpp API` fields embedded through `sd_cpp_extra_args` inside `prompt`
 
-Reference image sizing follows `auto_resize_ref_image`, as in the native and SDAPI APIs.
-The server default is `true`; `--disable-auto-resize-ref-image` sets it to `false`.
-When enabled, uploaded references are center-cropped and resized to the request dimensions.
-If `size` is omitted, the first decoded image establishes those dimensions.
-When disabled, each reference retains its original dimensions.
-The init image and mask still use the request dimensions, independently of this option.
+Uploaded images are decoded at their original dimensions. The first decoded
+image establishes the generation dimensions if `size` is omitted. Input
+geometry follows `image_preprocess`: references preserve their dimensions by
+default, while init and mask use the generation canvas preset.
 
-To override the server default for one request, include this in `prompt`:
+Reference encoding then follows model presets and `ref_image_args`. To skip
+input geometry for references and disable resizing before VAE encoding, include
+this in `prompt`:
 
 ```text
-edit this image <sd_cpp_extra_args>{"auto_resize_ref_image":false}</sd_cpp_extra_args>
+edit this image <sd_cpp_extra_args>{"image_preprocess":"target=ref,mode=none","ref_image_args":"resize_before_vae=false"}</sd_cpp_extra_args>
 ```
 
 Response fields:
@@ -539,7 +539,7 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
 | Field | Type |
 | --- | --- |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |
@@ -666,7 +666,7 @@ Example:
   "strength": 0.75,
   "seed": -1,
   "batch_count": 1,
-  "auto_resize_ref_image": true,
+  "ref_image_args": "",
   "increase_ref_index": false,
   "control_strength": 0.9,
   "ip_adapter_strength": 1.0,
@@ -741,6 +741,17 @@ Example:
 
 ### Image Encoding Rules
 
+Native image/video requests and SDAPI accept `image_preprocess` as a rule string
+or array of rule strings. OpenAI-compatible requests can supply it in
+`sd_cpp_extra_args`. See [Image preprocessing](../../docs/image_preprocessing.md)
+for one-time input geometry, native-resolution decoding, mask alignment, and
+`canny=true` for edge detection on any supported image input.
+
+Image generation also accepts `ref_image_args` as a string (for example,
+`"resize_before_vae=false"`) in native and SDAPI requests, or through
+`sd_cpp_extra_args` in OpenAI-compatible requests. It controls downstream
+reference encoding and is independent of input geometry rules.
+
 Any image field accepts:
 
 - a raw base64 string, or
@@ -776,7 +787,8 @@ Top-level scalar fields:
 | `strength` | `number` |
 | `seed` | `integer` |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
+| `image_preprocess` | `string \| array<string>` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |

examples/server/main.cpp

@@ -76,9 +76,9 @@ int main(int argc, const char** argv) {
     SDSvrParams svr_params;
     SDContextParams ctx_params;
     SDGenerationParams default_gen_params;
-    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     sd_set_log_callback(sd_log_cb, (void*)&svr_params);
+    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     LOG_VERBOSE("version: %s", version_string().c_str());
     LOG_VERBOSE("%s", sd_get_system_info());

examples/server/routes_openai.cpp

@@ -158,18 +158,6 @@ static bool build_openai_edit_request(const httplib::Request& req,
     request.gen_params.batch_count = n;
 
     std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
-    if (!sd_cpp_extra_args_str.empty()) {
-        const json extra_args = json::parse(sd_cpp_extra_args_str, nullptr, false);
-        if (extra_args.is_discarded()) {
-            error_message = "invalid sd_cpp_extra_args";
-            return false;
-        }
-        // Resolve resizing before decoding while keeping embedded image overrides last.
-        if (extra_args.contains("auto_resize_ref_image") && extra_args["auto_resize_ref_image"].is_boolean()) {
-            request.gen_params.auto_resize_ref_image = extra_args["auto_resize_ref_image"].get<bool>();
-        }
-    }
-
     for (auto& bytes : images_bytes) {
         int img_w            = 0;
         int img_h            = 0;
@@ -178,12 +166,7 @@ static bool build_openai_edit_request(const httplib::Request& req,
              reinterpret_cast<const char*>(bytes.data()),
              static_cast<int>(bytes.size()),
              img_w, img_h, resolved_channel,
-            request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set()
-                 ? request.gen_params.width
-                 : 0,
-            request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set()
-                 ? request.gen_params.height
-                 : 0,
+             0, 0,
              0);
         if (raw_pixels == nullptr) {
             continue;
@@ -194,23 +177,10 @@ static bool build_openai_edit_request(const httplib::Request& req,
         request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
 
         if (is_first_ref_image) {
-            int init_w = 0;
-            int init_h = 0;
-            if (request.gen_params.width_and_height_are_set()) {
-                init_w = request.gen_params.width;
-                init_h = request.gen_params.height;
-            }
-
-            int init_img_w            = 0;
-            int init_img_h            = 0;
-            int init_resolved_channel = 0;
-            uint8_t* init_pixels      = load_image_from_memory(
-                     reinterpret_cast<const char*>(bytes.data()),
-                     static_cast<int>(bytes.size()),
-                     init_img_w, init_img_h, init_resolved_channel,
-                     init_w, init_h, 0);
-            if (init_pixels != nullptr) {
-                request.gen_params.init_image.reset({(uint32_t)init_img_w, (uint32_t)init_img_h, (uint32_t)init_resolved_channel, init_pixels});
+            request.gen_params.init_image = image_owner;
+            if (request.gen_params.init_image.get().data == nullptr) {
+                error_message = "could not allocate init image";
+                return false;
             }
         }
 
@@ -218,12 +188,6 @@ static bool build_openai_edit_request(const httplib::Request& req,
     }
 
     if (!mask_bytes.empty()) {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (request.gen_params.width_and_height_are_set()) {
-            expected_width  = request.gen_params.width;
-            expected_height = request.gen_params.height;
-        }
         int mask_w       = 0;
         int mask_h       = 0;
         int mask_channel = 0;
@@ -232,7 +196,7 @@ static bool build_openai_edit_request(const httplib::Request& req,
             reinterpret_cast<const char*>(mask_bytes.data()),
             static_cast<int>(mask_bytes.size()),
             mask_w, mask_h, mask_channel,
-            expected_width, expected_height, 1);
+            0, 0, 1);
         request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
         const sd_image_t& mask_image = request.gen_params.mask_image.get();
         request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);

examples/server/routes_sdapi.cpp

@@ -80,17 +80,6 @@ static enum sample_method_t get_sdapi_sample_method(std::string name) {
     return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
 }
 
-static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
-    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
-    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
-    if (raw_mask == nullptr) {
-        mask_owner.reset({0, 0, 1, nullptr});
-        return;
-    }
-    std::memset(raw_mask, 255, pixel_count);
-    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
-}
-
 static bool build_sdapi_img_gen_request(const json& j,
                                         ServerRuntime& runtime,
                                         bool img2img,
@@ -193,15 +182,25 @@ static bool build_sdapi_img_gen_request(const json& j,
         }
     }
 
-    if (img2img) {
-        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
-        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
+    if (j.contains("ref_image_args")) {
+        if (!j["ref_image_args"].is_string()) {
+            error_message = "ref_image_args must be a string";
+            return false;
+        }
+        request.gen_params.ref_image_args = j["ref_image_args"].get<std::string>();
+    }
 
+    if (j.contains("image_preprocess") && !request.gen_params.parse_image_preprocess_json(j["image_preprocess"].dump())) {
+        error_message = "invalid image_preprocess";
+        return false;
+    }
+
+    if (img2img) {
         if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
             if (decode_base64_image(j["init_images"][0].get<std::string>(),
                                     0,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
                                     request.gen_params.init_image)) {
                 const sd_image_t& image = request.gen_params.init_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -211,8 +210,8 @@ static bool build_sdapi_img_gen_request(const json& j,
         if (j.contains("mask") && j["mask"].is_string()) {
             if (decode_base64_image(j["mask"].get<std::string>(),
                                     1,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
                                     request.gen_params.mask_image)) {
                 const sd_image_t& image = request.gen_params.mask_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -225,9 +224,7 @@ static bool build_sdapi_img_gen_request(const json& j,
                 }
             }
         } else {
-            const int resolved_width  = request.gen_params.get_resolved_width();
-            const int resolved_height = request.gen_params.get_resolved_height();
-            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
+            request.gen_params.mask_image.reset({0, 0, 1, nullptr});
         }
 
         float denoising_strength = j.value("denoising_strength", -1.f);
@@ -244,12 +241,7 @@ static bool build_sdapi_img_gen_request(const json& j,
             SDImageOwner image_owner;
             if (decode_base64_image(extra_image.get<std::string>(),
                                     0,
-                                    request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set()
-                                        ? request.gen_params.width
-                                        : 0,
-                                    request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set()
-                                        ? request.gen_params.height
-                                        : 0,
+                                    0, 0,
                                     image_owner)) {
                 const sd_image_t& image = image_owner.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);

examples/server/routes_sdcpp.cpp

@@ -127,7 +127,8 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
         {"seed", defaults.seed},
         {"batch_count", defaults.batch_count},
         {"qwen_image_layers", defaults.qwen_image_layers},
-        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
+        {"ref_image_args", defaults.ref_image_args},
+        {"image_preprocess", defaults.image_preprocess},
         {"increase_ref_index", defaults.increase_ref_index},
         {"control_strength", defaults.control_strength},
         {"ip_adapter_strength", defaults.ip_adapter_strength},
@@ -153,6 +154,7 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const
         {"strength", defaults.strength},
         {"seed", defaults.seed},
         {"video_frames", defaults.video_frames},
+        {"image_preprocess", defaults.image_preprocess},
         {"fps", defaults.fps},
         {"moe_boundary", defaults.moe_boundary},
         {"vace_strength", defaults.vace_strength},