diff --git a/examples/server/api.md b/examples/server/api.md index c5cae54c..9bcdc9f0 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -56,6 +56,7 @@ Current endpoints include: - `GET /sdcpp/v1/jobs/{id}` - `POST /sdcpp/v1/jobs/{id}/cancel` - `POST /sdcpp/v1/vid_gen` +- `POST /sdcpp/v1/upscale` ## `sd_cpp_extra_args` @@ -434,7 +435,8 @@ Top-level fields: | `samplers` | `array` | Available sampling methods | | `schedulers` | `array` | Available schedulers | | `loras` | `array` | Available LoRA entries | -| `upscalers` | `array` | Available model-backed highres upscalers | +| `upscalers` | `array` | Available highres upscalers, built-in and model-backed | +| `upscale` | `boolean` | Whether a compatible RGB ESRGAN model is available for `POST /sdcpp/v1/upscale` | | `limits` | `object` | Shared queue and size limits | `model` @@ -476,6 +478,8 @@ Shared nested fields: | Field | Type | Notes | | --- | --- | --- | | `upscalers[].name` | `string` | Built-in name or model stem; use this value in `hires.upscaler` | +| `upscalers[].model` | `boolean` | True for a model-backed upscaler, false for a built-in scaling filter | +| `upscalers[].image_upscale` | `boolean` | Whether this model can be selected by `POST /sdcpp/v1/upscale`; false for latent upscalers and built-in filters | Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (nearest)`, `Latent (nearest-exact)`, `Latent (antialiased)`, `Latent (bicubic)`, and `Latent (bicubic antialiased)`. Model-backed entries are scanned from the top level of `--hires-upscalers-dir`; subdirectories are not scanned. @@ -489,6 +493,8 @@ Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (neares | `limits.max_height` | `integer` | | `limits.max_batch_count` | `integer` | | `limits.max_queue_size` | `integer` | +| `limits.max_upscale_width` | `integer` | +| `limits.max_upscale_height` | `integer` | Shared default fields used by both `img_gen` and `vid_gen`: @@ -641,6 +647,52 @@ Typical status codes: - `404 Not Found` - `410 Gone` +#### `POST /sdcpp/v1/upscale` + +Runs one RGB ESRGAN upscaler over an image, with no generation involved. Latent upscaler models remain available for hires generation but cannot be used here. + +This is the HTTP equivalent of `sd-cli -M upscale`: no diffusion model, text +encoder or sampling is used, so it is fast enough to answer synchronously and +does not create a job. + +Request fields: + +| Field | Type | Notes | +| --- | --- | --- | +| `image` | `string` | Required. Base64 or data URL image | +| `upscaler` | `string` | A name from `upscalers` with `image_upscale: true`; the first compatible entry when omitted | +| `repeats` | `integer` | Run the upscaler this many times, 1 to 4 (default `1`) | +| `tile_size` | `integer` | Tile size, defaulting to the server's `--upscale-tile-size` | +| `output_format` | `string` | `png`, `jpeg`, or `webp` when built with WebP support (default `png`); unsupported formats return 400 | +| `output_compression` | `integer` | Range is clamped to `0..100` | + +Response fields: + +| Field | Type | Notes | +| --- | --- | --- | +| `images` | `array` | One image | +| `images[].index` | `integer` | | +| `images[].b64_json` | `string` | Base64-encoded image bytes | +| `upscaler` | `string` | The upscaler actually used | +| `scale` | `integer` | The model's scale factor | +| `repeats` | `integer` | How many times it was run | +| `width` | `integer` | Result width | +| `height` | `integer` | Result height | +| `output_format` | `string` | Final encoded image format | + +Typical status codes: + +- `200 OK` +- `400 Bad Request` (invalid request, unsupported output format, unreadable image, incompatible upscaler, or output dimensions exceeding the limit) +- `500 Internal Server Error` + +Notes: + +- Final output dimensions, including all repeats, must not exceed 8192 pixels on either axis (`limits.max_upscale_width` and `limits.max_upscale_height`). Requests exceeding this bound are rejected before upscaling. +- The upscaler models are three-channel; alpha is not preserved. +- The request holds the generation context lock, so an upscale and a + generation never run on the device at the same time. + #### `POST /sdcpp/v1/jobs/{id}/cancel` Attempts to cancel an accepted job. diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index a1eeb775..60e68ca2 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -3,12 +3,33 @@ #include #include #include +#include #include "async_jobs.h" #include "common/common.h" +#include "common/media_io.h" +#include "common/resource_owners.hpp" namespace fs = std::filesystem; +static constexpr uint32_t k_max_upscale_dimension = 8192; + +static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) { + if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) { + return false; + } + uint32_t width = image.width; + uint32_t height = image.height; + for (int i = 0; i < repeats; ++i) { + if (width > k_max_upscale_dimension / factor || height > k_max_upscale_dimension / factor) { + return false; + } + width *= factor; + height *= factor; + } + return true; +} + static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) { if (mode_str == "disabled") { mode_out = SD_CACHE_DISABLED; @@ -241,37 +262,59 @@ static json make_capabilities_json(ServerRuntime& runtime) { available_upscalers.push_back({ {"name", "None"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Lanczos"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Nearest"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent (nearest)"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent (nearest-exact)"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent (antialiased)"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent (bicubic)"}, + {"model", false}, + {"image_upscale", false}, }); available_upscalers.push_back({ {"name", "Latent (bicubic antialiased)"}, + {"model", false}, + {"image_upscale", false}, }); + bool have_upscaler_models = false; { std::lock_guard lock(*runtime.upscaler_mutex); for (const auto& entry : *runtime.upscaler_cache) { available_upscalers.push_back({ {"name", entry.name}, + {"model", true}, + {"image_upscale", entry.image_upscale_factor > 0}, }); + have_upscaler_models = have_upscaler_models || entry.image_upscale_factor > 0; } } @@ -341,6 +384,8 @@ static json make_capabilities_json(ServerRuntime& runtime) { {"max_height", 4096}, {"max_batch_count", 8}, {"max_queue_size", manager.max_pending_jobs}, + {"max_upscale_width", k_max_upscale_dimension}, + {"max_upscale_height", k_max_upscale_dimension}, }; result["samplers"] = samplers; result["schedulers"] = schedulers; @@ -350,6 +395,7 @@ static json make_capabilities_json(ServerRuntime& runtime) { result["features_by_mode"] = features_by_mode; result["loras"] = available_loras; result["upscalers"] = available_upscalers; + result["upscale"] = have_upscaler_models; return result; } @@ -415,6 +461,171 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(make_capabilities_json(*runtime).dump(), "application/json"); }); + svr.Post("/sdcpp/v1/upscale", [runtime](const httplib::Request& req, httplib::Response& res) { + try { + if (req.body.empty()) { + res.status = 400; + res.set_content(R"({"error":"empty body"})", "application/json"); + return; + } + json body = json::parse(req.body); + if (!body.is_object()) { + res.status = 400; + res.set_content(R"({"error":"body must be an object"})", "application/json"); + return; + } + for (const char* key : {"repeats", "tile_size", "output_compression"}) { + if (!body.contains(key)) { + continue; + } + const auto& value = body[key]; + const bool valid = value.is_number_unsigned() + ? value.get() <= static_cast(std::numeric_limits::max()) + : value.is_number_integer() && value.get() >= std::numeric_limits::min() && + value.get() <= std::numeric_limits::max(); + if (!valid) { + res.status = 400; + res.set_content(json({{"error", std::string(key) + " must be a 32-bit integer"}}).dump(), "application/json"); + return; + } + } + ImgGenJobRequest output_options; + std::string error_message; + if (!assign_output_options(output_options, + body.value("output_format", std::string("png")), + body.value("output_compression", 100), + true, + error_message)) { + res.status = 400; + res.set_content(json({{"error", error_message}}).dump(), "application/json"); + return; + } + const int tile_size = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size)); + const int repeats = std::clamp(body.value("repeats", 1), 1, 4); + const std::string wanted = body.value("upscaler", std::string()); + + const std::string encoded = body.value("image", std::string()); + if (encoded.empty()) { + res.status = 400; + res.set_content(R"({"error":"image is required"})", "application/json"); + return; + } + SDImageOwner input; + if (!decode_base64_image(encoded, 3, 0, 0, input) || input.get().data == nullptr) { + res.status = 400; + res.set_content(R"({"error":"image could not be read"})", "application/json"); + return; + } + + refresh_upscaler_cache(*runtime); + int model_scale = 0; + std::string model_path; + std::string used_name; + { + std::lock_guard lock(*runtime->upscaler_mutex); + for (const auto& entry : *runtime->upscaler_cache) { + if (entry.image_upscale_factor > 0 && (wanted.empty() || entry.name == wanted)) { + model_path = entry.fullpath; + used_name = entry.name; + model_scale = entry.image_upscale_factor; + break; + } + } + } + if (model_path.empty()) { + res.status = 400; + res.set_content(json({{"error", wanted.empty() + ? std::string("no RGB ESRGAN upscaler models are available; " + "start the server with --hires-upscalers-dir") + : "no compatible image upscaler called " + wanted}}) + .dump(), + "application/json"); + return; + } + + if (!valid_upscale_dimensions(input.get(), model_scale, repeats)) { + res.status = 400; + res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json"); + return; + } + + // One GPU: an upscale must not run while a generation is using it. + std::lock_guard ctx_lock(*runtime->sd_ctx_mutex); + UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(), + runtime->ctx_params->diffusion_conv_direct, + runtime->ctx_params->n_threads, + tile_size, + runtime->ctx_params->backend.c_str(), + runtime->ctx_params->params_backend.c_str())); + if (upscaler_ctx == nullptr) { + res.status = 500; + res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json"); + return; + } + const int factor = get_upscale_factor(upscaler_ctx.get()); + // The model file may have changed since its metadata was cached. + if (!valid_upscale_dimensions(input.get(), factor, repeats)) { + res.status = 400; + res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json"); + return; + } + + SDImageOwner current(input.release()); + for (int i = 0; i < repeats; ++i) { + sd_image_t* out_images = nullptr; + int out_count = 0; + if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) || + out_count <= 0 || out_images[0].data == nullptr) { + free_sd_images(out_images, out_count); + res.status = 500; + res.set_content(R"({"error":"upscale failed"})", "application/json"); + return; + } + sd_image_t produced = out_images[0]; + out_images[0] = {0, 0, 0, nullptr}; + free_sd_images(out_images, out_count); + current.reset(produced); + } + + const std::string& format = output_options.output_format; + const int compression = output_options.output_compression; + const sd_image_t result = current.get(); + auto image_bytes = encode_image_to_vector(format == "jpeg" ? EncodedImageFormat::JPEG + : format == "webp" ? EncodedImageFormat::WEBP + : EncodedImageFormat::PNG, + result.data, + result.width, + result.height, + result.channel, + "", + compression); + if (image_bytes.empty()) { + res.status = 500; + res.set_content(R"({"error":"the result could not be encoded"})", "application/json"); + return; + } + + json out; + out["upscaler"] = used_name; + out["scale"] = factor; + out["repeats"] = repeats; + out["width"] = result.width; + out["height"] = result.height; + out["output_format"] = format; + json images = json::array(); + images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}}); + out["images"] = std::move(images); + res.set_content(out.dump(), "application/json"); + res.status = 200; + } catch (const json::exception& e) { + res.status = 400; + res.set_content(json({{"error", "invalid request"}, {"message", e.what()}}).dump(), "application/json"); + } catch (const std::exception& e) { + res.status = 500; + res.set_content(json({{"error", std::string("server_error: ") + e.what()}}).dump(), "application/json"); + } + }); + svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) { try { if (req.body.empty()) { diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp index cd0e2e72..beb553e4 100644 --- a/examples/server/runtime.cpp +++ b/examples/server/runtime.cpp @@ -295,6 +295,11 @@ std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) { void refresh_upscaler_cache(ServerRuntime& rt) { std::vector new_cache; + std::vector previous_cache; + { + std::lock_guard lock(*rt.upscaler_mutex); + previous_cache = *rt.upscaler_cache; + } fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir; if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) { @@ -308,10 +313,24 @@ void refresh_upscaler_cache(ServerRuntime& rt) { } UpscalerEntry upscaler_entry; - upscaler_entry.name = p.stem().u8string(); - upscaler_entry.fullpath = fs::absolute(p).lexically_normal().u8string(); - upscaler_entry.model_name = "ESRGAN_4x"; - upscaler_entry.path = p.filename().u8string(); + upscaler_entry.name = p.stem().u8string(); + upscaler_entry.fullpath = fs::absolute(p).lexically_normal().u8string(); + upscaler_entry.model_name = "ESRGAN_4x"; + upscaler_entry.path = p.filename().u8string(); + upscaler_entry.file_size = entry.file_size(); + upscaler_entry.last_modified = entry.last_write_time(); + auto previous = std::find_if(previous_cache.begin(), previous_cache.end(), [&](const UpscalerEntry& cached) { + return cached.fullpath == upscaler_entry.fullpath && + cached.file_size == upscaler_entry.file_size && + cached.last_modified == upscaler_entry.last_modified; + }); + upscaler_entry.image_upscale_factor = previous != previous_cache.end() + ? previous->image_upscale_factor + : get_upscaler_model_scale(upscaler_entry.fullpath.c_str()); + if (upscaler_entry.image_upscale_factor > 0) { + upscaler_entry.scale = upscaler_entry.image_upscale_factor; + upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x"; + } new_cache.push_back(std::move(upscaler_entry)); } diff --git a/examples/server/runtime.h b/examples/server/runtime.h index 677bb0ba..b4560f12 100644 --- a/examples/server/runtime.h +++ b/examples/server/runtime.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,9 @@ struct UpscalerEntry { std::string fullpath; std::string model_name; int scale = 4; + int image_upscale_factor = 0; + uintmax_t file_size = 0; + std::filesystem::file_time_type last_modified; }; struct ServerRuntime { diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index a2bfdb68..616bc377 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -558,6 +558,8 @@ SD_API bool upscale(upscaler_ctx_t* upscaler_ctx, int* num_images_out); SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx); +// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model. +SD_API int get_upscaler_model_scale(const char* model_path); typedef struct adetailer_ctx_t adetailer_ctx_t; diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 6f1b598e..32516823 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1349,9 +1349,8 @@ bool is_first_stage_model_name(const std::string& name) { } static std::string convert_esrgan_tensor_name(std::string name) { - static std::unordered_map esrgan_name_map; - - if (esrgan_name_map.empty()) { + static const auto esrgan_name_map = [] { + std::unordered_map esrgan_name_map; esrgan_name_map["model.0."] = "conv_first."; constexpr int max_num_blocks = 64; @@ -1377,7 +1376,8 @@ static std::string convert_esrgan_tensor_name(std::string name) { esrgan_name_map["model.7."] = "conv_last."; esrgan_name_map["model.8."] = "conv_hr."; esrgan_name_map["model.10."] = "conv_last."; - } + return esrgan_name_map; + }(); replace_with_prefix_map(name, esrgan_name_map); return name; diff --git a/src/upscaler.cpp b/src/upscaler.cpp index 3c72dbf3..dc3498e6 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -262,6 +262,30 @@ int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { return upscaler_ctx->upscaler->esrgan_upscaler->config.scale; } +int get_upscaler_model_scale(const char* model_path) { + if (model_path == nullptr || model_path[0] == '\0') { + return 0; + } + try { + ModelLoader loader; + if (!loader.init_from_file_and_convert_name(model_path, "", VERSION_ESRGAN)) { + return 0; + } + const auto& tensors = loader.get_tensor_storage_map(); + auto first = tensors.find("conv_first.weight"); + auto last = tensors.find("conv_last.weight"); + if (first == tensors.end() || last == tensors.end() || + tensors.count("body.0.rdb1.conv1.weight") == 0 || + first->second.n_dims != 4 || last->second.n_dims != 4 || + first->second.ne[2] != 3 || last->second.ne[3] != 3) { + return 0; + } + return ESRGANConfig::detect_from_weights(tensors).scale; + } catch (const std::exception&) { + return 0; + } +} + void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) { if (upscaler_ctx->upscaler != nullptr) { delete upscaler_ctx->upscaler;