feat(server): make async job queue limits configurable (#2316)

examples/server/README.md

@@ -287,6 +287,16 @@ If you want to use a different host or port, pass:
 --listen-ip <ip> --listen-port <port>
 ```
 
+The native async job queue can be tuned with:
+
+```bash
+--max-pending-jobs <count> --completed-ttl-seconds <seconds>
+```
+
+`--max-pending-jobs` defaults to 64 and limits how many async jobs may be
+queued or generating. `--completed-ttl-seconds` defaults to 600 and controls
+how long completed jobs remain available through the async API.
+
 # Frontend
 
 ## Build with Frontend

examples/server/main.cpp

@@ -113,6 +113,10 @@ int main(int argc, const char** argv) {
     std::vector<UpscalerEntry> upscaler_cache;
     std::mutex upscaler_mutex;
     AsyncJobManager async_job_manager;
+    async_job_manager.max_pending_jobs =
+        static_cast<size_t>(svr_params.max_pending_jobs);
+    async_job_manager.completed_ttl_seconds =
+        static_cast<int64_t>(svr_params.completed_ttl_seconds);
     ServerRuntime runtime = {
         models.context.get(),
         &sd_ctx_mutex,

examples/server/runtime.cpp

@@ -351,6 +351,12 @@ ArgOptions SDSvrParams::get_options() {
 
     options.int_options = {
         {"", "--listen-port", "server listen port (default: 1234)", &listen_port},
+        {"", "--max-pending-jobs",
+         "maximum number of pending async jobs (default: 64)",
+         &max_pending_jobs},
+        {"", "--completed-ttl-seconds",
+         "retention time for completed async jobs in seconds (default: 600)",
+         &completed_ttl_seconds},
     };
 
     options.bool_options = {
@@ -388,6 +394,16 @@ bool SDSvrParams::validate() {
         return false;
     }
 
+    if (max_pending_jobs <= 0) {
+        LOG_ERROR("error: max_pending_jobs must be greater than 0");
+        return false;
+    }
+
+    if (completed_ttl_seconds < 0) {
+        LOG_ERROR("error: completed_ttl_seconds must be non-negative");
+        return false;
+    }
+
     if (!serve_html_path.empty() && !fs::exists(fs::u8path(serve_html_path))) {
         LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
         return false;
@@ -408,6 +424,8 @@ std::string SDSvrParams::to_string() const {
         << "  log_level: " << log_level_name(log_level) << ",\n"
         << "  listen_ip: " << listen_ip << ",\n"
         << "  listen_port: \"" << listen_port << "\",\n"
+        << "  max_pending_jobs: " << max_pending_jobs << ",\n"
+        << "  completed_ttl_seconds: " << completed_ttl_seconds << ",\n"
         << "  serve_html_path: \"" << serve_html_path << "\",\n"
         << "}";
     return oss.str();

examples/server/runtime.h

@@ -22,6 +22,8 @@ struct ServerModels;
 struct SDSvrParams {
     std::string listen_ip     = "127.0.0.1";
     int listen_port           = 1234;
+    int max_pending_jobs      = 64;
+    int completed_ttl_seconds = 600;
     std::string serve_html_path;
     std::vector<std::string> model_dirs;
     bool normal_exit         = false;