feat(server): accept ref_videos, ref_audios, ref_video_audios in sdcpp vid_gen (#2194)

examples/server/api.md

@@ -183,6 +183,9 @@ Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`,
 | `clip_skip` | CLIP Skip |
 | `img2img`, `mask_image` | Image initialization/strength and masks |
 | `ref_images` | Reference image inputs |
+| `ref_videos` | Reference videos as frame arrays |
+| `ref_audios` | Standalone reference audio; runtime requires an Audio VAE or Audio Encoder |
+| `ref_video_audios` | Soundtracks paired with reference videos; runtime requires an Audio VAE |
 | `controlnet` | Control image and ControlNet strength |
 | `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) |
 | `ip_adapter` | IP-Adapter image and strength |

examples/server/model_catalog.cpp

@@ -291,6 +291,7 @@ static void apply_recipe_context(const json& settings, SDContextParams& params)
     }
 }
 
+// Known recipe feature names with their defaults; recipes may flip any of these.
 static const std::map<std::string, bool> k_recipe_features = {
     {"img2img", true},
     {"mask_image", true},
@@ -312,6 +313,9 @@ static const std::map<std::string, bool> k_recipe_features = {
     {"init_image", false},
     {"end_image", false},
     {"control_frames", false},
+    {"ref_videos", false},
+    {"ref_audios", false},
+    {"ref_video_audios", false},
     {"vace", false},
     {"high_noise", false},
     {"cache_unet", false},
@@ -369,6 +373,9 @@ json model_generation_features(const ServerRuntime& runtime) {
     features["controlnet"]  = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
     features["high_noise"]  = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
     features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+    features["ref_audios"]  = features.value("ref_audios", false) &&
+                             (!runtime.ctx_params->audio_vae_path.empty() || !runtime.ctx_params->audio_encoder_path.empty());
+    features["ref_video_audios"] = features.value("ref_video_audios", false) && !runtime.ctx_params->audio_vae_path.empty();
     if (!runtime.ctx_params->pulid_weights_path.empty())
         features["slg"] = false;
     return features;

examples/server/recipes/minimax-h3-ref2va.json

@@ -45,7 +45,10 @@
   },
   "features": [
     "flow_shift",
-    "ref_images"
+    "ref_images",
+    "ref_videos",
+    "ref_audios",
+    "ref_video_audios"
   ],
   "disabled_features": [
     "img2img",

examples/server/recipes/wan22-s2v.json

@@ -44,7 +44,8 @@
   },
   "features": [
     "flow_shift",
-    "init_image"
+    "init_image",
+    "ref_audios"
   ],
   "disabled_features": [
     "img2img",

examples/server/routes_sdcpp.cpp

@@ -212,11 +212,15 @@ static json make_img_gen_features_json() {
     };
 }
 
+/** Builds the default feature set advertised for vid_gen requests. */
 static json make_vid_gen_features_json() {
     return {
         {"init_image", true},
         {"end_image", true},
         {"control_frames", true},
+        {"ref_videos", true},
+        {"ref_audios", true},
+        {"ref_video_audios", true},
         {"high_noise_sample_params", true},
         {"lora", true},
         {"vae_tiling", true},
@@ -235,6 +239,7 @@ static json make_preview_modes_json() {
     return modes;
 }
 
+/** Builds the /capabilities payload: model, defaults, limits and feature maps. */
 static json make_capabilities_json(ServerRuntime& runtime) {
     refresh_lora_cache(runtime);
     refresh_upscaler_cache(runtime);
@@ -351,8 +356,10 @@ static json make_capabilities_json(ServerRuntime& runtime) {
     const json generation_features = model_generation_features(runtime);
     auto recipe_features           = [&](json features, bool video) {
         features["init_image"] = generation_features.at(video ? "init_image" : "img2img");
-        for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) {
-            if (features.contains(key) || std::string(key) == "ref_images")
+        for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "ref_videos", "ref_audios", "ref_video_audios", "lora", "vae_tiling", "cache", "preview"}) {
+            if (features.contains(key) || std::string(key) == "ref_images" ||
+                (video && (std::string(key) == "ref_videos" || std::string(key) == "ref_audios" ||
+                           std::string(key) == "ref_video_audios")))
                 features[key] = generation_features.at(key);
         }
         if (video)