fix: safely handle whitespace and consecutive newlines (#1288)

src/stable-diffusion.cpp

@@ -107,6 +107,7 @@ public:
 
     SDVersion version;
     bool vae_decode_only         = false;
+    bool external_vae_is_invalid = false;
     bool free_params_immediately = false;
 
     std::shared_ptr<RNG> rng         = std::make_shared<PhiloxRNG>();
@@ -321,6 +322,7 @@ public:
             LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
             if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) {
                 LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
+                external_vae_is_invalid = true;
             }
         }
 
@@ -619,10 +621,10 @@ public:
                         first_stage_model->set_conv2d_direct_enabled(true);
                     }
                     if (sd_version_is_sdxl(version) &&
-                        (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale)) {
+                        (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || external_vae_is_invalid)) {
                         float vae_conv_2d_scale = 1.f / 32.f;
                         LOG_WARN(
-                            "No VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
+                            "No valid VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
                             "using Conv2D scale %.3f",
                             vae_conv_2d_scale);
                         first_stage_model->set_conv2d_scale(vae_conv_2d_scale);

src/tokenize_util.cpp

@@ -919,17 +919,23 @@ std::vector<std::string> token_split(const std::string& text) {
 
         // `\s*[\r\n]+|\s+(?!\S)|\s+`
         if (is_space(cp)) {
-            std::string token = codepoint_to_utf8(cp);
+            std::string token;
-            ++i;
+            bool saw_new_line = false;
 
             while (i < cps.size() && is_space(cps[i])) {
                 token += codepoint_to_utf8(cps[i]);
-                ++i;
+
                 if (cps[i] == U'\r' || cps[i] == U'\n') {
+                    saw_new_line = true;
+                } else {
+                    if (saw_new_line) {
                         break;
                     }
                 }
 
+                ++i;
+            }
+
             tokens.push_back(token);
             continue;
         }