src/stable-diffusion.cpp

@@ -107,7 +107,6 @@ public:
 
     SDVersion version;
     bool vae_decode_only         = false;
-    bool external_vae_is_invalid = false;
     bool free_params_immediately = false;
 
     std::shared_ptr<RNG> rng         = std::make_shared<PhiloxRNG>();
@@ -322,7 +321,6 @@ public:
             LOG_INFO("loading vae from '%s'", sd_ctx_params->vae_path);
             if (!model_loader.init_from_file(sd_ctx_params->vae_path, "vae.")) {
                 LOG_WARN("loading vae from '%s' failed", sd_ctx_params->vae_path);
-                external_vae_is_invalid = true;
             }
         }
 
@@ -621,10 +619,10 @@ public:
                         first_stage_model->set_conv2d_direct_enabled(true);
                     }
                     if (sd_version_is_sdxl(version) &&
-                        (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || external_vae_is_invalid)) {
+                        (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale)) {
                         float vae_conv_2d_scale = 1.f / 32.f;
                         LOG_WARN(
-                            "No valid VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
+                            "No VAE specified with --vae or --force-sdxl-vae-conv-scale flag set, "
                             "using Conv2D scale %.3f",
                             vae_conv_2d_scale);
                         first_stage_model->set_conv2d_scale(vae_conv_2d_scale);

src/tokenize_util.cpp

@@ -919,21 +919,15 @@ std::vector<std::string> token_split(const std::string& text) {
 
         // `\s*[\r\n]+|\s+(?!\S)|\s+`
         if (is_space(cp)) {
-            std::string token;
+            std::string token = codepoint_to_utf8(cp);
-            bool saw_new_line = false;
+            ++i;
 
             while (i < cps.size() && is_space(cps[i])) {
                 token += codepoint_to_utf8(cps[i]);
-
-                if (cps[i] == U'\r' || cps[i] == U'\n') {
-                    saw_new_line = true;
-                } else {
-                    if (saw_new_line) {
-                        break;
-                    }
-                }
-
                 ++i;
+                if (cps[i] == U'\r' || cps[i] == U'\n') {
+                    break;
+                }
             }
 
             tokens.push_back(token);