fix: open model files with UTF-8 paths on Windows (#2144)

src/convert.cpp

@@ -3,6 +3,7 @@
 #include <cstdint>
 #include <cstring>
 #include <exception>
+#include <filesystem>
 #include <fstream>
 #include <map>
 #include <memory>
@@ -108,7 +109,7 @@ static bool preallocate_output_file(const std::string& output_path, uint64_t fil
         return true;
     }
 
-    std::fstream file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+    std::fstream file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
     if (!file.is_open()) {
         if (error != nullptr) {
             *error = "failed to open output file '" + output_path + "' for preallocation";
@@ -251,7 +252,7 @@ static bool stream_tensor_data(ModelLoader& model_loader,
 
     for (int worker = 0; worker < n_threads; worker++) {
         workers.emplace_back([&]() {
-            std::fstream output_file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+            std::fstream output_file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
             if (!output_file.is_open()) {
                 fail("failed to open output file '" + output_path + "' for tensor writing");
                 return;

src/core/ggml_tensor_utils.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 
 #include "core/ggml_extend_backend.h"
@@ -100,7 +101,7 @@ void ggml_ext_tensor_diff(
 }
 
 ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         LOG_ERROR("failed to open '%s'", file_path.c_str());
         return nullptr;

src/core/tensor_ggml.hpp

@@ -3,6 +3,7 @@
 
 #include <array>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <stdexcept>
 #include <string>
@@ -105,7 +106,7 @@ namespace sd {
 
     template <typename T>
     inline Tensor<T> load_tensor_from_file_as_tensor(const std::string& file_path) {
-        std::ifstream file(file_path, std::ios::binary);
+        std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
         if (!file.is_open()) {
             throw std::runtime_error("failed to open tensor file: " + file_path);
         }

src/core/util.cpp

@@ -7,6 +7,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <exception>
+#include <filesystem>
 #include <fstream>
 #include <locale>
 #include <regex>
@@ -123,12 +124,12 @@ int round_up_to(int value, int base) {
 #include <windows.h>
 
 bool file_exists(const std::string& filename) {
-    DWORD attributes = GetFileAttributesA(filename.c_str());
+    DWORD attributes = GetFileAttributesW(std::filesystem::u8path(filename).c_str());
     return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY));
 }
 
 bool is_directory(const std::string& path) {
-    DWORD attributes = GetFileAttributesA(path.c_str());
+    DWORD attributes = GetFileAttributesW(std::filesystem::u8path(path).c_str());
     return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY));
 }
 
@@ -152,8 +153,8 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bo
     void* mapped_data = nullptr;
     size_t file_size  = 0;
 
-    HANDLE file_handle = CreateFileA(
-        filename.c_str(),
+    HANDLE file_handle = CreateFileW(
+        std::filesystem::u8path(filename).c_str(),
         GENERIC_READ,
         FILE_SHARE_READ,
         nullptr,

src/model_io/gguf_io.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstdio>
+#include <filesystem>
 #include <fstream>
 #include <ostream>
 #include <string>
@@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 bool is_gguf_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -178,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path,
     }
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    FILE* file = fopen(file_path.c_str(), "wb+");
+    FILE* file = ggml_fopen(file_path.c_str(), "wb+");
     if (file == nullptr) {
         set_error(error, "failed to open output file '" + file_path + "'");
         close();

src/model_io/gguf_reader_ext.h

@@ -2,6 +2,7 @@
 #define __SD_MODEL_IO_GGUF_READER_EXT_H__
 
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <vector>
@@ -184,7 +185,7 @@ private:
 
 public:
     bool load(const std::string& file_path) {
-        std::ifstream fin(file_path, std::ios::binary | std::ios::ate);
+        std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate);
         if (!fin) {
             LOG_ERROR("failed to open '%s'", file_path.c_str());
             return false;

src/model_io/safetensors_io.cpp

@@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) {
-    fs::path shard_fs_path(shard_path);
+    fs::path shard_fs_path = fs::u8path(shard_path);
     if (shard_fs_path.is_absolute()) {
-        return shard_fs_path.lexically_normal().string();
+        return shard_fs_path.lexically_normal().u8string();
     }
-    return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string();
+    return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string();
 }
 
 bool is_safetensors_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path,
                            std::string* error,
                            std::map<std::string, std::string>* metadata,
                            std::set<std::string>* tensor_names) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path,
                                  std::string* error) {
     shard_paths.clear();
 
-    std::ifstream file(file_path);
+    std::ifstream file(fs::u8path(file_path));
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path,
 
     const std::string header_str = header.dump();
 
-    std::ofstream file(file_path, std::ios::binary);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;
@@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path,
     data_start_                  = ST_HEADER_SIZE_LEN + header_str.size();
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    std::ofstream file(file_path, std::ios::binary | std::ios::trunc);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;

src/model_io/torch_legacy_io.cpp

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <unordered_map>
@@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const
 bool read_torch_legacy_file(const std::string& file_path,
                             std::vector<TensorStorage>& tensor_storages,
                             std::string* error) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;

src/model_loader.cpp

@@ -4,6 +4,7 @@
 #include <cinttypes>
 #include <cstdarg>
 #include <cstdlib>
+#include <filesystem>
 #include <fstream>
 #include <functional>
 #include <mutex>
@@ -333,7 +334,7 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
         }
     }
 
-    std::ifstream index_file(file_path);
+    std::ifstream index_file(std::filesystem::u8path(file_path));
     const auto index = nlohmann::json::parse(index_file);
     for (const auto& entry : index.at("weight_map").items()) {
         const auto expected = (std::filesystem::u8path(file_path).parent_path() /
@@ -1150,7 +1151,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                         return;
                     }
                 } else if (!mmapped) {
-                    file.open(file_path, std::ios::binary);
+                    file.open(std::filesystem::u8path(file_path), std::ios::binary);
                     if (!file.is_open()) {
                         LOG_ERROR("failed to open '%s'", file_path.c_str());
                         failed = true;

src/runtime/imatrix.cpp

@@ -10,6 +10,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 
 static IMatrixCollector imatrix_collector;
 
@@ -227,7 +228,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
         LOG_WARN("storing only %zu out of %zu entries\n", to_store.size(), stats_.size());
     }
 
-    std::ofstream out(fname, std::ios::binary);
+    std::ofstream out(std::filesystem::u8path(fname), std::ios::binary);
     out.write((const char*)&n_entries, sizeof(n_entries));
     for (const auto& name : to_store) {
         const auto& stat = stats_.at(name);
@@ -251,7 +252,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
 }
 
 bool IMatrixCollector::load_imatrix(const char* fname) {
-    std::ifstream in(fname, std::ios::binary);
+    std::ifstream in(std::filesystem::u8path(fname), std::ios::binary);
     if (!in) {
         LOG_ERROR("failed to open %s\n", fname);
         return false;

src/tokenizers/hf_tokenizer.cpp

@@ -4,6 +4,7 @@
 #include <array>
 #include <climits>
 #include <cstdlib>
+#include <filesystem>
 #include <fstream>
 #include <queue>
 #include <sstream>
@@ -495,7 +496,7 @@ struct HFTokenizer::Impl {
 
 HFTokenizer::HFTokenizer(const std::string& path)
     : impl_(new Impl) {
-    std::ifstream stream(path, std::ios::binary);
+    std::ifstream stream(std::filesystem::u8path(path), std::ios::binary);
     tokenizer_require(stream.good(), "cannot open " + path);
     TokenizerJSON config;
     stream >> config;