fix: open model files with UTF-8 paths on Windows (#2144)

src/model_io/gguf_io.cpp

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstdio>
+#include <filesystem>
 #include <fstream>
 #include <ostream>
 #include <string>
@@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 bool is_gguf_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -178,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path,
     }
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    FILE* file = fopen(file_path.c_str(), "wb+");
+    FILE* file = ggml_fopen(file_path.c_str(), "wb+");
     if (file == nullptr) {
         set_error(error, "failed to open output file '" + file_path + "'");
         close();

src/model_io/gguf_reader_ext.h

@@ -2,6 +2,7 @@
 #define __SD_MODEL_IO_GGUF_READER_EXT_H__
 
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <vector>
@@ -184,7 +185,7 @@ private:
 
 public:
     bool load(const std::string& file_path) {
-        std::ifstream fin(file_path, std::ios::binary | std::ios::ate);
+        std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate);
         if (!fin) {
             LOG_ERROR("failed to open '%s'", file_path.c_str());
             return false;

src/model_io/safetensors_io.cpp

@@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) {
-    fs::path shard_fs_path(shard_path);
+    fs::path shard_fs_path = fs::u8path(shard_path);
     if (shard_fs_path.is_absolute()) {
-        return shard_fs_path.lexically_normal().string();
+        return shard_fs_path.lexically_normal().u8string();
     }
-    return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string();
+    return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string();
 }
 
 bool is_safetensors_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path,
                            std::string* error,
                            std::map<std::string, std::string>* metadata,
                            std::set<std::string>* tensor_names) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path,
                                  std::string* error) {
     shard_paths.clear();
 
-    std::ifstream file(file_path);
+    std::ifstream file(fs::u8path(file_path));
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path,
 
     const std::string header_str = header.dump();
 
-    std::ofstream file(file_path, std::ios::binary);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;
@@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path,
     data_start_                  = ST_HEADER_SIZE_LEN + header_str.size();
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    std::ofstream file(file_path, std::ios::binary | std::ios::trunc);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;

src/model_io/torch_legacy_io.cpp

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <unordered_map>
@@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const
 bool read_torch_legacy_file(const std::string& file_path,
                             std::vector<TensorStorage>& tensor_storages,
                             std::string* error) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;