fix: open model files with UTF-8 paths on Windows (#2144)

This commit is contained in:
therealshulgin authored and GitHub committed 2026-10-17 00:50:09 +08:00
1 parent 8cd731e7b4
commit 8c442fb0ad
11 files changed
+35 -25

No files matched your search

+2 -1
View File
@@ -4,6 +4,7 @@
#include <array>
#include <climits>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <queue>
#include <sstream>
@@ -495,7 +496,7 @@ struct HFTokenizer::Impl {
HFTokenizer::HFTokenizer(const std::string& path)
: impl_(new Impl) {
std::ifstream stream(path, std::ios::binary);
std::ifstream stream(std::filesystem::u8path(path), std::ios::binary);
tokenizer_require(stream.good(), "cannot open " + path);
TokenizerJSON config;
stream >> config;