DarthAffe/stable-diffusion.cpp
1
0 Fork 0
mirror of https://github.com/leejet/stable-diffusion.cpp.git synced 2026-11-11 11:26:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add verbose logging and log-level selection (#1941)

Browse Source
This commit is contained in:
leejet authored and GitHub committed 2026-09-06 23:14:03 +08:00
1 parent 462d675018
commit dbb611264e
69 files changed
+551 -484
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/tokenizers/bpe_tokenizer.cpp
+1 -1
View File
@@ -205,7 +205,7 @@ std::vector<int> BPETokenizer::encode(const std::string& text, on_new_token_cb_t
ss << "\"" << token << "\", ";
}
ss << "]";
LOG_DEBUG("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
return bpe_tokens;
}
src/tokenizers/clip_tokenizer.cpp
+1 -1
View File
@@ -63,7 +63,7 @@ void CLIPTokenizer::load_from_merges(const std::string& merges_utf8_str) {
}
vocab.push_back(utf8_to_utf32("<|startoftext|>"));
vocab.push_back(utf8_to_utf32("<|endoftext|>"));
LOG_DEBUG("vocab size: %zu", vocab.size());
LOG_VERBOSE("vocab size: %zu", vocab.size());
int i = 0;
for (const auto& token : vocab) {
encoder[token] = i;
src/tokenizers/gemma_tokenizer.cpp
+4 -4
View File
@@ -29,7 +29,7 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
decoder[i] = token;
}
encoder_len = static_cast<int>(vocab.size());
LOG_DEBUG("vocab size: %d", encoder_len);
LOG_VERBOSE("vocab size: %d", encoder_len);
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
@@ -37,7 +37,7 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
size_t space_pos = merge.find(' ');
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
}
LOG_DEBUG("merges size %zu", merge_pairs.size());
LOG_VERBOSE("merges size %zu", merge_pairs.size());
int rank = 0;
for (const auto& merge : merge_pairs) {
@@ -214,7 +214,7 @@ void Gemma2Tokenizer::load_from_merges(const std::string& merges_utf8_str, const
decoder[i] = token;
}
encoder_len = static_cast<int>(vocab.size());
LOG_DEBUG("vocab size: %d", encoder_len);
LOG_VERBOSE("vocab size: %d", encoder_len);
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
@@ -222,7 +222,7 @@ void Gemma2Tokenizer::load_from_merges(const std::string& merges_utf8_str, const
size_t space_pos = merge.find(' ');
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
}
LOG_DEBUG("merges size %zu", merge_pairs.size());
LOG_VERBOSE("merges size %zu", merge_pairs.size());
int rank = 0;
for (const auto& merge : merge_pairs) {
src/tokenizers/gpt_oss_tokenizer.cpp
+2 -2
View File
@@ -31,7 +31,7 @@ void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const
encoder_len++;
}
encoder_len = static_cast<int>(encoder.size());
LOG_DEBUG("vocab size: %d", encoder_len);
LOG_VERBOSE("vocab size: %d", encoder_len);
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
@@ -39,7 +39,7 @@ void GPTOSSTokenizer::load_from_merges(const std::string& merges_utf8_str, const
size_t space_pos = merge.find(' ');
merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
}
LOG_DEBUG("merges size %zu", merge_pairs.size());
LOG_VERBOSE("merges size %zu", merge_pairs.size());
int rank = 0;
for (const auto& merge : merge_pairs) {
src/tokenizers/mistral_tokenizer.cpp
+2 -2
View File
@@ -20,7 +20,7 @@ void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, cons
decoder[i] = token;
}
encoder_len = static_cast<int>(vocab.size());
LOG_DEBUG("vocab size: %d", encoder_len);
LOG_VERBOSE("vocab size: %d", encoder_len);
auto byte_unicode_pairs = bytes_to_unicode();
byte_encoder = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
@@ -28,7 +28,7 @@ void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, cons
byte_decoder[pair.second] = pair.first;
}
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
LOG_DEBUG("merges size %zu", merges.size());
LOG_VERBOSE("merges size %zu", merges.size());
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
for (const auto& merge : merges) {
size_t space_pos = merge.find(' ');
src/tokenizers/qwen2_tokenizer.cpp
+2 -2
View File
@@ -11,7 +11,7 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
}
std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
LOG_DEBUG("merges size %zu", merges.size());
LOG_VERBOSE("merges size %zu", merges.size());
std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
for (const auto& merge : merges) {
size_t space_pos = merge.find(' ');
@@ -36,7 +36,7 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
i++;
}
encoder_len = i;
LOG_DEBUG("vocab size: %d", encoder_len);
LOG_VERBOSE("vocab size: %d", encoder_len);
int rank = 0;
for (const auto& merge : merge_pairs) {
src/tokenizers/t5_unigram_tokenizer.cpp
+1 -1
View File
@@ -333,7 +333,7 @@ std::vector<int> T5UniGramTokenizer::encode(const std::string& input, on_new_tok
ss << "\"" << token_str << "\", ";
}
ss << "]";
LOG_DEBUG("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
return tokens;
}