fix: avoid repeated T5 EOS tokens in Anima prompt weights (#1501)

This commit is contained in:
leejet 2026-05-16 21:22:46 +08:00 committed by GitHub
parent 36330724bd
commit d7ecbe1d01
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -1625,10 +1625,11 @@ struct AnimaConditioner : public Conditioner {
for (const auto& item : parsed_attention) { for (const auto& item : parsed_attention) {
const std::string& curr_text = item.first; const std::string& curr_text = item.first;
float curr_weight = item.second; float curr_weight = item.second;
std::vector<int> curr_tokens = t5_tokenizer.tokenize(curr_text, nullptr, true); std::vector<int> curr_tokens = t5_tokenizer.encode(curr_text);
t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end()); t5_tokens.insert(t5_tokens.end(), curr_tokens.begin(), curr_tokens.end());
t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight); t5_weights.insert(t5_weights.end(), curr_tokens.size(), curr_weight);
} }
t5_tokenizer.pad_tokens(t5_tokens, &t5_weights, nullptr);
return {qwen_tokens, qwen_weights, t5_tokens, t5_weights}; return {qwen_tokens, qwen_weights, t5_tokens, t5_weights};
} }