fix: correct tensor deduplication logic (#844)

This commit is contained in:
rmatif 2025-09-24 17:22:40 +02:00 committed by GitHub
parent fd693ac6a2
commit 1e0d2821bb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -1,4 +1,5 @@
#include <stdarg.h> #include <stdarg.h>
#include <algorithm>
#include <atomic> #include <atomic>
#include <chrono> #include <chrono>
#include <fstream> #include <fstream>
@ -2006,13 +2007,25 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread
w.join(); w.join();
} }
std::unordered_map<std::string, IndexedStorage> latest_map; std::vector<IndexedStorage> deduplicated;
deduplicated.reserve(all_results.size());
std::unordered_map<std::string, size_t> name_to_pos;
for (auto& entry : all_results) { for (auto& entry : all_results) {
latest_map[entry.ts.name] = entry; auto it = name_to_pos.find(entry.ts.name);
if (it == name_to_pos.end()) {
name_to_pos.emplace(entry.ts.name, deduplicated.size());
deduplicated.push_back(entry);
} else if (deduplicated[it->second].index < entry.index) {
deduplicated[it->second] = entry;
}
} }
processed_tensor_storages.reserve(latest_map.size()); std::sort(deduplicated.begin(), deduplicated.end(), [](const IndexedStorage& a, const IndexedStorage& b) {
for (auto& [name, entry] : latest_map) { return a.index < b.index;
});
processed_tensor_storages.reserve(deduplicated.size());
for (auto& entry : deduplicated) {
processed_tensor_storages.push_back(entry.ts); processed_tensor_storages.push_back(entry.ts);
} }
} }