diff --git a/docs/backend.md b/docs/backend.md index 846aafb8..58c2ca8b 100644 --- a/docs/backend.md +++ b/docs/backend.md @@ -156,8 +156,11 @@ the runner's graph-cut capacity checks. Runtime capacity checks also leave 512 MiB of currently free device memory for backend scratch buffers and pipelines, including with explicit backend assignments. -They cap stale free-memory reports by the device's total memory minus tracked -resident allocations and reject reports that exceed the device's total memory. +They cap free-memory reports by the device's total memory minus tracked +resident allocations. Vulkan reports exceeding total memory are rejected because +its heap-budget subtraction can underflow. Other backends use the cap instead of +treating such reports as zero free memory. Failed checks log the reported free and +total memory alongside tracked weight and runtime allocations. Components are considered in `diffusion`, `te`, `vae` order so that repeatedly used diffusion weights have priority. Each component's weights use the first diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index db5ad319..d9000a32 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -2219,7 +2219,10 @@ struct LLMEmbedder : public Conditioner { false, deepstack_image_embeds, image_grids); - GGML_ASSERT(!hidden_states.empty()); + if (hidden_states.empty()) { + LOG_ERROR("LLM prompt encoding failed"); + return {}; + } hidden_states = apply_token_weights(std::move(hidden_states), weights); GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx); diff --git a/src/model_manager.cpp b/src/model_manager.cpp index 9a9fe062..930c6f49 100644 --- a/src/model_manager.cpp +++ b/src/model_manager.cpp @@ -1613,7 +1613,8 @@ void ModelManager::remove_runtime_owner(uintptr_t owner_id) { ModelManager::CapacityCheck ModelManager::check_capacity( const DeviceMemoryRequest& request, - const std::vector& states) const { + const std::vector& states, + bool log_details) const { CapacityCheck result; if (request.compute_backend == nullptr || sd_backend_is_cpu(request.compute_backend)) { return result; @@ -1631,16 +1632,23 @@ ModelManager::CapacityCheck ModelManager::check_capacity( } size_t free_bytes = 0, total_bytes = 0; ggml_backend_dev_memory(device, &free_bytes, &total_bytes); + const size_t weights_resident = compute_backend_resident_bytes(backend); + const size_t other_runtime = other_runtime_resident_bytes(request.owner_id, backend); + const size_t resident = add(weights_resident, add(other_runtime, request.runtime_resident_bytes)); + if (log_details) { + LOG_WARN("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB", + ggml_backend_name(backend), + free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0), + weights_resident / (1024.0 * 1024.0), other_runtime / (1024.0 * 1024.0), + request.runtime_resident_bytes / (1024.0 * 1024.0)); + } if (free_bytes == 0 && total_bytes == 0) { return SIZE_MAX; } // Vulkan's heap budget subtraction can underflow when usage exceeds the budget. - if (total_bytes > 0 && free_bytes > total_bytes) { + if (total_bytes > 0 && free_bytes > total_bytes && sd_backend_is(backend, "Vulkan")) { return size_t{0}; } - const size_t resident = add(compute_backend_resident_bytes(backend), - add(other_runtime_resident_bytes(request.owner_id, backend), - request.runtime_resident_bytes)); if (total_bytes > 0) { free_bytes = std::min(free_bytes, resident < total_bytes ? total_bytes - resident : 0); } @@ -1786,7 +1794,7 @@ bool ModelManager::ensure_compute_backend_capacity( } } - const auto capacity = check_capacity(request, required_states); + const auto capacity = check_capacity(request, required_states, true); const std::string available_device = capacity.available_device_bytes == SIZE_MAX ? "unknown" : sd_format("%.2f MB", capacity.available_device_bytes / (1024.0 * 1024.0)); diff --git a/src/model_manager.h b/src/model_manager.h index 7f8df59c..f3a6e328 100644 --- a/src/model_manager.h +++ b/src/model_manager.h @@ -157,7 +157,8 @@ private: } }; CapacityCheck check_capacity(const DeviceMemoryRequest& request, - const std::vector& states) const; + const std::vector& states, + bool log_details = false) const; ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const; ggml_backend_buffer_type_t split_buffer_type_for(const TensorState& state) const;