From c8fcaad39a52f7777d9be6983163264a1b23d8e1 Mon Sep 17 00:00:00 2001 From: Javinator9889 Date: Thu, 17 Sep 2026 17:16:26 +0200 Subject: [PATCH] fix: let an unchanged conversation hit the prompt cache can_use_cache required at least two new messages (messages.size() - 2), but message_checksums_ is updated to the full incoming list after every request. A resend of an unchanged conversation therefore fails the bound and misses, re-prefilling everything -- which is slower than the request that timed out, so the client's retry times out too and the loop sustains itself. Bound by the full incoming length instead. Also guard the case this exposes: with nothing left to prefill, _chunked_insert computes zero chunks and hands an empty logits buffer to sampler->sample(). Refs #733 --- src/common/AutoModel/automodel.cpp | 7 +++++++ src/include/prompt_cache.hpp | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 78a4a0f88..55324a213 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -198,6 +198,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to clear_context(); skip_count = 0; } + // A fully cached prompt leaves nothing to prefill, and _chunked_insert then + // computes zero chunks and returns a default-constructed (empty) logits + // buffer straight into sampler->sample(). Re-prefill instead. + if (skip_count == tokens.size()) { + clear_context(); + skip_count = 0; + } tokens.erase(tokens.begin(), tokens.begin() + skip_count); if (this->total_tokens + tokens.size() >= this->MAX_L){ diff --git a/src/include/prompt_cache.hpp b/src/include/prompt_cache.hpp index 57342e84a..536dc50c3 100644 --- a/src/include/prompt_cache.hpp +++ b/src/include/prompt_cache.hpp @@ -173,9 +173,11 @@ class PromptCache { // (in order) at the start of the new conversation, allowing rounds // produced by other backends (cloud) to be appended without // invalidating the locally-built KV cache prefix. - const size_t prefix_len = messages.size() - 2; + // Bound by the full incoming length, not length-2. Requiring two *new* + // messages made a resend of an unchanged conversation miss every time, + // which is exactly what a client sends after a timeout. const bool can_use_message = - message_checksums_.size() <= prefix_len && + message_checksums_.size() <= messages.size() && matched == message_checksums_.size(); const bool can_use_tools = tool_checksums_ == new_tool_checksums; info.tools_matched = can_use_tools;