diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 78a4a0f8..55324a21 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -198,6 +198,13 @@ bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& to clear_context(); skip_count = 0; } + // A fully cached prompt leaves nothing to prefill, and _chunked_insert then + // computes zero chunks and returns a default-constructed (empty) logits + // buffer straight into sampler->sample(). Re-prefill instead. + if (skip_count == tokens.size()) { + clear_context(); + skip_count = 0; + } tokens.erase(tokens.begin(), tokens.begin() + skip_count); if (this->total_tokens + tokens.size() >= this->MAX_L){ diff --git a/src/include/prompt_cache.hpp b/src/include/prompt_cache.hpp index 57342e84..536dc50c 100644 --- a/src/include/prompt_cache.hpp +++ b/src/include/prompt_cache.hpp @@ -173,9 +173,11 @@ class PromptCache { // (in order) at the start of the new conversation, allowing rounds // produced by other backends (cloud) to be appended without // invalidating the locally-built KV cache prefix. - const size_t prefix_len = messages.size() - 2; + // Bound by the full incoming length, not length-2. Requiring two *new* + // messages made a resend of an unchanged conversation miss every time, + // which is exactly what a client sends after a timeout. const bool can_use_message = - message_checksums_.size() <= prefix_len && + message_checksums_.size() <= messages.size() && matched == message_checksums_.size(); const bool can_use_tools = tool_checksums_ == new_tool_checksums; info.tools_matched = can_use_tools;