From 47e940a9a229c0b219cad6a26927321cea4b8a12 Mon Sep 17 00:00:00 2001 From: Javinator9889 Date: Thu, 17 Sep 2026 17:17:43 +0200 Subject: [PATCH] fix: send SSE headers before prefill and beat during it write_streaming_response() sends the HTTP headers on its first call, and in the streaming chat path that call cannot happen until the first token is generated -- insert() runs to completion first. On a long prompt that is minutes during which the client receives zero bytes, not even response headers, and cannot distinguish a working server from a dead one. It times out and retries, and the retry queues behind the request still running. Emit an SSE comment before prefill and every five seconds during it. Lines starting with ':' are comments that clients ignore, but they put the headers on the wire and keep the connection alive. is_cancelled() is already invoked once per prefill chunk, so it carries the heartbeat with no new plumbing. Measured on qwen3.5:9b with a 9038-token prompt: time to first byte drops from the full prefill (23s) to 2ms. Answer, usage chunk and [DONE] unchanged. Refs #733 --- src/server/rest_handler.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 7890a4b5..a719a212 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -1199,9 +1199,28 @@ void RestHandler::handle_openai_chat_completion(const json& request, }; streaming_ostream_openai_chat ostream(model, auto_chat_engine.get(), openai_stream_callback); // streaming in chat completion format + // write_streaming_response() only sends the HTTP headers on its first + // call, which cannot happen until the first token is generated. On a + // long prompt that leaves the client with zero bytes for minutes, so + // it cannot tell a working server from a dead one and times out. + // ':' lines are SSE comments: clients ignore them, but they put the + // headers on the wire and keep the connection demonstrably alive. + // is_cancelled() is already called once per prefill chunk, so it + // carries the heartbeat without new plumbing. + send_streaming_response(json(": flm prefill started\n\n"), false); + auto last_beat = std::chrono::steady_clock::now(); + auto prefill_tick = [&]() { + auto now = std::chrono::steady_clock::now(); + if (now - last_beat >= std::chrono::seconds(5)) { + last_beat = now; + send_streaming_response(json(": flm prefilling\n\n"), false); + } + return cancellation_token->cancelled(); + }; + header_print("FLM", "Start prefill..."); try { - bool success = auto_chat_engine->insert(meta_info, uniformed_input, [&] { return cancellation_token->cancelled(); }); + bool success = auto_chat_engine->insert(meta_info, uniformed_input, prefill_tick); if (!success) { if (meta_info.stop_reason == CANCEL_DETECTED || cancellation_token->cancelled()) { meta_info.stop_reason = CANCEL_DETECTED;