diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 0b237c94a..612534d0f 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -554,6 +554,7 @@ add_library(dflash_common STATIC src/server/tool_hint.cpp src/server/reasoning.cpp src/server/tool_memory.cpp + src/server/response_error.cpp src/server/sse_emitter.cpp src/server/prefix_cache.cpp src/server/pin_friendly_prompt.cpp diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 221e54ab5..78baf9e16 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -207,6 +207,7 @@ struct GenerateRequest { enum class GenerateErrorCode { Incomplete, AdapterUnavailable, + ResourceExhausted, ContextOverflow, SamplingUnsupported, PrefillFailed, @@ -221,6 +222,7 @@ constexpr std::string_view generate_error_code(GenerateErrorCode error) { switch (error) { case GenerateErrorCode::Incomplete: return "incomplete"; case GenerateErrorCode::AdapterUnavailable: return "adapter_unavailable"; + case GenerateErrorCode::ResourceExhausted: return "resource_exhausted"; case GenerateErrorCode::ContextOverflow: return "context_overflow"; case GenerateErrorCode::SamplingUnsupported: return "sampling_unsupported"; case GenerateErrorCode::PrefillFailed: return "prefill_failed"; diff --git a/server/src/gemma4/gemma4_backend.cpp b/server/src/gemma4/gemma4_backend.cpp index d9cdd9ad6..e36a5c755 100644 --- a/server/src/gemma4/gemma4_backend.cpp +++ b/server/src/gemma4/gemma4_backend.cpp @@ -900,7 +900,7 @@ GenerateResult Gemma4Backend::restore_and_generate_impl(int slot, if (snap_pos > kvflash_tokens_ - kvflash_pager_.chunk_tokens()) { std::fprintf(stderr, "[kvflash] restored prefix (%d) exceeds pool %d\n", snap_pos, kvflash_tokens_); - result.fail(GenerateErrorCode::ContextOverflow); + result.fail(GenerateErrorCode::ResourceExhausted); return result; } kvflash_pager_.reset(); diff --git a/server/src/laguna/laguna_backend.cpp b/server/src/laguna/laguna_backend.cpp index cf42b013c..51eb9f2d0 100644 --- a/server/src/laguna/laguna_backend.cpp +++ b/server/src/laguna/laguna_backend.cpp @@ -1584,7 +1584,7 @@ GenerateResult LagunaBackend::restore_and_generate_impl(int slot, N > kvflash_tokens_ - kvflash_pager_.chunk_tokens()) { std::fprintf(stderr, "[kvflash] restore prompt (%d) exceeds pool %d; " "raise --kvflash\n", N, kvflash_tokens_); - result.fail(GenerateErrorCode::ContextOverflow); + result.fail(GenerateErrorCode::ResourceExhausted); return result; } if (kvflash_active()) { @@ -2743,7 +2743,7 @@ GenerateResult LagunaBackend::generate_hybrid(const GenerateRequest & req, N > kvflash_tokens_ - kvflash_pager_.chunk_tokens()) { std::fprintf(stderr, "[kvflash] hybrid prompt (%d) exceeds pool %d; " "raise --kvflash\n", N, kvflash_tokens_); - result.fail(GenerateErrorCode::ContextOverflow); + result.fail(GenerateErrorCode::ResourceExhausted); return result; } diff --git a/server/src/qwen35moe/qwen35moe_backend.cpp b/server/src/qwen35moe/qwen35moe_backend.cpp index 1f6ea7074..221615958 100644 --- a/server/src/qwen35moe/qwen35moe_backend.cpp +++ b/server/src/qwen35moe/qwen35moe_backend.cpp @@ -862,7 +862,7 @@ GenerateResult Qwen35MoeBackend::generate_impl(const GenerateRequest & req, std::fprintf(stderr, "[kvflash] hybrid prompt (%d) exceeds pool %d; raise --kvflash " "or enable pflash compression\n", prompt_len, kvflash_tokens_); - result.fail(GenerateErrorCode::ContextOverflow); + result.fail(GenerateErrorCode::ResourceExhausted); cleanup_graphs(); return result; } @@ -1526,7 +1526,7 @@ GenerateResult Qwen35MoeBackend::restore_and_generate_impl(int slot, std::fprintf(stderr, "[kvflash] hybrid restore prompt (%d) exceeds pool %d; raise " "--kvflash\n", prompt_len, kvflash_tokens_); - result.fail(GenerateErrorCode::ContextOverflow); + result.fail(GenerateErrorCode::ResourceExhausted); out_io.emit(-1); return result; } diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index a5a8032ae..8a760c825 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -18,6 +18,7 @@ #include "http_server.h" #include "admission.h" +#include "response_error.h" #include "sse_emitter.h" #include "prompt_normalize.h" #include "tool_hint.h" @@ -3566,7 +3567,7 @@ void HttpServer::finalize_generation_cache( } } - if (disk_cache_.disabled()) return; + if (disk_cache_.disabled() || !result.ok()) return; if (!prepared.compressed) { recent_disk_prompts_.insert( @@ -3905,15 +3906,6 @@ void HttpServer::send_nonstream_response( } } -std::array HttpServer::sse_error_close_chunks( - const std::string & message) { - const json err = {{"error", { - {"message", message}, - {"type", "server_error"}, - }}}; - return {"data: " + err.dump() + "\n\n", "data: [DONE]\n\n"}; -} - void HttpServer::worker_loop() { while (true) { ServerJob * job = dequeue(); @@ -3965,19 +3957,6 @@ void HttpServer::process_job(ServerJob * job) { job->done = true; job->cv.notify_one(); }; - auto fail_request = [&](int status, const std::string & message) { - std::fprintf(stderr, "[server] request failed: %s\n", message.c_str()); - if (req.stream) { - stop_job_stream(job); - for (const std::string & chunk : sse_error_close_chunks(message)) { - send_job_bytes(job, chunk.data(), chunk.size()); - } - } else { - send_error(fd, status, message); - } - finish_job(); - }; - std::fprintf(stderr, "[server] chat START %s format=%s stream=%s prompt_tokens=%zu " "max_tokens=%d tools=%zu\n", @@ -4021,6 +4000,31 @@ void HttpServer::process_job(ServerJob * job) { } if (req.stream) start_job_stream(job); + auto fail_request = [&](int status, const std::string & message) { + std::fprintf(stderr, "[server] request failed: %s\n", message.c_str()); + ResponseError error; + if (status == 400) { + error = ResponseError::invalid_request( + "invalid_request", message); + } else if (status == 503) { + error = ResponseError::unavailable("unavailable", message); + } else { + error = ResponseError::internal("server_error", message); + } + stop_job_stream(job); + if (req.stream) { + for (const std::string & chunk : emitter.emit_error(error)) { + send_job_bytes(job, chunk.data(), chunk.size()); + } + } else { + const json body = build_error_response( + req.format, error, req.response_id); + send_response(fd, response_error_http_status(error), + "application/json", body.dump() + "\n"); + } + finish_job(); + }; + PreparedPrompt prepared = prepare_prompt(req); if (prepared.error_status != 0) { fail_request(prepared.error_status, prepared.error); @@ -4096,7 +4100,7 @@ void HttpServer::process_job(ServerJob * job) { // Bandit: update when spec decode actually ran — including 0-accept case, // which signals the current keep_ratio is too low. - if (!req.session_id.empty() && result.spec_decode_ran) { + if (result.ok() && !req.session_id.empty() && result.spec_decode_ran) { float old_keep = sessions_.get_keep_ratio(req.session_id); int old_turn = sessions_.turn_count(req.session_id); sessions_.update(req.session_id, result.accept_rate); @@ -4135,29 +4139,91 @@ void HttpServer::process_job(ServerJob * job) { agent_turn_cache_hit, }; - // Record performance for /status page. - if (result.ok()) { - PerfRecord perf; - perf.prompt_tokens = (int)req.prompt_tokens.size(); - perf.completion_tokens = completion_tokens; - // Use actual prefilled token count: on cache hit the backend only - // prefills the delta beyond the cached prefix, so dividing the full - // prompt size by delta time would be wrong. - const int prefill_tokens = - (std::max)(0, effective_prompt_tokens - cached_prefix_tokens); - perf.prefill_tok_s = (result.prefill_s > 0.0) - ? (double)prefill_tokens / result.prefill_s : 0.0; - perf.decode_tok_s = (result.decode_s > 0.0) - ? (double)completion_tokens / result.decode_s : 0.0; - perf.accept_rate = result.accept_rate; - perf.cache_hit = cache_hit; - perf.pflash = pflash_compressed; - perf.spec_decode = result.spec_decode_ran; - perf.timestamp = std::chrono::steady_clock::now(); - status_.record_perf(perf); - status_.update_completion_tokens(completion_tokens); - broadcast_status(); + auto log_done = [&]() { + const auto done_at = std::chrono::steady_clock::now(); + const double elapsed_s = + std::chrono::duration(done_at - started_at).count(); + const int result_tokens = (int)result.tokens.size(); + const int out_tokens = (std::max)(completion_tokens, result_tokens); + const double tok_s = elapsed_s > 0.0 ? out_tokens / elapsed_s : 0.0; + const double decode_tok_s = + result.decode_s > 0.0 ? out_tokens / result.decode_s : 0.0; + const std::string finish = client_disconnected + ? "client_disconnect" + : (result.ok() ? emitter.finish_reason() : "error"); + + std::fprintf(stderr, + "[server] chat DONE %s ok=%s in=%zu effective_in=%zu out=%d " + "%.1fs %.1f tok/s finish=%s restore=%s slot=%d prefix_len=%d " + "prefill=%.1fs decode=%.1fs(%.1ftok/s) error=%s detail=%s\n", + req.response_id.c_str(), + result.ok() ? "true" : "false", + req.prompt_tokens.size(), + effective_prompt.size(), + out_tokens, + elapsed_s, + tok_s, + finish.c_str(), + using_restore ? "true" : "false", + cache_slot, + prefix_len, + result.prefill_s, + result.decode_s, + decode_tok_s, + result.ok() ? "-" : result.error_code().data(), + result.error_detail().empty() ? "-" : result.error_detail().data()); + }; + + // A backend failure terminates the request here. Everything below this + // branch records or frames a successful generation. + if (!result.ok()) { + stop_job_stream(job); + if (job->client_disconnected.load(std::memory_order_acquire)) { + client_disconnected = true; + } + if (!client_disconnected) { + const ResponseError error = to_response_error(*result.error); + if (req.stream) { + for (const std::string & chunk : emitter.emit_error(error)) { + if (!send_job_bytes(job, chunk.data(), chunk.size())) { + client_disconnected = true; + break; + } + } + } else { + const json body = build_error_response( + req.format, error, req.response_id); + sock_set_block(fd); + send_response(fd, response_error_http_status(error), + "application/json", body.dump() + "\n"); + } + } + log_done(); + finish_job(); + return; } + + // Record performance for /status page. + PerfRecord perf; + perf.prompt_tokens = (int)req.prompt_tokens.size(); + perf.completion_tokens = completion_tokens; + // Use actual prefilled token count: on cache hit the backend only + // prefills the delta beyond the cached prefix, so dividing the full + // prompt size by delta time would be wrong. + const int prefill_tokens = + (std::max)(0, effective_prompt_tokens - cached_prefix_tokens); + perf.prefill_tok_s = (result.prefill_s > 0.0) + ? (double)prefill_tokens / result.prefill_s : 0.0; + perf.decode_tok_s = (result.decode_s > 0.0) + ? (double)completion_tokens / result.decode_s : 0.0; + perf.accept_rate = result.accept_rate; + perf.cache_hit = cache_hit; + perf.pflash = pflash_compressed; + perf.spec_decode = result.spec_decode_ran; + perf.timestamp = std::chrono::steady_clock::now(); + status_.record_perf(perf); + status_.update_completion_tokens(completion_tokens); + broadcast_status(); // Serialize final frames after disabling heartbeat comments so no comment // can appear after the protocol's [DONE] marker. stop_job_stream(job); @@ -4205,38 +4271,7 @@ void HttpServer::process_job(ServerJob * job) { req.prompt_tokens.size(), completion_tokens); } - const auto done_at = std::chrono::steady_clock::now(); - const double elapsed_s = - std::chrono::duration(done_at - started_at).count(); - const int result_tokens = (int)result.tokens.size(); - const int out_tokens = (std::max)(completion_tokens, result_tokens); - const double tok_s = elapsed_s > 0.0 ? out_tokens / elapsed_s : 0.0; - const double decode_tok_s = - result.decode_s > 0.0 ? out_tokens / result.decode_s : 0.0; - const std::string finish = client_disconnected - ? "client_disconnect" - : (result.ok() ? emitter.finish_reason() : "error"); - - std::fprintf(stderr, - "[server] chat DONE %s ok=%s in=%zu effective_in=%zu out=%d " - "%.1fs %.1f tok/s finish=%s restore=%s slot=%d prefix_len=%d " - "prefill=%.1fs decode=%.1fs(%.1ftok/s) error=%s detail=%s\n", - req.response_id.c_str(), - result.ok() ? "true" : "false", - req.prompt_tokens.size(), - effective_prompt.size(), - out_tokens, - elapsed_s, - tok_s, - finish.c_str(), - using_restore ? "true" : "false", - cache_slot, - prefix_len, - result.prefill_s, - result.decode_s, - decode_tok_s, - result.ok() ? "-" : result.error_code().data(), - result.error_detail().empty() ? "-" : result.error_detail().data()); + log_done(); // Signal client thread that we're done. finish_job(); diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 52c36473b..611e967b2 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -487,9 +487,6 @@ class HttpServer { std::string format_http_response( int status, const std::string & content_type, const std::string & body); - static std::array sse_error_close_chunks( - const std::string & message); - // Parse HTTP request from socket. struct HttpRequest { std::string method; diff --git a/server/src/server/response_error.cpp b/server/src/server/response_error.cpp new file mode 100644 index 000000000..ca404c83c --- /dev/null +++ b/server/src/server/response_error.cpp @@ -0,0 +1,135 @@ +#include "response_error.h" + +#include "common/model_backend.h" + +#include + +namespace dflash::common { + +namespace { + +std::string message_or(std::string message, const char * fallback) { + return message.empty() ? fallback : std::move(message); +} + +const char * fallback_message(GenerateErrorCode code) { + switch (code) { + case GenerateErrorCode::Incomplete: + return "generation did not complete"; + case GenerateErrorCode::AdapterUnavailable: + return "requested adapter is unavailable"; + case GenerateErrorCode::ResourceExhausted: + return "request exceeds available generation capacity"; + case GenerateErrorCode::ContextOverflow: + return "request exceeds the model context"; + case GenerateErrorCode::SamplingUnsupported: + return "requested sampling settings are unsupported"; + case GenerateErrorCode::PrefillFailed: + return "generation prefill failed"; + case GenerateErrorCode::DecodeSeedMissing: + return "generation decode seed is missing"; + case GenerateErrorCode::DecodeFailed: + return "generation decode failed"; + case GenerateErrorCode::InvalidSnapshotSlot: + return "generation snapshot is unavailable"; + case GenerateErrorCode::ModelParked: + return "model is unavailable"; + case GenerateErrorCode::BackendSpecific: + return "generation failed"; + } + return "generation failed"; +} + +const char * openai_error_type(ResponseErrorKind kind) { + return kind == ResponseErrorKind::InvalidRequest + ? "invalid_request_error" : "server_error"; +} + +const char * anthropic_error_type(ResponseErrorKind kind) { + switch (kind) { + case ResponseErrorKind::InvalidRequest: return "invalid_request_error"; + case ResponseErrorKind::Unavailable: return "overloaded_error"; + case ResponseErrorKind::Internal: return "api_error"; + } + return "api_error"; +} + +} // namespace + +ResponseError ResponseError::invalid_request( + std::string code, std::string message) { + return {ResponseErrorKind::InvalidRequest, + std::move(code), + message_or(std::move(message), "invalid request")}; +} + +ResponseError ResponseError::unavailable( + std::string code, std::string message) { + return {ResponseErrorKind::Unavailable, + std::move(code), + message_or(std::move(message), "service unavailable")}; +} + +ResponseError ResponseError::internal( + std::string code, std::string message) { + return {ResponseErrorKind::Internal, + std::move(code), + message_or(std::move(message), "generation failed")}; +} + +ResponseError to_response_error(const GenerateError & error) { + const std::string code(generate_error_code(error.code)); + const std::string message = error.detail.empty() + ? fallback_message(error.code) : error.detail; + + switch (error.code) { + case GenerateErrorCode::ContextOverflow: + case GenerateErrorCode::SamplingUnsupported: + return ResponseError::invalid_request(code, message); + case GenerateErrorCode::AdapterUnavailable: + case GenerateErrorCode::ResourceExhausted: + case GenerateErrorCode::ModelParked: + return ResponseError::unavailable(code, message); + case GenerateErrorCode::Incomplete: + case GenerateErrorCode::PrefillFailed: + case GenerateErrorCode::DecodeSeedMissing: + case GenerateErrorCode::DecodeFailed: + case GenerateErrorCode::InvalidSnapshotSlot: + case GenerateErrorCode::BackendSpecific: + return ResponseError::internal(code, message); + } + return ResponseError::internal("unknown_error", "generation failed"); +} + +int response_error_http_status(const ResponseError & error) { + switch (error.kind) { + case ResponseErrorKind::InvalidRequest: return 400; + case ResponseErrorKind::Unavailable: return 503; + case ResponseErrorKind::Internal: return 500; + } + return 500; +} + +nlohmann::json build_error_response( + ApiFormat format, const ResponseError & error, + const std::string & request_id) { + if (format == ApiFormat::ANTHROPIC) { + nlohmann::json body = { + {"type", "error"}, + {"error", { + {"type", anthropic_error_type(error.kind)}, + {"message", error.message}, + }}, + }; + if (!request_id.empty()) body["request_id"] = request_id; + return body; + } + + return {{"error", { + {"message", error.message}, + {"type", openai_error_type(error.kind)}, + {"code", error.code}, + }}}; +} + +} // namespace dflash::common diff --git a/server/src/server/response_error.h b/server/src/server/response_error.h new file mode 100644 index 000000000..aa81ecbf9 --- /dev/null +++ b/server/src/server/response_error.h @@ -0,0 +1,48 @@ +// Server-side generation failure classification and API response encoding. +#pragma once + +#include "api_types.h" + +#include + +#include + +namespace dflash::common { + +struct GenerateError; + +enum class ResponseErrorKind { + InvalidRequest, + Unavailable, + Internal, +}; + +// API-neutral description of a failed serving request. Backends remain the +// source of truth for generation failures through GenerateError; this value +// records only the consequence at the server boundary. +struct ResponseError { + ResponseErrorKind kind = ResponseErrorKind::Internal; + std::string code; + std::string message; + + static ResponseError invalid_request( + std::string code, std::string message); + static ResponseError unavailable( + std::string code, std::string message); + static ResponseError internal( + std::string code, std::string message); +}; + +// Total backend-to-server mapping. Every backend error receives a stable code, +// a non-empty client message, and a server-owned availability classification. +ResponseError to_response_error(const GenerateError & error); + +int response_error_http_status(const ResponseError & error); + +// Build the protocol body for a non-streaming request. HTTP framing and socket +// ownership stay with HttpServer. +nlohmann::json build_error_response( + ApiFormat format, const ResponseError & error, + const std::string & request_id = {}); + +} // namespace dflash::common diff --git a/server/src/server/scheduler.cpp b/server/src/server/scheduler.cpp index 6c7e79d3c..96d20e777 100644 --- a/server/src/server/scheduler.cpp +++ b/server/src/server/scheduler.cpp @@ -4,17 +4,19 @@ // Split from http_server.cpp: this TU owns non-blocking admission (one // prefill chunk per engine step, fused with the live decode batch), FIFO // pool-full deferrals, per-slot streaming through ClientSendBuffer, and -// retirement. SSE emission, error-close chunks, and HTTP response +// retirement. SSE emission, terminal errors, and HTTP response // formatting are shared with the classic worker so both paths emit // matching wire formats. #include "http_server.h" #include "common/concurrency/seq_engine.h" +#include "response_error.h" #include #include #include #include +#include #include namespace dflash::common { @@ -37,8 +39,7 @@ struct SchedSlot { int n_gen_cap = 0; int completion_tokens = 0; bool client_disconnected = false; - bool failed = false; - std::string error; + std::optional error; bool finished = false; std::vector gen_tokens; // committed + pending, in order int32_t pending_tok = -1; // sampled, fed back next step @@ -236,7 +237,7 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { } }; - auto retire_slot = [&](int idx, bool backend_ok) { + auto retire_slot = [&](int idx) { SchedSlot & s = slots[(size_t)idx]; if (!s.job) return; const ParsedRequest & req = s.job->req; @@ -254,7 +255,7 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { /*effective_prompt_tokens=*/prompt_tokens, }; - if (backend_ok && !s.failed) { + if (!s.error) { PerfRecord perf; perf.prompt_tokens = (int)req.prompt_tokens.size(); perf.completion_tokens = s.completion_tokens; @@ -265,20 +266,19 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { status_.record_perf(perf); } - if (s.failed || !backend_ok) { - const std::string message = - s.error.empty() ? "generation failed" : s.error; + if (s.error) { if (!s.client_disconnected) { if (req.stream) { for (const std::string & chunk : - sse_error_close_chunks(message)) { + s.emitter->emit_error(*s.error)) { s.send_buffer.append(chunk); } } else { - json err = {{"error", {{"message", message}, - {"type", "invalid_request_error"}}}}; + const json body = build_error_response( + req.format, *s.error, req.response_id); s.send_buffer.append(format_http_response( - 500, "application/json", err.dump() + "\n")); + response_error_http_status(*s.error), + "application/json", body.dump() + "\n")); } } } else if (req.stream && !s.client_disconnected) { @@ -305,7 +305,7 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { "[server] chat DONE %s ok=%s in=%zu out=%d %.1fs %.1f tok/s " "finish=%s slot=%d prefill=%.1fs decode=%.1fs(%.1ftok/s) parallel\n", req.response_id.c_str(), - (!s.failed && backend_ok) ? "true" : "false", + s.error ? "false" : "true", req.prompt_tokens.size(), out_tokens, elapsed_s, elapsed_s > 0.0 ? out_tokens / elapsed_s : 0.0, s.client_disconnected ? "client_disconnect" @@ -455,16 +455,21 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { if (ar.status != SeqEngine::AdmitResult::Status::admitted) { std::fprintf(stderr, "[server] admit failed: %s\n", ar.error.c_str()); + const ResponseError error = ResponseError::internal( + "admission_failed", "admission failed: " + ar.error); if (req.stream && job->sse_started) { stop_job_stream(job); // Headers are already on the wire: report in-stream, like // the classic worker's fail_request after SSE start. - for (const std::string & chunk : sse_error_close_chunks( - "admission failed: " + ar.error)) { + for (const std::string & chunk : + job->emitter->emit_error(error)) { send_job_bytes(job, chunk.data(), chunk.size()); } } else { - send_error(job->fd, 500, "admission failed: " + ar.error); + const json body = build_error_response( + req.format, error, req.response_id); + send_response(job->fd, response_error_http_status(error), + "application/json", body.dump() + "\n"); } finish_job(job); return AdmissionDisposition::Retired; @@ -609,7 +614,7 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { std::memory_order_acquire)) { s.client_disconnected = true; s.finished = true; - retire_slot(i, true); + retire_slot(i); } } if (live_slots == 0) continue; @@ -654,9 +659,9 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { "failing all live requests\n", error.c_str()); for (int i = 0; i < n_slots; i++) { if (slots[(size_t)i].job) { - slots[(size_t)i].failed = true; - slots[(size_t)i].error = error; - retire_slot(i, false); + slots[(size_t)i].error = ResponseError::internal( + "engine_step_failed", error); + retire_slot(i); } } continue; @@ -666,8 +671,10 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { SchedSlot & s = slots[(size_t)out.slot]; if (!s.job) continue; if (out.failed) { - s.failed = true; - s.error = out.error; + s.error = ResponseError::internal( + "decode_failed", + out.error.empty() + ? "generation decode failed" : out.error); s.finished = true; continue; } @@ -683,8 +690,10 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { SchedSlot & s = slots[(size_t)out.slot]; if (!s.job) continue; if (out.status == PrefillStatus::failed) { - s.failed = true; - s.error = out.error; + s.error = ResponseError::internal( + "prefill_failed", + out.error.empty() + ? "generation prefill failed" : out.error); s.finished = true; continue; } @@ -733,7 +742,7 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { service_drains(); for (int i = 0; i < n_slots; i++) { if (slots[(size_t)i].job && slots[(size_t)i].finished) { - retire_slot(i, true); + retire_slot(i); } } } @@ -741,8 +750,9 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { // Shutdown: unblock every parked client thread. for (int i = 0; i < n_slots; i++) { if (slots[(size_t)i].job) { - slots[(size_t)i].failed = true; - retire_slot(i, false); + slots[(size_t)i].error = ResponseError::unavailable( + "server_shutting_down", "server shutting down"); + retire_slot(i); } } service_drains(); @@ -754,13 +764,21 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { // live on the wire. Close that protocol cleanly on shutdown instead // of waking the client thread and letting it truncate the response. const ParsedRequest & req = deferred->req; + const ResponseError error = ResponseError::unavailable( + "server_shutting_down", "server shutting down"); if (req.stream && deferred->sse_started) { + stop_job_stream(deferred); for (const std::string & chunk : - sse_error_close_chunks("server shutting down")) { - send_all(deferred->fd, chunk.data(), chunk.size()); + deferred->emitter->emit_error(error)) { + if (!send_job_bytes(deferred, chunk.data(), chunk.size())) { + break; + } } } else { - send_error(deferred->fd, 503, "server shutting down"); + const json body = build_error_response( + req.format, error, req.response_id); + send_response(deferred->fd, response_error_http_status(error), + "application/json", body.dump() + "\n"); } finish_job(deferred); } @@ -769,7 +787,13 @@ void HttpServer::scheduler_loop(SeqEngine & engine) { // its client-shutdown timeout and the destructor never has to wake threads // after the server/backend teardown has already started. while (ServerJob * queued = try_dequeue()) { - send_error(queued->fd, 503, "server shutting down"); + const ParsedRequest & req = queued->req; + const ResponseError error = ResponseError::unavailable( + "server_shutting_down", "server shutting down"); + const json body = build_error_response( + req.format, error, req.response_id); + send_response(queued->fd, response_error_http_status(error), + "application/json", body.dump() + "\n"); finish_job(queued); } } diff --git a/server/src/server/sse_emitter.cpp b/server/src/server/sse_emitter.cpp index 157be8526..34ad5d11b 100644 --- a/server/src/server/sse_emitter.cpp +++ b/server/src/server/sse_emitter.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace dflash::common { @@ -178,6 +179,7 @@ std::string SseEmitter::format_responses_event(const std::string & event_type, // ─── emit_start ───────────────────────────────────────────────────────── std::vector SseEmitter::emit_start() { + if (terminal_) return {}; std::vector out; switch (format_) { @@ -252,7 +254,7 @@ std::vector SseEmitter::emit_start() { // ─── emit_token ───────────────────────────────────────────────────────── std::vector SseEmitter::emit_token(const std::string & raw_piece) { - if (stop_hit_) return {}; // already stopped + if (terminal_ || stop_hit_) return {}; // already stopped // Track the first emit_token call whose mode-on-entry is CONTENT — // that's the first token attributed to the visible reply. Mode-on- @@ -698,6 +700,8 @@ std::vector SseEmitter::emit_finish(int completion_tokens, const GenTimings * timings, int generation_cap, bool ended_on_eos) { + if (terminal_) return {}; + terminal_ = true; std::vector out; // A tail still pending at end-of-stream is a genuinely truncated @@ -1066,6 +1070,46 @@ std::vector SseEmitter::emit_finish(int completion_tokens, return out; } +std::vector SseEmitter::emit_error( + const ResponseError & error) { + if (terminal_) return {}; + terminal_ = true; + finish_reason_ = "error"; + + switch (format_) { + case ApiFormat::OPENAI_CHAT: + case ApiFormat::COMPLETIONS: + return { + sse_data(build_error_response(format_, error, request_id_).dump()), + sse_data("[DONE]"), + }; + + case ApiFormat::ANTHROPIC: + return {sse_event( + "error", + build_error_response(format_, error, request_id_).dump())}; + + case ApiFormat::RESPONSES: { + json shell = { + {"id", request_id_}, + {"object", "response"}, + {"created_at", created_at_}, + {"status", "failed"}, + {"model", model_name_}, + {"output", json::array()}, + {"error", { + {"code", error.code}, + {"message", error.message}, + }}, + }; + return {format_responses_event( + "response.failed", {{"response", std::move(shell)}})}; + } + } + + return {}; +} + std::string SseEmitter::finish_reason() const { return finish_reason_; } diff --git a/server/src/server/sse_emitter.h b/server/src/server/sse_emitter.h index f22ff283c..4ee74949e 100644 --- a/server/src/server/sse_emitter.h +++ b/server/src/server/sse_emitter.h @@ -9,6 +9,7 @@ #include "tool_memory.h" #include "reasoning.h" #include "api_types.h" +#include "response_error.h" #include #include @@ -93,6 +94,10 @@ class SseEmitter { int generation_cap = -1, bool ended_on_eos = false); + // Emit the selected API's terminal failure sequence. Error and success + // terminals are mutually exclusive; repeated terminal calls are no-ops. + std::vector emit_error(const ResponseError & error); + // Get the finish_reason for non-streaming responses. std::string finish_reason() const; @@ -206,6 +211,7 @@ class SseEmitter { int64_t created_at_; std::string finish_reason_ = "stop"; + bool terminal_ = false; // Responses API IDs std::string msg_item_id_; diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index fabd44408..8a2cfda19 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -12,6 +12,7 @@ #include "server/tool_parser.h" #include "server/model_card.h" #include "server/reasoning.h" +#include "server/response_error.h" #include "server/prefix_cache.h" #include "server/pin_friendly_prompt.h" #include "server/disk_prefix_cache.h" @@ -6224,6 +6225,114 @@ TEST_CASE(ServerUnitFixture, test_generate_result_error_state_is_consistent) { TEST_ASSERT(result.error_detail().empty()); } +TEST_CASE(ServerUnitFixture, test_response_error_maps_every_generation_code) { + struct Case { + GenerateErrorCode code; + int status; + }; + const Case cases[] = { + {GenerateErrorCode::Incomplete, 500}, + {GenerateErrorCode::AdapterUnavailable, 503}, + {GenerateErrorCode::ResourceExhausted, 503}, + {GenerateErrorCode::ContextOverflow, 400}, + {GenerateErrorCode::SamplingUnsupported, 400}, + {GenerateErrorCode::PrefillFailed, 500}, + {GenerateErrorCode::DecodeSeedMissing, 500}, + {GenerateErrorCode::DecodeFailed, 500}, + {GenerateErrorCode::InvalidSnapshotSlot, 500}, + {GenerateErrorCode::ModelParked, 503}, + {GenerateErrorCode::BackendSpecific, 500}, + }; + + for (const Case & c : cases) { + const ResponseError error = to_response_error({c.code, {}}); + TEST_ASSERT(!error.code.empty()); + TEST_ASSERT(!error.message.empty()); + TEST_ASSERT(error.code == generate_error_code(c.code)); + TEST_ASSERT(response_error_http_status(error) == c.status); + } + + const ResponseError detailed = to_response_error( + {GenerateErrorCode::DecodeFailed, "device execution failed"}); + TEST_ASSERT(detailed.message == "device execution failed"); +} + +TEST_CASE(ServerUnitFixture, test_response_error_nonstream_formats) { + const ResponseError error = ResponseError::internal( + "decode_failed", "generation decode failed"); + + const json openai = build_error_response( + ApiFormat::OPENAI_CHAT, error, "chat_123"); + TEST_ASSERT(openai["error"]["type"] == "server_error"); + TEST_ASSERT(openai["error"]["code"] == "decode_failed"); + TEST_ASSERT(openai["error"]["message"] == "generation decode failed"); + + const json anthropic = build_error_response( + ApiFormat::ANTHROPIC, error, "msg_123"); + TEST_ASSERT(anthropic["type"] == "error"); + TEST_ASSERT(anthropic["error"]["type"] == "api_error"); + TEST_ASSERT(anthropic["request_id"] == "msg_123"); + + const json responses = build_error_response( + ApiFormat::RESPONSES, error, "resp_123"); + TEST_ASSERT(responses["error"]["type"] == "server_error"); + TEST_ASSERT(responses["error"]["code"] == "decode_failed"); +} + +TEST_CASE(ServerUnitFixture, test_response_error_factories_fill_empty_messages) { + TEST_ASSERT(ResponseError::invalid_request("invalid_request", {}).message == + "invalid request"); + TEST_ASSERT(ResponseError::unavailable("unavailable", {}).message == + "service unavailable"); + TEST_ASSERT(ResponseError::internal("engine_step_failed", {}).message == + "generation failed"); +} + +TEST_CASE(ServerUnitFixture, test_sse_emitter_openai_error_is_terminal) { + auto emitter = make_emitter(ApiFormat::OPENAI_CHAT); + std::string wire = concat(emitter.emit_start()); + wire += concat(emitter.emit_token("pending output that was already flushed")); + + wire += concat(emitter.emit_error( + ResponseError::internal("decode_failed", "decode failed"))); + TEST_ASSERT(wire.find("\"error\"") != std::string::npos); + TEST_ASSERT(wire.find("decode_failed") != std::string::npos); + TEST_ASSERT(wire.find("[DONE]") != std::string::npos); + TEST_ASSERT(wire.find("pending output") != std::string::npos); + TEST_ASSERT(emitter.finish_reason() == "error"); + TEST_ASSERT(emitter.emit_finish(1).empty()); + TEST_ASSERT(emitter.emit_token("late token").empty()); + TEST_ASSERT(emitter.emit_error(ResponseError::internal( + "late_error", "late error")).empty()); +} + +TEST_CASE(ServerUnitFixture, test_sse_emitter_anthropic_error_is_terminal) { + auto emitter = make_emitter(ApiFormat::ANTHROPIC); + emitter.emit_start(); + + const std::string wire = concat(emitter.emit_error( + ResponseError::unavailable("model_parked", "model unavailable"))); + TEST_ASSERT(wire.find("event: error") != std::string::npos); + TEST_ASSERT(wire.find("overloaded_error") != std::string::npos); + TEST_ASSERT(wire.find("message_stop") == std::string::npos); + TEST_ASSERT(emitter.emit_finish(0).empty()); +} + +TEST_CASE(ServerUnitFixture, test_sse_emitter_responses_error_is_terminal) { + auto emitter = make_emitter(ApiFormat::RESPONSES); + emitter.emit_start(); + + const std::string wire = concat(emitter.emit_error( + ResponseError::internal("prefill_failed", "prefill failed"))); + TEST_ASSERT(wire.find("event: response.failed") != std::string::npos); + TEST_ASSERT(wire.find("\"status\":\"failed\"") != + std::string::npos); + TEST_ASSERT(wire.find("prefill_failed") != std::string::npos); + TEST_ASSERT(wire.find("response.completed") == std::string::npos); + TEST_ASSERT(wire.find("[DONE]") == std::string::npos); + TEST_ASSERT(emitter.emit_finish(0).empty()); +} + // ═══════════════════════════════════════════════════════════════════════ // normalize_system_for_cache — header-strip tests // ═══════════════════════════════════════════════════════════════════════