| 11511 | req_flags[0] ? " " : "", |
| 11512 | req_flags, |
| 11513 | now_sec() - t0); |
| 11514 | if (cold_store_len == prompt_for_sync->len) { |
| 11515 | if (kv_cache_store_live_prefix(s, slot, prompt_for_sync, |
| 11516 | cold_store_len, "cold")) { |
| 11517 | kv_cache_slot_note_store(slot, cold_store_len); |
| 11518 | suppressed_continued_last = -1; |
| 11519 | } else { |
| 11520 | kv_cache_slot_restore_suppressed(slot, suppressed_continued_last, |
| 11521 | cold_store_len); |
| 11522 | } |
| 11523 | } |
| 11524 | const uint64_t response_seq = server_next_sequence(s); |
| 11525 | char id[96]; |
| 11526 | snprintf(id, sizeof(id), "%s-%llu", |
| 11527 | j->req.kind == REQ_CHAT ? "chatcmpl" : "cmpl", |
| 11528 | (unsigned long long)response_seq); |
| 11529 | |
| 11530 | bool structured_stream = request_uses_structured_stream(&j->req); |
| 11531 | anthropic_stream anthropic_live = {0}; |
| 11532 | openai_stream openai_live = {0}; |
| 11533 | responses_stream responses_live = {0}; |
| 11534 | const bool openai_live_chat = request_uses_openai_live_stream(&j->req); |
| 11535 | const bool responses_live_chat = request_uses_responses_live_stream(&j->req); |
| 11536 | long responses_created_at = (long)time(NULL); |
| 11537 | if (j->req.stream) { |
| 11538 | if (progress.stream_failed) { |
| 11539 | server_log(DS4_LOG_GENERATION, |
| 11540 | "ds4-server: %s ctx=%s%s%s stream closed during prefill", |
| 11541 | j->req.kind == REQ_CHAT ? "chat" : "completion", |
| 11542 | ctx_span, |
| 11543 | req_flags[0] ? " " : "", |
| 11544 | req_flags); |
| 11545 | request_live_state_clear(s, slot); |
| 11546 | ds4_tokens_free(&effective_prompt); |
| 11547 | return; |
| 11548 | } |
| 11549 | /* The prefill progress callback may have already sent the SSE headers |
| 11550 | * to keep the connection alive during a long prefill. Only emit them |
| 11551 | * here when prefill never fired (e.g. fully cached prompt). */ |
| 11552 | if (!progress.headers_sent && !sse_headers(j->fd, s->enable_cors)) { |
| 11553 | job_mark_cancelled(j); |
| 11554 | server_log(DS4_LOG_GENERATION, |
| 11555 | "ds4-server: %s ctx=%s%s%s sse headers failed", |
| 11556 | j->req.kind == REQ_CHAT ? "chat" : "completion", |
| 11557 | ctx_span, |
| 11558 | req_flags[0] ? " " : "", |
| 11559 | req_flags); |
| 11560 | request_live_state_clear(s, slot); |
| 11561 | ds4_tokens_free(&effective_prompt); |
| 11562 | return; |
| 11563 | } |
| 11564 | progress.headers_sent = true; |
| 11565 | if (j->req.api == API_ANTHROPIC && |
| 11566 | !anthropic_sse_start_live(j->fd, &j->req, id, |
| 11567 | prompt_tokens, &anthropic_live)) { |
| 11568 | job_mark_cancelled(j); |
| 11569 | server_log(DS4_LOG_GENERATION, "ds4-server: chat ctx=%s anthropic stream start failed", ctx_span); |
| 11570 | request_live_state_clear(s, slot); |
no test coverage detected