Function generate_text_blocking

cake-core/src/cake/sharding/api/text.rs:121–180

(
    state: web::Data<Arc<RwLock<Master<M>>>>,
    request: ChatRequest,
)

}

async fn generate_text_blocking<M: Model>(
    state: web::Data<Arc<RwLock<Master<M>>>>,
    request: ChatRequest,
) -> HttpResponse {
    let mut master = state.write().await;

    if !master.model.as_ref().is_some_and(|m| m.output_modality() == OutputModality::Text) {
        return HttpResponse::NotFound()
            .json(serde_json::json!({"error": "No text model loaded"}));
    }

    if let Err(e) = master.reset() {
        return HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{e}")}));
    }

    let num_messages = request.messages.len();
    let model = master.model.as_mut().unwrap();
    for message in request.messages {
        if let Err(e) = model.add_message(message) {
            return HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{e}")}));
        }
    }

    let mut resp = String::new();
    let mut finish_reason = "length".to_string();

    let gen_result = master
        .generate_text(request.max_tokens, |data| {
            if data.is_empty() {
                finish_reason = "stop".to_string();
            } else {
                resp += data;
            }
        })
        .await;

    println!();

    let completion_tokens = master
        .model
        .as_ref()
        .map(|m| m.generated_tokens())
        .unwrap_or(0);

    let _ = master.goodbye().await;

    if let Err(e) = gen_result {
        return HttpResponse::InternalServerError().json(serde_json::json!({"error": format!("{e}")}));
    }

    let response = ChatResponse::new(
        M::MODEL_NAME.to_string(),
        resp,
        num_messages,
        completion_tokens,
        finish_reason,
    );