| 1137 | } |
| 1138 | |
| 1139 | void send_partial_response(llama_client_slot &slot, completion_token_output tkn) |
| 1140 | { |
| 1141 | std::lock_guard<std::mutex> lock(mutex_results); |
| 1142 | task_result res; |
| 1143 | res.id = slot.task_id; |
| 1144 | res.error = false; |
| 1145 | res.stop = false; |
| 1146 | |
| 1147 | res.result_json = json |
| 1148 | { |
| 1149 | {"content", tkn.text_to_send}, |
| 1150 | {"stop", false}, |
| 1151 | {"slot_id", slot.id}, |
| 1152 | {"multimodal", multimodal} |
| 1153 | }; |
| 1154 | |
| 1155 | if (slot.sparams.n_probs > 0) |
| 1156 | { |
| 1157 | std::vector<completion_token_output> probs_output = {}; |
| 1158 | const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false); |
| 1159 | size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size()); |
| 1160 | size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size()); |
| 1161 | if (probs_pos < probs_stop_pos) |
| 1162 | { |
| 1163 | probs_output = std::vector<completion_token_output>(slot.generated_token_probs.begin() + probs_pos, slot.generated_token_probs.begin() + probs_stop_pos); |
| 1164 | } |
| 1165 | slot.sent_token_probs_index = probs_stop_pos; |
| 1166 | res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs_output); |
| 1167 | } |
| 1168 | |
| 1169 | queue_results.push_back(res); |
| 1170 | } |
| 1171 | |
| 1172 | void send_final_response(llama_client_slot &slot) |
| 1173 | { |
nothing calls this directly
no test coverage detected