(
self,
request: CompletionRequest,
prompt_output_index: Optional[int],
prompt_logits: Optional[np.ndarray] = None,
)
| 14615 | return None |
| 14616 | |
| 14617 | def start_completions( |
| 14618 | self, |
| 14619 | request: CompletionRequest, |
| 14620 | prompt_output_index: Optional[int], |
| 14621 | prompt_logits: Optional[np.ndarray] = None, |
| 14622 | ) -> None: |
| 14623 | if request.completions: |
| 14624 | return |
| 14625 | assert request.base_seq_id is not None |
| 14626 | prompt_tokens = list(request.prompt_plan.text_tokens) |
| 14627 | prompt_length = len(request.prompt_tokens) |
| 14628 | multimodal_prompt = any( |
| 14629 | segment.kind != "text" for segment in request.prompt_plan.segments |
| 14630 | ) |
| 14631 | prompt_text = request.prompt_text |
| 14632 | if request.payload.stop is None: |
| 14633 | stop_sequences: List[bytes] = [] |
| 14634 | elif isinstance(request.payload.stop, str): |
| 14635 | stop_sequences = [request.payload.stop.encode("utf-8")] |
| 14636 | else: |
| 14637 | stop_sequences = [item.encode("utf-8") for item in request.payload.stop] |
| 14638 | logit_bias = ( |
| 14639 | {int(token): float(bias) for token, bias in request.payload.logit_bias.items()} |
| 14640 | if request.payload.logit_bias |
| 14641 | else None |
| 14642 | ) |
| 14643 | prompt_text_bytes = self.model.detokenize(prompt_tokens) if prompt_tokens else b"" |
| 14644 | for offset, seq_id in enumerate(request.completion_seq_ids): |
| 14645 | if offset > 0: |
| 14646 | if prompt_length: |
| 14647 | self.memory_policy.copy_prompt_state( |
| 14648 | request.base_seq_id, |
| 14649 | seq_id, |
| 14650 | prompt_length, |
| 14651 | ) |
| 14652 | sampler = Sampler( |
| 14653 | seed=(request.payload.seed or llama_cpp.LLAMA_DEFAULT_SEED) + offset, |
| 14654 | vocab=self.model.vocab, |
| 14655 | n_vocab=self.model.n_vocab, |
| 14656 | top_p=request.payload.top_p, |
| 14657 | temperature=request.payload.temperature, |
| 14658 | frequency_penalty=request.payload.frequency_penalty or 0.0, |
| 14659 | presence_penalty=request.payload.presence_penalty or 0.0, |
| 14660 | logit_bias=logit_bias, |
| 14661 | grammar_text=request.grammar_text, |
| 14662 | grammar_root=request.grammar_root, |
| 14663 | ) |
| 14664 | request.completions.append( |
| 14665 | Completion( |
| 14666 | request_id=request.id, |
| 14667 | index=offset, |
| 14668 | seq_id=seq_id, |
| 14669 | sampler=sampler, |
| 14670 | prompt_tokens=prompt_tokens, |
| 14671 | prompt_length=prompt_length, |
| 14672 | prompt_text=prompt_text, |
| 14673 | multimodal_prompt=multimodal_prompt, |
| 14674 | max_total_tokens=request.effective_max_len, |
no test coverage detected