Evaluate a list of tokens. Args: tokens: The list of tokens to evaluate.
(self, tokens: Sequence[int])
| 657 | llama_cpp.llama_memory_clear(mem, True) |
| 658 | |
| 659 | def eval(self, tokens: Sequence[int]): |
| 660 | """Evaluate a list of tokens. |
| 661 | |
| 662 | Args: |
| 663 | tokens: The list of tokens to evaluate. |
| 664 | """ |
| 665 | self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1) |
| 666 | for i in range(0, len(tokens), self.n_batch): |
| 667 | batch = tokens[i : min(len(tokens), i + self.n_batch)] |
| 668 | n_past = self.n_tokens |
| 669 | n_tokens = len(batch) |
| 670 | self._batch.set_batch( |
| 671 | batch=batch, n_past=n_past, logits_all=self._logits_all |
| 672 | ) |
| 673 | self._ctx.decode(self._batch) |
| 674 | # Save tokens |
| 675 | self.input_ids[n_past : n_past + n_tokens] = batch |
| 676 | # Save logits |
| 677 | if self._logits_all: |
| 678 | rows = n_tokens |
| 679 | cols = self._n_vocab |
| 680 | logits = np.ctypeslib.as_array( |
| 681 | self._ctx.get_logits(), shape=(rows * cols,) |
| 682 | ) |
| 683 | self.scores[n_past : n_past + n_tokens, :].reshape(-1)[::] = logits |
| 684 | else: |
| 685 | # rows = 1 |
| 686 | # cols = self._n_vocab |
| 687 | # logits = np.ctypeslib.as_array( |
| 688 | # self._ctx.get_logits(), shape=(rows * cols,) |
| 689 | # ) |
| 690 | # self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits |
| 691 | # NOTE: Now that sampling is done inside the sampler, logits are only needed for logprobs which requires logits_all |
| 692 | pass |
| 693 | # Update n_tokens |
| 694 | self.n_tokens += n_tokens |
| 695 | self._requires_eval = False |
| 696 | |
| 697 | def _init_sampler( |
| 698 | self, |