(
self,
top_k: int = 40,
top_p: float = 0.95,
min_p: float = 0.05,
typical_p: float = 1.0,
temp: float = 0.80,
repeat_penalty: float = 1.0,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
tfs_z: float = 1.0,
mirostat_mode: int = 0,
mirostat_eta: float = 0.1,
mirostat_tau: float = 5.0,
penalize_nl: bool = True,
logits_processor: Optional[LogitsProcessorList] = None,
grammar: Optional[LlamaGrammar] = None,
)
| 695 | self._requires_eval = False |
| 696 | |
| 697 | def _init_sampler( |
| 698 | self, |
| 699 | top_k: int = 40, |
| 700 | top_p: float = 0.95, |
| 701 | min_p: float = 0.05, |
| 702 | typical_p: float = 1.0, |
| 703 | temp: float = 0.80, |
| 704 | repeat_penalty: float = 1.0, |
| 705 | frequency_penalty: float = 0.0, |
| 706 | presence_penalty: float = 0.0, |
| 707 | tfs_z: float = 1.0, |
| 708 | mirostat_mode: int = 0, |
| 709 | mirostat_eta: float = 0.1, |
| 710 | mirostat_tau: float = 5.0, |
| 711 | penalize_nl: bool = True, |
| 712 | logits_processor: Optional[LogitsProcessorList] = None, |
| 713 | grammar: Optional[LlamaGrammar] = None, |
| 714 | ): |
| 715 | sampler = internals.LlamaSampler() |
| 716 | |
| 717 | if logits_processor is not None: |
| 718 | # Create and add a custom sampler |
| 719 | def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): |
| 720 | size = token_data_array.contents.size |
| 721 | data_soa = token_data_array.contents.data |
| 722 | data_soa_address = ctypes.addressof(data_soa.contents) |
| 723 | # NOTE: This is probably broken |
| 724 | recarray = np.recarray( |
| 725 | shape=(size,), |
| 726 | dtype=np.dtype( |
| 727 | [("id", np.intc), ("logit", np.single), ("p", np.single)], |
| 728 | align=True, |
| 729 | ), |
| 730 | buf=(llama_cpp.llama_token_data * size).from_address( |
| 731 | data_soa_address |
| 732 | ), |
| 733 | ) |
| 734 | for logit_processor in logits_processor: |
| 735 | recarray.logit[:] = logit_processor(self._input_ids, recarray.logit) |
| 736 | |
| 737 | sampler.add_custom(apply_func) |
| 738 | |
| 739 | sampler.add_penalties( |
| 740 | # n_vocab=self._n_vocab, |
| 741 | # special_eos_id=self._token_eos, |
| 742 | # linefeed_id=self._token_nl, |
| 743 | penalty_last_n=self.last_n_tokens_size, |
| 744 | penalty_repeat=repeat_penalty, |
| 745 | penalty_freq=frequency_penalty, |
| 746 | penalty_present=presence_penalty, |
| 747 | # penalize_nl=penalize_nl, |
| 748 | # ignore_eos=False, |
| 749 | ) |
| 750 | |
| 751 | if grammar is not None: |
| 752 | sampler.add_grammar(self._model, grammar) |
| 753 | |
| 754 | if temp < 0.0: |
no test coverage detected