Create a generator of tokens from a prompt. Examples: >>> llama = Llama("models/ggml-7b.bin") >>> tokens = llama.tokenize(b"Hello, world!") >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.0): ... pr
(
self,
tokens: Sequence[int],
top_k: int = 40,
top_p: float = 0.95,
min_p: float = 0.05,
typical_p: float = 1.0,
temp: float = 0.80,
repeat_penalty: float = 1.0,
reset: bool = True,
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
tfs_z: float = 1.0,
mirostat_mode: int = 0,
mirostat_tau: float = 5.0,
mirostat_eta: float = 0.1,
penalize_nl: bool = True,
logits_processor: Optional[LogitsProcessorList] = None,
stopping_criteria: Optional[StoppingCriteriaList] = None,
grammar: Optional[LlamaGrammar] = None,
)
| 845 | return token |
| 846 | |
| 847 | def generate( |
| 848 | self, |
| 849 | tokens: Sequence[int], |
| 850 | top_k: int = 40, |
| 851 | top_p: float = 0.95, |
| 852 | min_p: float = 0.05, |
| 853 | typical_p: float = 1.0, |
| 854 | temp: float = 0.80, |
| 855 | repeat_penalty: float = 1.0, |
| 856 | reset: bool = True, |
| 857 | frequency_penalty: float = 0.0, |
| 858 | presence_penalty: float = 0.0, |
| 859 | tfs_z: float = 1.0, |
| 860 | mirostat_mode: int = 0, |
| 861 | mirostat_tau: float = 5.0, |
| 862 | mirostat_eta: float = 0.1, |
| 863 | penalize_nl: bool = True, |
| 864 | logits_processor: Optional[LogitsProcessorList] = None, |
| 865 | stopping_criteria: Optional[StoppingCriteriaList] = None, |
| 866 | grammar: Optional[LlamaGrammar] = None, |
| 867 | ) -> Generator[int, Optional[Sequence[int]], None]: |
| 868 | """Create a generator of tokens from a prompt. |
| 869 | |
| 870 | Examples: |
| 871 | >>> llama = Llama("models/ggml-7b.bin") |
| 872 | >>> tokens = llama.tokenize(b"Hello, world!") |
| 873 | >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.0): |
| 874 | ... print(llama.detokenize([token])) |
| 875 | |
| 876 | Args: |
| 877 | tokens: The prompt tokens. |
| 878 | top_k: The top-k sampling parameter. |
| 879 | top_p: The top-p sampling parameter. |
| 880 | temp: The temperature parameter. |
| 881 | repeat_penalty: The repeat penalty parameter. |
| 882 | reset: Whether to reset the model state. |
| 883 | |
| 884 | Yields: |
| 885 | The generated tokens. |
| 886 | """ |
| 887 | # Reset mirostat sampling |
| 888 | self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau) |
| 889 | self._sampler = self._init_sampler( |
| 890 | top_k=top_k, |
| 891 | top_p=top_p, |
| 892 | min_p=min_p, |
| 893 | typical_p=typical_p, |
| 894 | temp=temp, |
| 895 | repeat_penalty=repeat_penalty, |
| 896 | frequency_penalty=frequency_penalty, |
| 897 | presence_penalty=presence_penalty, |
| 898 | tfs_z=tfs_z, |
| 899 | mirostat_mode=mirostat_mode, |
| 900 | mirostat_tau=mirostat_tau, |
| 901 | mirostat_eta=mirostat_eta, |
| 902 | penalize_nl=penalize_nl, |
| 903 | logits_processor=logits_processor, |
| 904 | grammar=grammar, |