(
self,
prompt: Union[str, List[int]],
suffix: Optional[str] = None,
max_tokens: Optional[int] = 16,
temperature: float = 0.8,
top_p: float = 0.95,
min_p: float = 0.05,
typical_p: float = 1.0,
logprobs: Optional[int] = None,
echo: bool = False,
stop: Optional[Union[str, List[str]]] = [],
frequency_penalty: float = 0.0,
presence_penalty: float = 0.0,
repeat_penalty: float = 1.0,
top_k: int = 40,
stream: bool = False,
seed: Optional[int] = None,
tfs_z: float = 1.0,
mirostat_mode: int = 0,
mirostat_tau: float = 5.0,
mirostat_eta: float = 0.1,
model: Optional[str] = None,
stopping_criteria: Optional[StoppingCriteriaList] = None,
logits_processor: Optional[LogitsProcessorList] = None,
grammar: Optional[LlamaGrammar] = None,
logit_bias: Optional[Dict[int, float]] = None,
)
| 1186 | return output |
| 1187 | |
| 1188 | def _create_completion( |
| 1189 | self, |
| 1190 | prompt: Union[str, List[int]], |
| 1191 | suffix: Optional[str] = None, |
| 1192 | max_tokens: Optional[int] = 16, |
| 1193 | temperature: float = 0.8, |
| 1194 | top_p: float = 0.95, |
| 1195 | min_p: float = 0.05, |
| 1196 | typical_p: float = 1.0, |
| 1197 | logprobs: Optional[int] = None, |
| 1198 | echo: bool = False, |
| 1199 | stop: Optional[Union[str, List[str]]] = [], |
| 1200 | frequency_penalty: float = 0.0, |
| 1201 | presence_penalty: float = 0.0, |
| 1202 | repeat_penalty: float = 1.0, |
| 1203 | top_k: int = 40, |
| 1204 | stream: bool = False, |
| 1205 | seed: Optional[int] = None, |
| 1206 | tfs_z: float = 1.0, |
| 1207 | mirostat_mode: int = 0, |
| 1208 | mirostat_tau: float = 5.0, |
| 1209 | mirostat_eta: float = 0.1, |
| 1210 | model: Optional[str] = None, |
| 1211 | stopping_criteria: Optional[StoppingCriteriaList] = None, |
| 1212 | logits_processor: Optional[LogitsProcessorList] = None, |
| 1213 | grammar: Optional[LlamaGrammar] = None, |
| 1214 | logit_bias: Optional[Dict[int, float]] = None, |
| 1215 | ) -> Union[ |
| 1216 | Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] |
| 1217 | ]: |
| 1218 | assert suffix is None or suffix.__class__ is str |
| 1219 | |
| 1220 | completion_id: str = f"cmpl-{str(uuid.uuid4())}" |
| 1221 | created: int = int(time.time()) |
| 1222 | bos_token_id: int = self.token_bos() |
| 1223 | cls_token_id: int = self._model.token_cls() |
| 1224 | sep_token_id: int = self._model.token_sep() |
| 1225 | prefix_token_id: int = 0 # self._model.token_prefix() # TODO: Fix |
| 1226 | middle_token_id: int = 0 # self._model.token_middle() # TODO: Fix |
| 1227 | suffix_token_id: int = 0 # self._model.token_suffix() # TODO: Fix |
| 1228 | add_space_prefix: bool = ( |
| 1229 | self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" |
| 1230 | ) |
| 1231 | bos_tokens: List[int] = [cls_token_id if cls_token_id != -1 else bos_token_id] |
| 1232 | eos_tokens: List[int] = [ |
| 1233 | sep_token_id if sep_token_id != -1 else self.token_eos() |
| 1234 | ] |
| 1235 | |
| 1236 | if ( |
| 1237 | (isinstance(prompt, list) and suffix is None) |
| 1238 | or not self._model.add_bos_token() |
| 1239 | or bos_tokens[:1] == [-1] |
| 1240 | ): |
| 1241 | bos_tokens = [] |
| 1242 | |
| 1243 | if (isinstance(prompt, list) and suffix is None) or ( |
| 1244 | not self._model.add_eos_token() and sep_token_id == -1 |
| 1245 | ): |
no test coverage detected