Generate function for the LLM class. Args: prompts (Union[str, list[str], list[int], list[list[int]], dict[str, Any], list[dict[str, Any]]]): The prompt to use for generating the response. sampling_params (Optional[Union[SamplingParams, list[
(
self,
prompts: Union[
str,
list[str],
list[int],
list[list[int]],
dict[str, Any],
list[dict[str, Any]],
],
sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
use_tqdm: bool = True,
stream: bool = False,
)
| 139 | llm_logger.error(f"Unexcepted error happened: {e}, {traceback.format_exc()!s}") |
| 140 | |
| 141 | def generate( |
| 142 | self, |
| 143 | prompts: Union[ |
| 144 | str, |
| 145 | list[str], |
| 146 | list[int], |
| 147 | list[list[int]], |
| 148 | dict[str, Any], |
| 149 | list[dict[str, Any]], |
| 150 | ], |
| 151 | sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None, |
| 152 | use_tqdm: bool = True, |
| 153 | stream: bool = False, |
| 154 | ): |
| 155 | """ |
| 156 | Generate function for the LLM class. |
| 157 | |
| 158 | Args: |
| 159 | prompts (Union[str, list[str], list[int], list[list[int]], dict[str, Any], list[dict[str, Any]]]): |
| 160 | The prompt to use for generating the response. |
| 161 | sampling_params (Optional[Union[SamplingParams, list[SamplingParams]]], optional): |
| 162 | The sampling parameters to use for generating the response. Defaults to None. |
| 163 | use_tqdm (bool, optional): Whether to use tqdm for the progress bar. Defaults to True. |
| 164 | stream (bool, optional): Whether to return a streaming iterator. Defaults to False. |
| 165 | |
| 166 | Returns: |
| 167 | If stream=False: Union[str, list[str]]: The generated response. |
| 168 | If stream=True: Iterator: An iterator that yields partial responses as they become available. |
| 169 | """ |
| 170 | |
| 171 | if not self._check_master(): |
| 172 | err_msg = f"Only master node can accept completion request, please send request to master node: {self.master_node_ip}" |
| 173 | raise ValueError(err_msg) |
| 174 | |
| 175 | if sampling_params is None: |
| 176 | sampling_params = self.default_sampling_params |
| 177 | |
| 178 | if isinstance(sampling_params, SamplingParams): |
| 179 | sampling_params_len = 1 |
| 180 | else: |
| 181 | sampling_params_len = len(sampling_params) |
| 182 | |
| 183 | if isinstance(prompts, str): |
| 184 | prompts = [prompts] |
| 185 | |
| 186 | if isinstance(prompts, list) and isinstance(prompts[0], int): |
| 187 | prompts = [prompts] |
| 188 | |
| 189 | if isinstance(prompts, dict): |
| 190 | if "prompt" not in prompts: |
| 191 | raise ValueError("prompts must be a input dict") |
| 192 | prompts = [prompts] |
| 193 | # sampling_params = None |
| 194 | |
| 195 | if sampling_params_len != 1 and len(prompts) != sampling_params_len: |
| 196 | raise ValueError("prompts and sampling_params must be the same length.") |
| 197 | |
| 198 | req_ids = self._add_request(prompts=prompts, sampling_params=sampling_params) |