(self, prompts, temperature = 0.7, n = 200)
| 268 | |
| 269 | |
| 270 | def infer_many(self, prompts, temperature = 0.7, n = 200): |
| 271 | self.sampling_params = SamplingParams(temperature=temperature, n = n, max_tokens = 512) |
| 272 | predictions = {} |
| 273 | try: |
| 274 | outputs = self.model.generate(prompts, self.sampling_params) |
| 275 | for index, out in enumerate(outputs): |
| 276 | preds = [] |
| 277 | for o in out.outputs: |
| 278 | preds.append(o.text) |
| 279 | predictions[prompts[index]] = [preds, True] |
| 280 | return predictions |
| 281 | except Exception as e: |
| 282 | logger.error("Error occurred with reason {} when processing multiple prompts".format(str(e))) |
| 283 | print(traceback.print_exc()) |
| 284 | return str(e) |
| 285 | |
| 286 | def infer_many_chats(self, dialogs, n = 10): |
| 287 | prompt_tokens = self.format_chats(dialogs, pad = True) |
no test coverage detected