Generates a response based on the given prompt using the model. Args: prompts (dict): The prompt to use for generating the response. stream (bool): Whether to stream the output or wait until completion. Yields: dict: The generated respon
(self, prompts, stream)
| 672 | return prompts["request_id"] |
| 673 | |
| 674 | def generate(self, prompts, stream): |
| 675 | """ |
| 676 | Generates a response based on the given prompt using the model. |
| 677 | |
| 678 | Args: |
| 679 | prompts (dict): The prompt to use for generating the response. |
| 680 | stream (bool): Whether to stream the output or wait until completion. |
| 681 | |
| 682 | Yields: |
| 683 | dict: The generated response. |
| 684 | """ |
| 685 | llm_logger.info(f"Starting generation for prompt: {prompts}") |
| 686 | try: |
| 687 | req_id = self._format_and_add_data(prompts) |
| 688 | except Exception as e: |
| 689 | llm_logger.error(f"Error happened while adding request, details={e}, {str(traceback.format_exc())}") |
| 690 | raise EngineError(str(e), error_code=400) |
| 691 | |
| 692 | # Get the result of the current request |
| 693 | for result in self._get_generated_tokens(req_id): |
| 694 | is_end = result.finished |
| 695 | if stream and not is_end: |
| 696 | processed = self.engine.data_processor.process_response(result) |
| 697 | if processed is None: |
| 698 | continue |
| 699 | output = processed.to_dict() |
| 700 | yield output |
| 701 | |
| 702 | # Exit loop if termination condition is met |
| 703 | if is_end: |
| 704 | processed = self.engine.data_processor.process_response(result) |
| 705 | output = processed.to_dict() |
| 706 | llm_logger.debug(f"Generate result: {output}") |
| 707 | if not stream: |
| 708 | yield output |
| 709 | else: |
| 710 | output["outputs"]["text"] = "" |
| 711 | output["outputs"]["reasoning_content"] = "" |
| 712 | yield output |
| 713 | |
| 714 | self.engine.check_and_free_block_tables() |
| 715 | |
| 716 | def _stop_profile(self): |
| 717 | """ |
nothing calls this directly
no test coverage detected