Receive output from token processor and store them in cache
(self)
| 122 | return self.llm_engine.cfg._check_master() |
| 123 | |
| 124 | def _receive_output(self): |
| 125 | """ |
| 126 | Receive output from token processor and store them in cache |
| 127 | """ |
| 128 | while True: |
| 129 | try: |
| 130 | results = self.llm_engine._get_generated_result() |
| 131 | for request_id, contents in results.items(): |
| 132 | with self.mutex: |
| 133 | for result in contents: |
| 134 | if request_id not in self.req_output: |
| 135 | self.req_output[request_id] = result |
| 136 | continue |
| 137 | self.req_output[request_id].add(result) |
| 138 | except Exception as e: |
| 139 | llm_logger.error(f"Unexcepted error happened: {e}, {traceback.format_exc()!s}") |
| 140 | |
| 141 | def generate( |
| 142 | self, |