Method _send_resp

lmdeploy/pytorch/engine/engine_loop.py:198–215

Send response.

(self, out: InferOutput)

            logger.debug(f'Response: num_outputs={len(outputs)}.')

    def _send_resp(self, out: InferOutput):
        """Send response."""
        logprobs = None if out.resp.data is None else out.resp.data.get('logprobs', None)
        if out.finish:
            resp_type = ResponseType.FINISH
        elif out.resp.is_done:
            resp_type = out.resp.type
        else:
            resp_type = ResponseType.SUCCESS
        response_reqs(self.req_manager,
                      out.resp,
                      resp_type,
                      data=dict(token_ids=out.token_ids,
                                logits=out.logits,
                                cache_block_ids=out.cache_block_ids,
                                req_metrics=out.req_metrics,
                                routed_experts=out.routed_experts,
                                logprobs=logprobs))

    @staticmethod
    def _update_logprobs(step_outputs: list[InferOutput]):