Method step

colossalai/inference/core/rpc_engine.py:273–284

(self)

        return ret[0]

    def step(self) -> List[str]:
        batch = self.request_handler.schedule()

        input_token_ids, input_meta_data = self.prepare_input(batch)
        # TODO: padding_id is used for generating attn_mask and will be removed if nopad version is supported.
        next_tokens = asyncio.run(self.step_(input_token_ids, input_meta_data))

        # update the request_handler
        next_tokens = torch.tensor(next_tokens, dtype=torch.int)
        self.request_handler.append_next_tokens(next_tokens)
        finished_sequences = self.request_handler.update()
        return finished_sequences

    def kill_workers(self):
        """