Method add_request

colossalai/inference/core/engine.py:87–105

Add requests. Args: request_ids (List[int], optional): The request ID. Defaults to None. prompts (Union[List[str], optional): Input prompts. Defaults to None. prompts_token_ids (List[List[int]], optional): token ids of input prompts. Defaults to

(
        self,
        request_ids: Union[List[int], int] = None,
        prompts: Union[List[str], str] = None,
        *args,
        **kwargs,
    )

        return self.engine.generate(request_ids=request_ids, prompts=prompts, *args, **kwargs)

    def add_request(
        self,
        request_ids: Union[List[int], int] = None,
        prompts: Union[List[str], str] = None,
        *args,
        **kwargs,
    ) -> None:
        """
        Add requests.

        Args:
            request_ids (List[int], optional): The request ID. Defaults to None.
            prompts (Union[List[str], optional): Input prompts. Defaults to None.
            prompts_token_ids (List[List[int]], optional): token ids of input prompts. Defaults to None.
            kwargs: for LLM, it could be max_length, max_new_tokens, etc
                    for diffusion, it could be prompt_2, prompt_3, num_images_per_prompt, do_classifier_free_guidance, negative_prompt, negative_prompt_2, negative_prompt_3, prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, clip_skip, which aligns with diffusers
        """
        assert self.engine is not None, "Please init Engine first"
        self.engine.add_request(request_ids=request_ids, prompts=prompts, *args, **kwargs)

    def step(self):
        assert self.engine is not None, "Please init Engine first"