Method add_requests

fastdeploy/engine/engine.py:255–343

Add a new request to the queue. Args: task: Request A dictionary representing the request. sampling_params: A dictionary representing the sampling parameters. Returns: None

(self, task, sampling_params=None, **kwargs)

        )

    def add_requests(self, task, sampling_params=None, **kwargs):
        """
        Add a new request to the queue.

        Args:
            task: Request A dictionary representing the request.
            sampling_params: A dictionary representing the sampling parameters.

        Returns:
            None
        """
        # TODO 输入输出长度确认

        if sampling_params is not None:
            task.update(asdict(sampling_params))
        request = Request.from_dict(task)
        request.metrics.scheduler_recv_req_time = time.time()
        llm_logger.info(f"Receive request {request}")
        if sampling_params is not None:
            if sampling_params.temperature is not None and abs(sampling_params.temperature) < 1e-06:
                sampling_params.temperature = 1e-06
            request.sampling_params = sampling_params
        request.metrics.preprocess_start_time = time.time()
        chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
        chat_template_kwargs["chat_template"] = kwargs.get("chat_template")
        kwargs["chat_template_kwargs"] = chat_template_kwargs
        request = self.engine.data_processor.process_request(request, self.cfg.model_config.max_model_len, **kwargs)
        request.prompt_token_ids_len = len(request.prompt_token_ids)
        request.need_prefill_tokens = request.prompt_token_ids_len
        input_ids_len = request.prompt_token_ids_len
        request.set(
            "max_tokens",
            min(
                self.cfg.model_config.max_model_len - input_ids_len,
                request.get("max_tokens"),
            ),
        )
        min_tokens = request.get("min_tokens")
        if input_ids_len + min_tokens >= self.cfg.model_config.max_model_len:
            error_msg = (
                f"Input text is too long, length of prompt token({input_ids_len}) "
                f"+ min_dec_len ({min_tokens}) >= max_model_len "
            )
            llm_logger.error(error_msg)
            raise EngineError(error_msg, error_code=400)

        if input_ids_len > self.cfg.model_config.max_model_len:
            error_msg = f"Length of input token({input_ids_len}) exceeds the limit max_model_len({self.cfg.model_config.max_model_len})."
            llm_logger.error(error_msg)
            raise EngineError(error_msg, error_code=400)

        if request.get("stop_seqs_len") is not None:
            stop_seqs_len = request.get("stop_seqs_len")
            max_stop_seqs_num = envs.FD_MAX_STOP_SEQS_NUM
            if len(stop_seqs_len) > max_stop_seqs_num:
                error_msg = (
                    f"Length of stop ({stop_seqs_len}) exceeds the limit max_stop_seqs_num({max_stop_seqs_num})."
                    "Please reduce the number of stop or set a lager max_stop_seqs_num by `FD_MAX_STOP_SEQS_NUM`"