Method __init__

tests/conftest.py:27–55

(
        self,
        model_name_or_path: str,
        tensor_parallel_size: int = 1,
        max_num_seqs: int = 1,
        max_model_len: int = 1024,
        load_choices: str = "default",
        quantization: str = "None",
        **kwargs,
    )

class FDRunner:
    def __init__(
        self,
        model_name_or_path: str,
        tensor_parallel_size: int = 1,
        max_num_seqs: int = 1,
        max_model_len: int = 1024,
        load_choices: str = "default",
        quantization: str = "None",
        **kwargs,
    ) -> None:
        from fastdeploy.entrypoints.llm import LLM

        clean_ports()
        time.sleep(10)
        graph_optimization_config = {"use_cudagraph": False}
        self.llm = LLM(
            model=model_name_or_path,
            tensor_parallel_size=tensor_parallel_size,
            max_num_seqs=max_num_seqs,
            max_model_len=max_model_len,
            load_choices=load_choices,
            quantization=quantization,
            max_num_batched_tokens=max_model_len,
            graph_optimization_config=graph_optimization_config,
            port=FD_API_PORT,
            cache_queue_port=FD_CACHE_QUEUE_PORT,
            engine_worker_queue_port=FD_ENGINE_QUEUE_PORT,
            **kwargs,
        )

    def generate(
        self,