Initializes a Language Model instance. Args: model (str): The name of the language model to use. Supported models are listed in `LLMEngine.SUPPORTED_MODELS`. tokenizer (Optional[str], optional): The name of the tokenizer to use. Defaults
| 55 | |
| 56 | |
| 57 | class LLM: |
| 58 | """ |
| 59 | Initializes a Language Model instance. |
| 60 | |
| 61 | Args: |
| 62 | model (str): |
| 63 | The name of the language model to use. Supported models are listed in |
| 64 | `LLMEngine.SUPPORTED_MODELS`. |
| 65 | tokenizer (Optional[str], optional): |
| 66 | The name of the tokenizer to use. Defaults to None. If not specified, the |
| 67 | default tokenizer for the selected model will be used. |
| 68 | gpu_memory_utilization: The ratio (between 0 and 1) of GPU memory to |
| 69 | reserve for the model weights, activations, and KV cache. Higher |
| 70 | values will increase the KV cache size and thus improve the model's |
| 71 | throughput. However, if the value is too high, it may cause out-of- |
| 72 | memory (OOM) errors. |
| 73 | **kwargs (optional): |
| 74 | Additional keyword arguments to pass to the `EngineArgs` constructor. See |
| 75 | `EngineArgs.__init__` for details. Defaults to {}. |
| 76 | |
| 77 | Raises: |
| 78 | ValueError: |
| 79 | If `model` is not in `LLMEngine.SUPPORTED_MODELS`. |
| 80 | """ |
| 81 | |
| 82 | def __init__( |
| 83 | self, |
| 84 | model: str, |
| 85 | revision: Optional[str] = "master", |
| 86 | tokenizer: Optional[str] = None, |
| 87 | enable_logprob: Optional[bool] = False, |
| 88 | chat_template: Optional[str] = None, |
| 89 | **kwargs, |
| 90 | ): |
| 91 | deprecated_kwargs_warning(**kwargs) |
| 92 | |
| 93 | model = retrive_model_from_server(model, revision) |
| 94 | tool_parser_plugin = kwargs.get("tool_parser_plugin") |
| 95 | if tool_parser_plugin: |
| 96 | ToolParserManager.import_tool_parser(tool_parser_plugin) |
| 97 | engine_args = EngineArgs( |
| 98 | model=model, |
| 99 | tokenizer=tokenizer, |
| 100 | enable_logprob=enable_logprob, |
| 101 | **kwargs, |
| 102 | ) |
| 103 | |
| 104 | # Create the Engine |
| 105 | self.llm_engine = LLMEngine.from_engine_args(engine_args=engine_args) |
| 106 | |
| 107 | self.default_sampling_params = SamplingParams(max_tokens=self.llm_engine.cfg.model_config.max_model_len) |
| 108 | |
| 109 | self.llm_engine.start() |
| 110 | |
| 111 | self.mutex = threading.Lock() |
| 112 | self.req_output = dict() |
| 113 | self.master_node_ip = self.llm_engine.cfg.master_ip |
| 114 | self._receive_output_thread = threading.Thread(target=self._receive_output, daemon=True) |
no outgoing calls