(
self,
name: str,
dataset: str,
force_base_prompt: bool = False,
tensor_parallel_size: int = 1,
**kwargs
)
| 12 | |
| 13 | class VllmDecoder(DecoderBase): |
| 14 | def __init__( |
| 15 | self, |
| 16 | name: str, |
| 17 | dataset: str, |
| 18 | force_base_prompt: bool = False, |
| 19 | tensor_parallel_size: int = 1, |
| 20 | **kwargs |
| 21 | ) -> None: |
| 22 | super().__init__(name, **kwargs) |
| 23 | |
| 24 | kwargs = { |
| 25 | "tensor_parallel_size": tensor_parallel_size, |
| 26 | "dtype": self.dtype, |
| 27 | "trust_remote_code": self.trust_remote_code, |
| 28 | "enable_prefix_caching": True, |
| 29 | } |
| 30 | |
| 31 | self.force_base_prompt = force_base_prompt |
| 32 | self.tokenizer = AutoTokenizer.from_pretrained(self.name, use_fast=False) |
| 33 | if self.is_direct_completion(): |
| 34 | self.eos += extra_eos_for_direct_completion(dataset) |
| 35 | else: |
| 36 | self.eos += ["\n```\n"] |
| 37 | self.llm = LLM(model=name, max_model_len=2048, **kwargs) |
| 38 | |
| 39 | def is_direct_completion(self) -> bool: |
| 40 | return self.force_base_prompt or self.tokenizer.chat_template is None |
nothing calls this directly
no test coverage detected