| 11 | |
| 12 | |
| 13 | class VllmDecoder(DecoderBase): |
| 14 | def __init__( |
| 15 | self, |
| 16 | name: str, |
| 17 | dataset: str, |
| 18 | force_base_prompt: bool = False, |
| 19 | tensor_parallel_size: int = 1, |
| 20 | **kwargs |
| 21 | ) -> None: |
| 22 | super().__init__(name, **kwargs) |
| 23 | |
| 24 | kwargs = { |
| 25 | "tensor_parallel_size": tensor_parallel_size, |
| 26 | "dtype": self.dtype, |
| 27 | "trust_remote_code": self.trust_remote_code, |
| 28 | "enable_prefix_caching": True, |
| 29 | } |
| 30 | |
| 31 | self.force_base_prompt = force_base_prompt |
| 32 | self.tokenizer = AutoTokenizer.from_pretrained(self.name, use_fast=False) |
| 33 | if self.is_direct_completion(): |
| 34 | self.eos += extra_eos_for_direct_completion(dataset) |
| 35 | else: |
| 36 | self.eos += ["\n```\n"] |
| 37 | self.llm = LLM(model=name, max_model_len=2048, **kwargs) |
| 38 | |
| 39 | def is_direct_completion(self) -> bool: |
| 40 | return self.force_base_prompt or self.tokenizer.chat_template is None |
| 41 | |
| 42 | def codegen( |
| 43 | self, prompt: str, do_sample: bool = True, num_samples: int = 200 |
| 44 | ) -> List[str]: |
| 45 | if do_sample: |
| 46 | assert self.temperature > 0, "Temperature must be greater than 0!" |
| 47 | batch_size = min(self.batch_size, num_samples) |
| 48 | |
| 49 | prompt = ( |
| 50 | prompt |
| 51 | if self.is_direct_completion() |
| 52 | else make_raw_chat_prompt( |
| 53 | prompt, self.instruction_prefix, self.response_prefix, self.tokenizer |
| 54 | ) |
| 55 | ) |
| 56 | |
| 57 | vllm_outputs = self.llm.generate( |
| 58 | [prompt] * batch_size, |
| 59 | SamplingParams( |
| 60 | temperature=self.temperature, |
| 61 | max_tokens=self.max_new_tokens, |
| 62 | top_p=0.95 if do_sample else 1.0, |
| 63 | stop=self.eos, |
| 64 | ), |
| 65 | use_tqdm=False, |
| 66 | ) |
| 67 | |
| 68 | gen_strs = [x.outputs[0].text.replace("\t", " ") for x in vllm_outputs] |
| 69 | return gen_strs |
no outgoing calls
no test coverage detected
searching dependent graphs…