(
self, prompt: str, do_sample: bool = True, num_samples: int = 200
)
| 40 | return self.force_base_prompt or self.tokenizer.chat_template is None |
| 41 | |
| 42 | def codegen( |
| 43 | self, prompt: str, do_sample: bool = True, num_samples: int = 200 |
| 44 | ) -> List[str]: |
| 45 | if do_sample: |
| 46 | assert self.temperature > 0, "Temperature must be greater than 0!" |
| 47 | batch_size = min(self.batch_size, num_samples) |
| 48 | |
| 49 | prompt = ( |
| 50 | prompt |
| 51 | if self.is_direct_completion() |
| 52 | else make_raw_chat_prompt( |
| 53 | prompt, self.instruction_prefix, self.response_prefix, self.tokenizer |
| 54 | ) |
| 55 | ) |
| 56 | |
| 57 | vllm_outputs = self.llm.generate( |
| 58 | [prompt] * batch_size, |
| 59 | SamplingParams( |
| 60 | temperature=self.temperature, |
| 61 | max_tokens=self.max_new_tokens, |
| 62 | top_p=0.95 if do_sample else 1.0, |
| 63 | stop=self.eos, |
| 64 | ), |
| 65 | use_tqdm=False, |
| 66 | ) |
| 67 | |
| 68 | gen_strs = [x.outputs[0].text.replace("\t", " ") for x in vllm_outputs] |
| 69 | return gen_strs |
nothing calls this directly
no test coverage detected