产出对应结果
(
self, prompts: str,
template: str = None,
generate_configs: GenerateConfigs =None,
)
| 22 | self.load_model(self.model_path, self.peft_path, self.trust_remote_code, self.tensor_parallel_size, self.gpu_memory_utilization) |
| 23 | |
| 24 | def generate( |
| 25 | self, prompts: str, |
| 26 | template: str = None, |
| 27 | generate_configs: GenerateConfigs =None, |
| 28 | ) -> list: |
| 29 | '''产出对应结果''' |
| 30 | |
| 31 | template = self.template if template is None else template |
| 32 | |
| 33 | params = self.generate_params(generate_configs) |
| 34 | |
| 35 | if template == "default": |
| 36 | inputs = self.tokenizer(prompts, return_tensors="pt") |
| 37 | inputs["input_ids"] = inputs["input_ids"].cuda() |
| 38 | inputs["attention_mask"] = inputs["attention_mask"].cuda() |
| 39 | |
| 40 | inputs.update(params) |
| 41 | output = self.model.generate(**inputs) |
| 42 | predict = self.tokenizer.decode(output[0].tolist()) |
| 43 | predict = predict.split("\n")[-1] |
| 44 | predict = predict.replace("<|endoftext|>", "").replace("</s>", "") |
| 45 | return predict |
| 46 | elif template != "default": |
| 47 | output, _ = self.model.chat(self.tokenizer, prompts, history=None, **params) |
| 48 | return output |
| 49 | # params = self.generate_params(generate_configs) |
| 50 | # sampling_params = SamplingParams(**params) |
| 51 | # prompts = [prompts] if isinstance(prompts, str) else prompts |
| 52 | # outputs = self.model.generate(prompts, sampling_params) |
| 53 | # return [i.outputs[0].text for i in outputs] |
| 54 | |
| 55 | def generate_params( |
| 56 | self, generate_configs: GenerateConfigs, |
nothing calls this directly
no test coverage detected