generate param
(
self, generate_configs: GenerateConfigs,
)
| 51 | # return [i.outputs[0].text for i in outputs] |
| 52 | |
| 53 | def generate_params( |
| 54 | self, generate_configs: GenerateConfigs, |
| 55 | ): |
| 56 | '''generate param''' |
| 57 | kargs = generate_configs.dict() |
| 58 | params = { |
| 59 | "max_new_tokens": kargs.get("max_new_tokens", 128), |
| 60 | "top_k": kargs.get("top_k", 50), |
| 61 | "top_p": kargs.get("top_p", 0.95), |
| 62 | "temperature": kargs.get("temperature", 1.0), |
| 63 | } |
| 64 | |
| 65 | # params = { |
| 66 | # "n": 1, |
| 67 | # "max_tokens": kargs.get("max_new_tokens", 128), |
| 68 | # "best_of": kargs.get("beam_bums", 1), |
| 69 | # "top_k": kargs.get("top_k", 50), |
| 70 | # "top_p": kargs.get("top_p", 0.95), |
| 71 | # "temperature": kargs.get("temperature", 1.0), |
| 72 | # "length_penalty": kargs.get("length_penalty", 1.0), |
| 73 | # "presence_penalty": kargs.get("presence_penalty", 1.0), |
| 74 | # "stop": kargs.get("stop_words", ["<|endoftext|>"]), |
| 75 | # } |
| 76 | return params |
| 77 | |
| 78 | def load_model(self, model_path, peft_path=None, trust_remote_code=True, tensor_parallel_size=1, gpu_memory_utilization=0.25): |
| 79 | '''加载模型''' |