| 71 | |
| 72 | # bloomz |
| 73 | def plain_chat( |
| 74 | prompt, |
| 75 | input=None, |
| 76 | temperature=0.7, |
| 77 | top_p=0.75, |
| 78 | top_k=40, |
| 79 | num_beams=4, |
| 80 | max_new_tokens=512, |
| 81 | **kwargs, |
| 82 | ): |
| 83 | #print("prompt:", prompt) |
| 84 | inputs = tokenizer(prompt, return_tensors="pt") |
| 85 | input_ids = inputs["input_ids"].to(device) |
| 86 | generation_config = GenerationConfig( |
| 87 | temperature=temperature, |
| 88 | top_p=top_p, |
| 89 | top_k=top_k, |
| 90 | num_beams=num_beams, |
| 91 | **kwargs, |
| 92 | ) |
| 93 | with torch.no_grad(): |
| 94 | generation_output = model.generate( |
| 95 | input_ids=input_ids, |
| 96 | generation_config=generation_config, |
| 97 | return_dict_in_generate=True, |
| 98 | output_scores=True, |
| 99 | max_new_tokens=max_new_tokens, |
| 100 | ) |
| 101 | s = generation_output.sequences[0] |
| 102 | output = tokenizer.decode(s) |
| 103 | return output.replace(prompt, "").replace("</s>", "").strip() |
| 104 | |
| 105 | def read_file_lines(file): |
| 106 | with open(file, 'r', errors='ignore', encoding='utf8') as f: |