(model, tokenizer,
query: str, history: List[Tuple[str, str]] = None,
max_length: int = 1024, num_beams=1, top_p=0.7, top_k=0, temperature=0.95)
| 68 | return response |
| 69 | |
| 70 | def chat(model, tokenizer, |
| 71 | query: str, history: List[Tuple[str, str]] = None, |
| 72 | max_length: int = 1024, num_beams=1, top_p=0.7, top_k=0, temperature=0.95): |
| 73 | if not history: |
| 74 | history = [] |
| 75 | prompt = "" |
| 76 | for i, (old_query, response) in enumerate(history): |
| 77 | prompt += "[Round {}]\n问：{}\n答：{}\n".format(i, old_query, response) |
| 78 | prompt += "[Round {}]\n问：{}\n答：".format(len(history), query) |
| 79 | # --------------- |
| 80 | # tokenizer, this is an example of huggingface tokenizer. |
| 81 | # input str, output['input_ids'] = tensor([[tokenized str, gmask, sop]]) |
| 82 | inputs = tokenizer([prompt], return_tensors="pt").to(model.parameters().__next__().device)['input_ids'][0] |
| 83 | # --------------- |
| 84 | # Next, we manually set the format to keep flexibility. |
| 85 | mask_position = len(inputs) - 2 |
| 86 | context_length = len(inputs) - 1 # all before sop |
| 87 | get_func = partial(get_masks_and_position_ids_glm, mask_position=mask_position, context_length=context_length) |
| 88 | seq = torch.cat( |
| 89 | [inputs, torch.tensor([-1]*(max_length-len(inputs)), device=inputs.device)], dim=0 |
| 90 | ) |
| 91 | # --------------- |
| 92 | strategy = BaseStrategy(temperature=temperature, top_p=top_p, top_k=0, end_tokens=[tokenizer.eos_token_id]) |
| 93 | strategy = BeamSearchStrategy(temperature=temperature, top_p=top_p, top_k=0, end_tokens=[tokenizer.eos_token_id], num_beams=num_beams, consider_end=True) |
| 94 | output = filling_sequence( |
| 95 | model, seq, |
| 96 | batch_size=1, |
| 97 | get_masks_and_position_ids=get_func, |
| 98 | strategy=strategy |
| 99 | )[0] # drop memory |
| 100 | |
| 101 | # --------------- |
| 102 | # port from inference_glm.py, more general than chat mode |
| 103 | # clip -1s and fill back generated things into seq |
| 104 | output_list = list(output) |
| 105 | for i in range(len(output_list)): |
| 106 | output = list(output_list[i]) |
| 107 | try: |
| 108 | unfinished = output.index(-1) |
| 109 | except ValueError: |
| 110 | unfinished = len(output) |
| 111 | if output[unfinished - 1] == tokenizer.eos_token_id: |
| 112 | unfinished -= 1 |
| 113 | bog = output.index(tokenizer.bos_token_id) |
| 114 | output_list[i] = output[:mask_position] + output[bog + 1:unfinished] + output[mask_position + 1:bog] |
| 115 | # --------------- |
| 116 | |
| 117 | response = tokenizer.decode(output_list[0]) |
| 118 | response = process_response(response).split('答：')[-1].strip() |
| 119 | history = history + [(query, response)] |
| 120 | return response, history |
| 121 | |
| 122 | if __name__ == "__main__": |
| 123 | import argparse |
no test coverage detected