(self, tokenizer, query: str, history: List[Dict] = None, role: str = "user",
max_length: int = 8192, num_beams=1, do_sample=True, top_p=0.8, temperature=0.8, logits_processor=None,
**kwargs)
| 1019 | |
| 1020 | @torch.inference_mode() |
| 1021 | def chat(self, tokenizer, query: str, history: List[Dict] = None, role: str = "user", |
| 1022 | max_length: int = 8192, num_beams=1, do_sample=True, top_p=0.8, temperature=0.8, logits_processor=None, |
| 1023 | **kwargs): |
| 1024 | if history is None: |
| 1025 | history = [] |
| 1026 | if logits_processor is None: |
| 1027 | logits_processor = LogitsProcessorList() |
| 1028 | logits_processor.append(InvalidScoreLogitsProcessor()) |
| 1029 | gen_kwargs = {"max_length": max_length, "num_beams": num_beams, "do_sample": do_sample, "top_p": top_p, |
| 1030 | "temperature": temperature, "logits_processor": logits_processor, **kwargs} |
| 1031 | inputs = tokenizer.build_chat_input(query, history=history, role=role) |
| 1032 | inputs = inputs.to(self.device) |
| 1033 | eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command("<|user|>"), |
| 1034 | tokenizer.get_command("<|observation|>")] |
| 1035 | outputs = self.generate(**inputs, **gen_kwargs, eos_token_id=eos_token_id) |
| 1036 | outputs = outputs.tolist()[0][len(inputs["input_ids"][0]):-1] |
| 1037 | response = tokenizer.decode(outputs) |
| 1038 | history.append({"role": role, "content": query}) |
| 1039 | response, history = self.process_response(response, history) |
| 1040 | return response, history |
| 1041 | |
| 1042 | @torch.inference_mode() |
| 1043 | def stream_chat(self, tokenizer, query: str, history: List[Dict] = None, role: str = "user", |
nothing calls this directly
no test coverage detected