(self, sample: List[str])
| 137 | return output |
| 138 | |
| 139 | def format(self, sample: List[str]) -> Dict[str, Any]: |
| 140 | context = sample |
| 141 | context = [get_special_prompt(i + (len(context) + 1) % 2, self.opt) + s for i, s in enumerate(context)] |
| 142 | context_vec = self.tokenizer.encode(get_model_prompt(context, self.tokenizer.eos_token, self.opt), add_special_tokens=True) |
| 143 | |
| 144 | # truncate to max_len |
| 145 | while len(context_vec) > self.opt.maxlen_prompt - self.opt.maxlen_res and len(context) > 1: |
| 146 | context = context[1:] |
| 147 | context_vec = self.tokenizer.encode(get_model_prompt(context, self.tokenizer.eos_token, self.opt), add_special_tokens=True) |
| 148 | |
| 149 | output = { |
| 150 | 'text': self.tokenizer.decode(context_vec, skip_special_tokens=False), |
| 151 | 'text_vec': context_vec |
| 152 | } |
| 153 | |
| 154 | return output |
| 155 | |
| 156 | # batchify for single format(sample) |
| 157 | def batchify(self, batch_samples: List[Dict[str, Any]]) -> Dict[str, Any]: |
nothing calls this directly
no test coverage detected