(self, data, tokenizer, tokenize)
| 261 | # answer_prefix = [] |
| 262 | |
| 263 | def process_line(self, data, tokenizer, tokenize): |
| 264 | prompts, texts = [], [] |
| 265 | ans_length = len(data.get("ans-content", "")) |
| 266 | ans_up = data.get("ans-up-num", "") |
| 267 | ans_up = int(ans_up) if ans_up else 0 |
| 268 | if ans_length > 100 or ans_up > 1000: |
| 269 | qtitle = data["q_title"] |
| 270 | qcontent = data["q-content"] |
| 271 | if qcontent is None: |
| 272 | qcontent = "" |
| 273 | qcontent = self.trim_field(qcontent, max_length=100) |
| 274 | user = data.get("user-signature", "") |
| 275 | prompt = self.qtitle_prefix + qtitle + self.qcontent_prefix + qcontent + self.user_prefix + user + self.answer_prefix |
| 276 | text = data["ans-content"] |
| 277 | prompt, text = self.process_sample(prompt, tokenizer, tokenize), self.process_sample(text, tokenizer, |
| 278 | tokenize) |
| 279 | prompts.append(prompt) |
| 280 | texts.append(text) |
| 281 | # prompt = data["q_title"] + data["q-content"] + data["user-signature"] |
| 282 | # text = data["ans-content"] |
| 283 | # prompts.append(prompt) |
| 284 | # texts.append(text) |
| 285 | return prompts, texts |
| 286 | |
| 287 | |
| 288 | class zhidao(PromptReader): |
nothing calls this directly
no test coverage detected