| 8 | |
| 9 | |
| 10 | class CustomJSON(Task): |
| 11 | def __init__(self, filepath, **kwargs): |
| 12 | super().__init__(**kwargs) |
| 13 | self.filepath = filepath |
| 14 | self.conversations = [] |
| 15 | |
| 16 | if not os.path.exists(filepath): |
| 17 | print(f"Warning: {filepath} not found") |
| 18 | else: |
| 19 | with open(filepath, "r", encoding="utf-8") as f: |
| 20 | for line in f: |
| 21 | line = line.strip() |
| 22 | if not line: |
| 23 | continue |
| 24 | messages = json.loads(line) |
| 25 | assert isinstance(messages, list) and len(messages) >= 2 |
| 26 | self.conversations.append(messages) |
| 27 | |
| 28 | self.length = len(self.conversations) |
| 29 | |
| 30 | def num_examples(self): |
| 31 | return self.length |
| 32 | |
| 33 | def get_example(self, index): |
| 34 | return {"messages": self.conversations[index]} |