A class that handles the reddit data files
| 221 | |
| 222 | |
| 223 | class RedditDataset: |
| 224 | """ |
| 225 | A class that handles the reddit data files |
| 226 | """ |
| 227 | def __init__(self, filepath): |
| 228 | self.filepath = filepath |
| 229 | |
| 230 | def __iter__(self): |
| 231 | while True: |
| 232 | with gzip.open(self.filepath, "rt") as fIn: |
| 233 | for line in fIn: |
| 234 | data = json.loads(line) |
| 235 | |
| 236 | if "response" in data and "context" in data: |
| 237 | yield [data["response"], data["context"]] |
| 238 | |
| 239 | class Dataset: |
| 240 | """ |