(self, collection_path, dataset)
| 99 | |
| 100 | class StreamIndexDataset(IterableDataset): |
| 101 | def __init__(self, collection_path, dataset): |
| 102 | super().__init__() |
| 103 | self.collection_path = collection_path |
| 104 | self.dataset = dataset |
| 105 | self.collection = [] |
| 106 | if self.dataset == 'topiocqa' or self.dataset == 'qrecc': |
| 107 | with jsonlines.open(collection_path) as fin: |
| 108 | for ln in fin: |
| 109 | self.collection.append([ln['id'], ln['contents']]) |
| 110 | else: |
| 111 | raise NotImplementedError |
| 112 | |
| 113 | def __len__(self): |
| 114 | return len(self.collection) |