(self, paths, tables_paths, db_path, limit=None)
| 190 | @registry.register('dataset', 'spider') |
| 191 | class SpiderDataset(torch.utils.data.Dataset): |
| 192 | def __init__(self, paths, tables_paths, db_path, limit=None): |
| 193 | self.paths = paths |
| 194 | self.db_path = db_path |
| 195 | self.examples = [] |
| 196 | |
| 197 | self.schemas, self.eval_foreign_key_maps = load_tables(tables_paths) |
| 198 | |
| 199 | for path in paths: |
| 200 | raw_data = json.load(open(path)) |
| 201 | for entry in raw_data: |
| 202 | item = SpiderItem( |
| 203 | text=entry['question_toks'], |
| 204 | code=entry['sql'], |
| 205 | schema=self.schemas[entry['db_id']], |
| 206 | orig=entry, |
| 207 | orig_schema=self.schemas[entry['db_id']].orig) |
| 208 | self.examples.append(item) |
| 209 | |
| 210 | def __len__(self): |
| 211 | return len(self.examples) |
nothing calls this directly
no test coverage detected