| 11 | |
| 12 | |
| 13 | class GraphDataModule(pl.LightningDataModule): |
| 14 | |
| 15 | def __init__(self, |
| 16 | tokenizer_class, |
| 17 | tokenizer_name, |
| 18 | cache_dir, |
| 19 | data_path, |
| 20 | dataset, |
| 21 | batch_size, |
| 22 | num_data_workers, |
| 23 | max_nodes, |
| 24 | max_edges, |
| 25 | edges_as_classes): |
| 26 | super().__init__() |
| 27 | |
| 28 | self.tokenizer = tokenizer_class.from_pretrained(tokenizer_name, cache_dir=cache_dir) |
| 29 | self.tokenizer.add_tokens('__no_node__') |
| 30 | self.tokenizer.add_tokens('__no_edge__') |
| 31 | self.tokenizer.add_tokens('__node_sep__') |
| 32 | |
| 33 | self.batch_size = batch_size |
| 34 | self.num_data_workers = num_data_workers |
| 35 | self.data_path = data_path |
| 36 | self.max_nodes = max_nodes |
| 37 | self.max_edges = max_edges |
| 38 | self.edges_as_classes = edges_as_classes |
| 39 | self.output_path = os.path.join(data_path, 'processed') |
| 40 | os.makedirs(self.output_path, exist_ok=True) |
| 41 | self.dataset = dataset |
| 42 | |
| 43 | def prepareWebNLG(self): |
| 44 | |
| 45 | splits = ['train', 'dev', 'test'] |
| 46 | |
| 47 | for split in splits: |
| 48 | |
| 49 | text_file = os.path.join(self.output_path, f'{split}.text') |
| 50 | graph_file = os.path.join(self.output_path, f'{split}.graph') |
| 51 | if os.path.exists(text_file) and os.path.exists(graph_file): |
| 52 | continue |
| 53 | |
| 54 | b = Benchmark() |
| 55 | if split == 'test': |
| 56 | files = [(os.path.join(self.data_path, split), 'semantic-parsing-test-data-with-refs-en.xml')] |
| 57 | else: |
| 58 | files = select_files(os.path.join(self.data_path, split)) |
| 59 | b.fill_benchmark(files) |
| 60 | b.b2json(self.output_path, f'{split}.json') |
| 61 | D = json.load(open(os.path.join(self.output_path, f'{split}.json'))) |
| 62 | |
| 63 | normalize = lambda text: ud.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii') |
| 64 | |
| 65 | triples_to_write = [] |
| 66 | text_to_write = [] |
| 67 | edge_classes_to_write = [] |
| 68 | for ind, entry in enumerate(D['entries']): |
| 69 | triples = entry[str(ind + 1)]['modifiedtripleset'] |
| 70 | proc_triples = [] |