| 41 | self.dataset = dataset |
| 42 | |
| 43 | def prepareWebNLG(self): |
| 44 | |
| 45 | splits = ['train', 'dev', 'test'] |
| 46 | |
| 47 | for split in splits: |
| 48 | |
| 49 | text_file = os.path.join(self.output_path, f'{split}.text') |
| 50 | graph_file = os.path.join(self.output_path, f'{split}.graph') |
| 51 | if os.path.exists(text_file) and os.path.exists(graph_file): |
| 52 | continue |
| 53 | |
| 54 | b = Benchmark() |
| 55 | if split == 'test': |
| 56 | files = [(os.path.join(self.data_path, split), 'semantic-parsing-test-data-with-refs-en.xml')] |
| 57 | else: |
| 58 | files = select_files(os.path.join(self.data_path, split)) |
| 59 | b.fill_benchmark(files) |
| 60 | b.b2json(self.output_path, f'{split}.json') |
| 61 | D = json.load(open(os.path.join(self.output_path, f'{split}.json'))) |
| 62 | |
| 63 | normalize = lambda text: ud.normalize('NFKD', text).encode('ascii', 'ignore').decode('ascii') |
| 64 | |
| 65 | triples_to_write = [] |
| 66 | text_to_write = [] |
| 67 | edge_classes_to_write = [] |
| 68 | for ind, entry in enumerate(D['entries']): |
| 69 | triples = entry[str(ind + 1)]['modifiedtripleset'] |
| 70 | proc_triples = [] |
| 71 | for triple in triples: |
| 72 | obj, rel, sub = triple['object'], triple['property'], triple['subject'] |
| 73 | obj = normalize(obj.strip('\"').replace('_', ' ')) |
| 74 | sub = normalize(sub.strip('\"').replace('_', ' ')) |
| 75 | proc_triples.append(f'__subject__ {sub} __predicate__ {rel} __object__ {obj}') |
| 76 | if split == 'train': |
| 77 | edge_classes_to_write.append(rel) |
| 78 | |
| 79 | merged_triples = ' '.join(proc_triples) |
| 80 | |
| 81 | proc_lexs = [normalize(l['lex']) for l in entry[str(ind + 1)]['lexicalisations']] |
| 82 | |
| 83 | for lex in proc_lexs: |
| 84 | text_to_write.append(lex) |
| 85 | triples_to_write.append(merged_triples) |
| 86 | |
| 87 | with open(text_file, 'w') as f: |
| 88 | f.writelines(s + '\n' for s in text_to_write) |
| 89 | |
| 90 | with open(graph_file, 'w') as f: |
| 91 | f.writelines(s + '\n' for s in triples_to_write) |
| 92 | |
| 93 | if split == 'train': |
| 94 | edge_classes_file = os.path.join(self.output_path, 'edge.classes') |
| 95 | with open(edge_classes_file, 'w') as f: |
| 96 | f.writelines(s + '\n' for s in list(set(edge_classes_to_write)) + ['__no_edge__']) |
| 97 | |
| 98 | def prepare_data(self): |
| 99 | if self.dataset == 'webnlg': |