| 92 | |
| 93 | |
| 94 | def process_data(opt, model, filenames, split): |
| 95 | outdir = os.path.join(opt.output, split) |
| 96 | Path(outdir).mkdir(parents=True, exist_ok=True) |
| 97 | total_entities = 0 |
| 98 | total_events = 0 |
| 99 | total_relations = 0 |
| 100 | total_event_arguments = 0 |
| 101 | total_sentences = 0 |
| 102 | total_words = 0 |
| 103 | for filename in tqdm(filenames, total=len(filenames)): |
| 104 | outfile = os.path.split(filename)[-1] |
| 105 | conllu_file = os.path.join(outdir, '{}.conllu'.format(outfile)) |
| 106 | with open(conllu_file, 'w') as fw: |
| 107 | conllu, num_sent, num_words = parse_sgm(model, filename) |
| 108 | total_sentences += num_sent |
| 109 | total_words += num_words |
| 110 | fw.write(conllu) |
| 111 | |
| 112 | json_file = os.path.join(outdir, '{}.v1.json'.format(outfile)) |
| 113 | with open(json_file, 'w') as fw: |
| 114 | jsonobj = parse_xml(filename) |
| 115 | total_entities += len(jsonobj['entities']) |
| 116 | total_events += len(jsonobj['events']) |
| 117 | total_relations += len(jsonobj['relations']) |
| 118 | total_event_arguments += sum([len(em['arguments']) for em in jsonobj['events']]) |
| 119 | json.dump(jsonobj, fw) |
| 120 | |
| 121 | return { |
| 122 | 'total_files': len(filenames), |
| 123 | 'total_sentences': total_sentences, |
| 124 | 'total_words': total_words, |
| 125 | 'total_entities': total_entities, |
| 126 | 'total_events': total_events, |
| 127 | 'total_relations': total_relations, |
| 128 | 'total_event_arguments': total_event_arguments |
| 129 | } |
| 130 | |
| 131 | |
| 132 | def main(args): |