| 21 | |
| 22 | |
| 23 | def load_processed_data(file_list, language='English', name='train'): |
| 24 | data = [] |
| 25 | # 读取句子和tokens |
| 26 | for file in file_list: |
| 27 | doc_data = [] |
| 28 | conll_path = r'cache_data/' + language + '/' + name + '/' + file + '.conllu' |
| 29 | with open(conll_path) as f: |
| 30 | conll_list = f.read().split('\n\n') |
| 31 | for conll in conll_list: |
| 32 | include_flag = 0 # 是否记录标记 |
| 33 | for i in range(len(conll.split('\n'))): |
| 34 | if 'sent_id = ' in conll.split('\n')[i]: |
| 35 | temp = dict() |
| 36 | temp['sent_id'] = conll.split('\n')[i].split()[-1] |
| 37 | temp['sentence'] = conll.split('\n')[i + 1][9:] |
| 38 | if len(temp['sentence'].split()) >= 5: # TODO仅保留单词数大于5的句子 |
| 39 | include_flag = 1 # 是否记录标记 |
| 40 | elif language == 'Chinese': |
| 41 | include_flag = 1 # 对于中文，因为没有分词长度，所以全部保留 |
| 42 | break |
| 43 | if include_flag: |
| 44 | temp['tokens'] = [] |
| 45 | for j in range(len(conll.split('\n')))[i + 2:]: |
| 46 | temp['tokens'].append(conll.split('\n')[j].split()[1]) |
| 47 | doc_data.append(temp) |
| 48 | |
| 49 | # 读取entity, event, relation |
| 50 | file_path = r'cache_data/' + language + '/' + name + '/' + file + '.v2.json' |
| 51 | with open(file_path) as f: |
| 52 | v2_data = json.loads(f.read()) |
| 53 | for sentence in doc_data: |
| 54 | sent_id = sentence['sent_id'] |
| 55 | sentence['golden-entity-mentions'] = [] # 添加实体 |
| 56 | sentence['golden-event-mentions'] = [] # 添加事件 |
| 57 | sentence['golden-relation-mentions'] = [] # 添加关系 |
| 58 | for entity in v2_data['entities']: |
| 59 | if entity['sent_id'] == sent_id: |
| 60 | sentence['golden-entity-mentions'].append(entity) |
| 61 | for event in v2_data['events']: |
| 62 | if event['sent_id'] == sent_id: |
| 63 | sentence['golden-event-mentions'].append(event) |
| 64 | for relation in v2_data['relations']: |
| 65 | if relation['sent_id'] == sent_id: |
| 66 | sentence['golden-relation-mentions'].append(relation) |
| 67 | del sentence['sent_id'] # 删除sent_id |
| 68 | data += doc_data |
| 69 | return data |
| 70 | |
| 71 | |
| 72 | def count_type(language='English'): |