interface
(self, file_path, cand_dic, is_train=True)
| 82 | self.type_label_map_reverse = {} |
| 83 | |
| 84 | def interface(self, file_path, cand_dic, is_train=True): |
| 85 | """interface""" |
| 86 | out_file = open( |
| 87 | './generated/' + |
| 88 | file_path.replace('.json', '') + '.txt', 'wb', |
| 89 | ) |
| 90 | out_file.write( |
| 91 | 'qid\tqid_text\ttext_a\ttext_b\ttext_c\tlabel\ttype\tent_id_b\tent_id_c\n', |
| 92 | ) |
| 93 | qid_mention = 1 |
| 94 | for line in open('./basic_data/' + file_path): |
| 95 | line_json = json.loads(line.strip()) |
| 96 | text_id = line_json.get('text_id') |
| 97 | query = line_json.get('text') |
| 98 | mention_data = line_json.get('mention_data') |
| 99 | for item in mention_data: |
| 100 | mention = item.get('mention') |
| 101 | if is_train: |
| 102 | kb_id = item.get('kb_id') |
| 103 | offset = item.get('offset') |
| 104 | if mention not in cand_dic: |
| 105 | continue |
| 106 | cand = cand_dic[mention] |
| 107 | iid_list = cand['iid_list'] |
| 108 | if 'NIL' in kb_id: |
| 109 | golden_desc = mention |
| 110 | if '|' in kb_id: |
| 111 | kb_id = kb_id.split('|')[0] |
| 112 | golden_type = kb_id.replace('NIL_', '') |
| 113 | kb_id = 'NIL' |
| 114 | else: |
| 115 | golden_desc = cand[kb_id]['ent_desc'] |
| 116 | golden_desc = golden_desc.replace('\015', '') |
| 117 | golden_type = cand[kb_id]['type'] |
| 118 | if '|' in golden_type: |
| 119 | golden_type = golden_type.split('|')[0] |
| 120 | if golden_type.decode('utf8') not in self.type_label_map: |
| 121 | self.type_label_map[golden_type.decode( |
| 122 | 'utf8', |
| 123 | )] = self.type_num |
| 124 | self.type_num += 1 |
| 125 | else: |
| 126 | if mention not in cand_dic: |
| 127 | cand = {} |
| 128 | iid_list = [] |
| 129 | else: |
| 130 | cand = cand_dic[mention] |
| 131 | iid_list = cand['iid_list'] |
| 132 | cand['NIL'] = {} |
| 133 | cand['NIL']['ent_desc'] = mention |
| 134 | iid_list.append('NIL') |
| 135 | iid_list = list(set(iid_list)) |
| 136 | |
| 137 | for iid in iid_list: |
| 138 | tmp_desc = cand[iid]['ent_desc'] |
| 139 | tmp_desc = tmp_desc.replace('\015', '') |
| 140 | if not is_train: |
| 141 | out_file.write( |