(file_path,output_path)
| 133 | # e_dic[tgt_src]['degree']+=1 |
| 134 | write_jsonl(all_relations,relation_output_path) |
| 135 | def process_triple(file_path,output_path): |
| 136 | create_if_not_exist(output_path) |
| 137 | with open(file_path,"r") as f: |
| 138 | entities={} |
| 139 | relations=[] |
| 140 | for uline in f: |
| 141 | line=json.loads(uline) |
| 142 | triple=line['triple'].split("\t") |
| 143 | doc_name=line['doc_name'] |
| 144 | page_idx=line['page_idx'] |
| 145 | paragraph_idx=line['paragraph_idx'] |
| 146 | source_id = doc_name + "_" + str(page_idx) + "_" + str(paragraph_idx) |
| 147 | head_entity=triple[0][1:-1] |
| 148 | head_description=triple[1][1:-1] |
| 149 | relation=triple[2][1:-1] |
| 150 | relation_description=triple[3][1:-1] |
| 151 | tail_entity=triple[4][1:-1] |
| 152 | tail_description=triple[5][1:-1] |
| 153 | |
| 154 | if head_entity not in entities.keys(): |
| 155 | entities[head_entity]=dict( |
| 156 | entity_name=str(head_entity), |
| 157 | description=head_description, |
| 158 | source_id=source_id, |
| 159 | degree=0, |
| 160 | ) |
| 161 | else: |
| 162 | entities[head_entity]['description']+=" | "+ head_description |
| 163 | if entities[head_entity]['source_id']!= source_id: |
| 164 | entities[head_entity]['source_id']+= "|"+source_id |
| 165 | if tail_entity not in entities.keys(): |
| 166 | entities[tail_entity]=dict( |
| 167 | entity_name=str(tail_entity), |
| 168 | description=tail_description, |
| 169 | source_id=source_id, |
| 170 | degree=0, |
| 171 | ) |
| 172 | else: |
| 173 | entities[tail_entity]['description']+=" | "+ tail_description |
| 174 | if entities[tail_entity]['source_id']!= source_id: |
| 175 | entities[tail_entity]['source_id']+= "|"+source_id |
| 176 | relations.append(dict( |
| 177 | src_tgt=head_entity, |
| 178 | tgt_src=tail_entity, |
| 179 | source=relation, |
| 180 | description=relation_description, |
| 181 | weight=1, |
| 182 | source_id=source_id |
| 183 | )) |
| 184 | write_jsonl(relations,f"{output_path}/relation.jsonl") |
| 185 | res_entity=[] |
| 186 | tokenizer = tiktoken.get_encoding("cl100k_base") |
| 187 | to_summarize = [] |
| 188 | summary_prompt=PROMPTS['summary_entities'] |
| 189 | for k,v in entities.items(): |
| 190 | v['source_id']="|".join(set(v['source_id'].split("|"))) |
| 191 | description=v['description'] |
| 192 | tokens = len(tokenizer.encode(description)) |
nothing calls this directly
no test coverage detected