()
| 21 | |
| 22 | |
| 23 | def truncate_data(): |
| 24 | # relation_path="/cpfs04/user/zhangyaoze/workspace/trag/processed_data/relation.jsonl" |
| 25 | # relation_output_path="/cpfs04/user/zhangyaoze/workspace/trag/late_data/relation.jsonl" |
| 26 | # entity_path="/cpfs04/user/zhangyaoze/workspace/trag/processed_data/entity.jsonl" |
| 27 | # entity_output_path="/cpfs04/user/zhangyaoze/workspace/trag/late_data/entity.jsonl" |
| 28 | relation_path="processed_data/relation.jsonl" |
| 29 | relation_output_path="data/relation.jsonl" |
| 30 | entity_path="processed_data/entity.jsonl" |
| 31 | entity_output_path="data/entity.jsonl" |
| 32 | res=[] |
| 33 | i=0 |
| 34 | with open(relation_path,"r") as f: |
| 35 | for uline in f: |
| 36 | line=json.loads(uline) |
| 37 | res.append(line) |
| 38 | i+=1 |
| 39 | if i==20000: |
| 40 | break |
| 41 | write_jsonl(res,relation_output_path) |
| 42 | |
| 43 | res=[] |
| 44 | i=0 |
| 45 | with open(entity_path,"r") as f: |
| 46 | for uline in f: |
| 47 | line=json.loads(uline) |
| 48 | if "wtr20" in line['source_id']: |
| 49 | res.append(line) |
| 50 | i+=1 |
| 51 | if i==20000: |
| 52 | break |
| 53 | write_jsonl(res,entity_output_path) |
| 54 | |
| 55 | def deal_duplicate_entity(working_dir,output_path): |
| 56 | relation_path=f"{working_dir}/relation.jsonl" |
nothing calls this directly
no test coverage detected