| 299 | if progress % 10000 == 0: |
| 300 | print(f"处理进度: {progress}") |
| 301 | def get_entities_from_wikidata(infilename, outfilename1, outfilename2, outfilename3): |
| 302 | with open(infilename, "r", encoding="utf-8") as infile, open(outfilename1, "w", encoding="utf-8") as outfile1, open(outfilename2, "w", encoding="utf-8") as outfile2, open(outfilename3, "w", encoding="utf-8") as outfile3: |
| 303 | index = 0 |
| 304 | for line in infile: # 假设顶层是一个数组try: |
| 305 | index += 1 |
| 306 | item = json.loads(line) |
| 307 | # 提取"en"字段的内容 |
| 308 | try: |
| 309 | entity = item.get("labels", {}).get("value", "") |
| 310 | aliases_dic = item.get("aliases", []) |
| 311 | aliases = [] |
| 312 | for item in aliases_dic: |
| 313 | aliases.append(item.get("value", "")) |
| 314 | descriptions = item.get("descriptions", "") |
| 315 | if descriptions != "": |
| 316 | descriptions = descriptions.get("value", "") |
| 317 | except: |
| 318 | pass |
| 319 | |
| 320 | # 将提取结果写入输出文件 |
| 321 | outfile1.write(entity + "\n") |
| 322 | outfile2.write(entity + "\n") |
| 323 | for item in aliases: |
| 324 | outfile2.write(item + "\n") |
| 325 | outfile3.write(f"<{entity}>\t<{aliases}>\t<{descriptions}>\n") |
| 326 | |
| 327 | # 输出处理进度，每处理 10,000 个对象输出一次 |
| 328 | progress = index + 1 |
| 329 | if progress % 10000 == 0: |
| 330 | print(f"处理进度: {progress}") |
| 331 | |
| 332 | def main(): |
| 333 | # infile = '/data/H-RAG/H_RAG/data/kg/wikidata-20240101-all.json' |