(file_paths)
| 20 | |
| 21 | |
| 22 | def process_wiki(file_paths): |
| 23 | data = [] |
| 24 | for file_path in tqdm(file_paths, desc="Processing wiki files"): |
| 25 | with bz2.open(file_path, "rb") as file: |
| 26 | for line in file: |
| 27 | line_decoded = _normalize(line.decode('utf-8')) |
| 28 | page_data = json.loads(line_decoded) |
| 29 | title = page_data["title"] |
| 30 | |
| 31 | try: |
| 32 | abstract = "".join(page_data["text"][1]) |
| 33 | text = " ".join(["".join(page_data["text"][i]) for i in range(len(page_data["text"]))]) |
| 34 | abs_hyperlink, full_hyperlink = get_hyperlink(text, abstract) |
| 35 | |
| 36 | new_page_data = { |
| 37 | "title": title, |
| 38 | "abs_hyperlink": abs_hyperlink, |
| 39 | "full_hyperlink": full_hyperlink, |
| 40 | } |
| 41 | data.append(new_page_data) |
| 42 | except: |
| 43 | continue |
| 44 | |
| 45 | return data |
| 46 | |
| 47 | |
| 48 | def get_degree_dict(): |
nothing calls this directly
no test coverage detected