Function process_wiki

preprocess/process_wiki_page.py:23–47

(file_paths)

def process_wiki(file_paths):
    data = []
    for file_path in tqdm(file_paths, desc="Processing wiki files"):
        with bz2.open(file_path, "rb") as file:
            for line in file:
                line_decoded = _normalize(line.decode('utf-8'))
                page_data = json.loads(line_decoded)
                title = convert_html(page_data["title"])
                abs_hyperlink, full_hyperlink = get_hyperlink(page_data["text"])

                new_page_data = {"title": title,
                                 "url": page_data["url"],
                                 "text": remove_hyperlink(page_data["text"], abstract=False),
                                 "abs_hyperlink": abs_hyperlink,
                                 "full_hyperlink": full_hyperlink}
                new_page_data["size"] = len(enc.encode(new_page_data["text"]))

                if title in corpus_title_set:
                    new_page_data["in_corpus"] = True
                else:
                    new_page_data["in_corpus"] = False

                data.append(new_page_data)

    return data


def get_degree_dict():