Class CorpusProcess

data_processor.py:97–149

class CorpusProcess:
    def __init__(self, dataset, encoding=None):
        corpus_path = "data/text_dataset/corpus"
        clean_corpus_path = "data/text_dataset/clean_corpus"
        if not os.path.exists(clean_corpus_path):
            os.makedirs(clean_corpus_path)

        self.dataset = dataset
        self.corpus_name = f"{corpus_path}/{dataset}.txt"
        self.save_name = f"{clean_corpus_path}/{dataset}.txt"
        self.context_dct = defaultdict(dict)

        self.encoding = encoding
        self.clean_text()

    def clean_text(self):
        sp = StringProcess()
        word_lst = list()
        with open(self.corpus_name, mode="rb", encoding=self.encoding) as fin:
            for indx, item in tqdm(enumerate(fin), desc="clean the text"):
                data = item.strip().decode('latin1')
                data = sp.clean_str(data)
                if self.dataset not in {"mr"}:
                    data = sp.remove_stopword(data)
                word_lst.extend(data.split())

        word_st = set()
        if self.dataset not in {"mr"}:
            for word, value in Counter(word_lst).items():
                if value < 5:
                    continue
                word_st.add(word)
        else:
            word_st = set(word_lst)

        doc_len_lst = list()
        with open(self.save_name, mode='w') as fout:
            with open(self.corpus_name, mode="rb", encoding=self.encoding) as fin:
                for line in tqdm(fin):
                    lines_str = line.strip().decode('latin1')
                    lines_str = sp.clean_str(lines_str)
                    if self.dataset not in {"mr"}:
                        lines_str = sp.remove_stopword(lines_str)
                        lines_str = remove_less_word(lines_str, word_st)

                    fout.write(lines_str)
                    fout.write(" \n")

                    doc_len_lst.append(len(lines_str.split()))

        print("Average length:", np.mean(doc_len_lst))
        print("doc count:", len(doc_len_lst))
        print("Total number of words:", len(word_st))


def main():