Method __init__

data_utils/corpora.py:365–369

(self, *args, **kwargs)

    assert_str = "make sure to set PATH for openwebtext data_utils/corpora.py"

    def __init__(self, *args, **kwargs):
        import fasttext
        super().__init__(*args, **kwargs)
        self.model = fasttext.load_model('/dataset/fd5061f6/english_data/lid.176.bin')
        print_rank_0("Load language detection model")

    def process_line(self, data, tokenizer, tokenize):
        text = data['text']