Method tokenize_words

walrus/methods/text_data/text_tokenizer.py:45–48

(self, text)

        return self.SENTENCE_TOKENIZER_FUNCTIONS.get(self.type)

    def tokenize_words(self, text) -> list:
        tokenize_words = self.get_word_tokenizer_function()
        result = tokenize_words(text)
        return result

    def tokenize_sentences(self, text) -> list:
        tokenize_senteces = self.get_sentence_tokenizer_function()