Class Tokenizer

utils/tokenizers.py:129–141

Base tokenizer class. Tokenizers implement tokenize, which should return a Tokens class.

class Tokenizer(object):
    """Base tokenizer class.
    Tokenizers implement tokenize, which should return a Tokens class.
    """

    def tokenize(self, text):
        raise NotImplementedError

    def shutdown(self):
        pass

    def __del__(self):
        self.shutdown()


class SimpleTokenizer(Tokenizer):