Method add

flagai/data/tokenizer/uni_tokenizer/tokenization_utils.py:14–22

(self, word: str)

        self.data = {}

    def add(self, word: str):
        if not word:
            # Prevent empty string
            return
        ref = self.data
        for char in word:
            ref[char] = char in ref and ref[char] or {}
            ref = ref[char]
        ref[""] = 1

    def split(self, text: str) -> List[str]:
        states = OrderedDict()