Function decode_whitespaces

codegeex/megatron/tokenizer/tokenizer.py:37–49

Decode the whitespace-encoded strings produced by encode_whitespace. >>> text = 'a\\n b\\n c' >>> s, l = 10, 10 >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l) True

(text: str, start_extra_id: int=10, max_len: int=10)

def decode_whitespaces(text: str, start_extra_id: int=10, max_len: int=10):
    """Decode the whitespace-encoded strings produced by encode_whitespace.

    >>> text = 'a\\n  b\\n   c'
    >>> s, l = 10, 10
    >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l)
    True
    """
    for l in range(2, max_len + 1):
        token_id = start_extra_id - 2 + l
        token = f"<|extratoken_{token_id}|>"
        text = text.replace(token, " " * l)
    return text


def build_hgf_tokenizer(args):