Function decode_whitespaces

codegeex/tokenizer/tokenizer.py:18–30

Decode the whitespace-encoded strings produced by encode_whitespace. >>> text = 'a\\n b\\n c' >>> s, l = 10, 10 >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l) True

(text: str, start_extra_id: int, max_len: int)

def decode_whitespaces(text: str, start_extra_id: int, max_len: int):
    """ Decode the whitespace-encoded strings produced by encode_whitespace.

    >>> text = 'a\\n  b\\n   c'
    >>> s, l = 10, 10
    >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l)
    True
    """
    for l in range(2, max_len + 1):
        token_id = start_extra_id - 2 + l
        token = f'<|extratoken_{token_id}|>'
        text = text.replace(token, ' ' * l)
    return text

    
class CodeGeeXTokenizer(object):