Decode the whitespace-encoded strings produced by encode_whitespace. >>> text = 'a\\n b\\n c' >>> s, l = 10, 10 >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l) True
(text: str, start_extra_id: int, max_len: int)
| 41 | |
| 42 | |
| 43 | def decode_whitespaces(text: str, start_extra_id: int, max_len: int): |
| 44 | """ Decode the whitespace-encoded strings produced by encode_whitespace. |
| 45 | |
| 46 | >>> text = 'a\\n b\\n c' |
| 47 | >>> s, l = 10, 10 |
| 48 | >>> text == decode_whitespaces(encode_whitespaces(text, s, l), s, l) |
| 49 | True |
| 50 | """ |
| 51 | for l in range(2, max_len + 1): |
| 52 | token_id = start_extra_id - 2 + l |
| 53 | token = f'<|extratoken_{token_id}|>' |
| 54 | text = text.replace(token, ' ' * l) |
| 55 | return text |
| 56 | |
| 57 | |
| 58 | class Code13BDictionary(object): |