Encode whitespaces to extra tokens in GPT-J. >>> encode_whitespaces('a\\n b\\n c', 10, 10) 'a\\n<|extratoken_10|>b\\n<|extratoken_11|>c'
(text, start_extra_id: int, max_len: int)
| 6 | |
| 7 | |
| 8 | def encode_whitespaces(text, start_extra_id: int, max_len: int): |
| 9 | """ Encode whitespaces to extra tokens in GPT-J. |
| 10 | |
| 11 | >>> encode_whitespaces('a\\n b\\n c', 10, 10) |
| 12 | 'a\\n<|extratoken_10|>b\\n<|extratoken_11|>c' |
| 13 | """ |
| 14 | |
| 15 | def push_acc_space(acc_len: int, text: str): |
| 16 | if acc_len == 0: |
| 17 | return text |
| 18 | if acc_len == 1: |
| 19 | return text + ' ' |
| 20 | assert acc_len <= max_len, f'Max whitespace run length {max_len}, but found {acc_len}' |
| 21 | extra_id = start_extra_id - 2 + acc_len |
| 22 | extra_token = f'<|extratoken_{extra_id}|>' |
| 23 | return text + extra_token |
| 24 | |
| 25 | acc_len = 0 |
| 26 | res = '' |
| 27 | for ch in text: |
| 28 | if ch == ' ': |
| 29 | acc_len += 1 |
| 30 | if acc_len == max_len: |
| 31 | res = push_acc_space(acc_len, res) |
| 32 | acc_len = 0 |
| 33 | else: |
| 34 | res = push_acc_space(acc_len, res) |
| 35 | acc_len = 0 |
| 36 | res = res + ch |
| 37 | |
| 38 | res = push_acc_space(acc_len, res) |
| 39 | |
| 40 | return res |
| 41 | |
| 42 | |
| 43 | def decode_whitespaces(text: str, start_extra_id: int, max_len: int): |
no test coverage detected