(string)
| 123 | |
| 124 | |
| 125 | def general_detokenize(string): |
| 126 | string = string.replace(" n't", "n't") |
| 127 | string = string.replace(" )", ")") |
| 128 | string = string.replace("( ", "(") |
| 129 | string = string.replace('" ', '"') |
| 130 | string = string.replace(' "', '"') |
| 131 | string = re.sub(r" (['.,])", r"\1", string) |
| 132 | return string |
| 133 | |
| 134 | |
| 135 | def get_rolling_token_windows(token_list, prefix_token, max_seq_len, context_len): |
no outgoing calls
no test coverage detected