(tokens, sentence, separator=' ')
| 8 | |
| 9 | # 通过span获取到具体的文本内容(start, end位置) |
| 10 | def get_str_from_tokens(tokens, sentence, separator=' '): |
| 11 | start, end_exclude = tokens[0], tokens[-1] + 1 |
| 12 | return separator.join(sentence[start:end_exclude]) |
| 13 | |
| 14 | # T5 tokenizer使用的特殊符号 |
| 15 | type_start = '<extra_id_0>' |
no outgoing calls
no test coverage detected