Safely tokenize text without warnings.
(text: str, add_special_tokens: bool = True)
| 89 | |
| 90 | |
| 91 | def safe_tokenize(text: str, add_special_tokens: bool = True) -> List[int]: |
| 92 | """ |
| 93 | Safely tokenize text without warnings. |
| 94 | """ |
| 95 | with warnings.catch_warnings(): |
| 96 | warnings.simplefilter("ignore") |
| 97 | return get_tokenizer().encode( |
| 98 | text, add_special_tokens=add_special_tokens, truncation=True, max_length=512 |
| 99 | ) |
| 100 | |
| 101 | |
| 102 | def clip_to_max_tokens(text: str, max_len: int = 512) -> str: |
no test coverage detected