Truncates a text string based on max number of tokens.
(string: str, encoding_name: str)
| 295 | return string |
| 296 | |
| 297 | def truncate_tokens_hf(string: str, encoding_name: str) -> str: |
| 298 | """Truncates a text string based on max number of tokens.""" |
| 299 | tokenizer = AutoTokenizer.from_pretrained(encoding_name) |
| 300 | max_tokens = tokenizer.model_max_length |
| 301 | encoded_string = tokenizer.encode(string, return_tensors="pt") |
| 302 | num_tokens = len(encoded_string[0]) |
| 303 | |
| 304 | if num_tokens > max_tokens: |
| 305 | string = tokenizer.decode(encoded_string[0][:max_tokens-400]) |
| 306 | |
| 307 | return string |
| 308 | |
| 309 | def find_non_utf8_files(path): |
| 310 | """ |
nothing calls this directly
no outgoing calls
no test coverage detected