Validate that text doesn't exceed token limit when encoded.
(text: str, max_len: int = 512)
| 118 | |
| 119 | |
| 120 | def validate_token_length(text: str, max_len: int = 512) -> bool: |
| 121 | """ |
| 122 | Validate that text doesn't exceed token limit when encoded. |
| 123 | """ |
| 124 | tokens = safe_tokenize(text, add_special_tokens=True) |
| 125 | return len(tokens) <= max_len |
| 126 | |
| 127 | |
| 128 | def load_queries_robust(dataset: str, save_dir: str) -> dict: |
no test coverage detected