(prompt)
| 190 | |
| 191 | |
| 192 | def get_encoded_length(prompt): |
| 193 | length_after_extensions = apply_extensions('tokenized_length', prompt) |
| 194 | if length_after_extensions is not None: |
| 195 | return length_after_extensions |
| 196 | |
| 197 | return len(encode(prompt)[0]) |
| 198 | |
| 199 | |
| 200 | def get_token_ids(prompt): |
no test coverage detected