TikToken-compatible factory function.
(
name: str,
*,
pat_str: str,
mergeable_ranks: dict[bytes, int],
special_tokens: dict[str, int] | None = None,
)
| 380 | ) |
| 381 | |
| 382 | def Encoding( |
| 383 | name: str, |
| 384 | *, |
| 385 | pat_str: str, |
| 386 | mergeable_ranks: dict[bytes, int], |
| 387 | special_tokens: dict[str, int] | None = None, |
| 388 | ) -> Tokenizer: |
| 389 | """TikToken-compatible factory function.""" |
| 390 | return Tokenizer( |
| 391 | name=name, |
| 392 | pat_str=pat_str, |
| 393 | mergeable_ranks=mergeable_ranks, |
| 394 | special_tokens=special_tokens or {} |
| 395 | ) |