Create a tokenizer from in-memory data. Args: name: Name of the tokenizer pattern: Regex pattern for text splitting vocab: Vocabulary items special_tokens: Special tokens mapping Returns: Initialized Tokenizer instance
(
name: str,
pattern: str,
vocab: list[dict],
special_tokens: dict[str, int] | None = None,
)
| 356 | |
| 357 | |
| 358 | def create_tokenizer( |
| 359 | name: str, |
| 360 | pattern: str, |
| 361 | vocab: list[dict], |
| 362 | special_tokens: dict[str, int] | None = None, |
| 363 | ) -> Tokenizer: |
| 364 | """Create a tokenizer from in-memory data. |
| 365 | |
| 366 | Args: |
| 367 | name: Name of the tokenizer |
| 368 | pattern: Regex pattern for text splitting |
| 369 | vocab: Vocabulary items |
| 370 | special_tokens: Special tokens mapping |
| 371 | |
| 372 | Returns: |
| 373 | Initialized Tokenizer instance |
| 374 | """ |
| 375 | return Tokenizer( |
| 376 | name=name, |
| 377 | pattern=pattern, |
| 378 | vocab=vocab, |
| 379 | special_tokens=special_tokens, |
| 380 | ) |
| 381 | |
| 382 | def Encoding( |
| 383 | name: str, |