Function load_tokenizer

tokendagger/wrapper.py:333–355

Load a tokenizer from files. Args: name: Name of the tokenizer vocab_file: Path to vocabulary file pattern: Regex pattern for text splitting special_tokens_file: Optional path to special tokens file Returns: Initialized Tokenizer instance

(
    name: str,
    vocab_file: str | Path,
    pattern: str,
    special_tokens_file: str | Path | None = None,
)

# ====================

def load_tokenizer(
    name: str,
    vocab_file: str | Path,
    pattern: str,
    special_tokens_file: str | Path | None = None,
) -> Tokenizer:
    """Load a tokenizer from files.
    
    Args:
        name: Name of the tokenizer
        vocab_file: Path to vocabulary file
        pattern: Regex pattern for text splitting
        special_tokens_file: Optional path to special tokens file
        
    Returns:
        Initialized Tokenizer instance
    """
    return Tokenizer(
        name=name,
        pattern=pattern,
        vocab_file=vocab_file,
        special_tokens_file=special_tokens_file,
    )


def create_tokenizer(