(self, tokenizer_dir: str | Path)
| 38 | """Thin wrapper around the HuggingFace ``tokenizers`` library.""" |
| 39 | |
| 40 | def __init__(self, tokenizer_dir: str | Path): |
| 41 | from tokenizers import Tokenizer as HFTokenizer |
| 42 | |
| 43 | tokenizer_path = Path(tokenizer_dir) / "tokenizer.json" |
| 44 | if not tokenizer_path.exists(): |
| 45 | raise FileNotFoundError(f"tokenizer.json not found in {tokenizer_dir}") |
| 46 | self._tok = HFTokenizer.from_file(str(tokenizer_path)) |
| 47 | log.info("Tokenizer loaded from %s (vocab=%d)", tokenizer_path, self._tok.get_vocab_size()) |
| 48 | |
| 49 | def encode(self, text: str) -> list[int]: |
| 50 | return self._tok.encode(text).ids |
nothing calls this directly
no outgoing calls
no test coverage detected