Get tokenizer instance, loading it lazily.
()
| 47 | # Helpers |
| 48 | # ------------------------------------------------------------------ |
| 49 | def get_tokenizer(): |
| 50 | """Get tokenizer instance, loading it lazily.""" |
| 51 | global TOKENIZER |
| 52 | if TOKENIZER is None: |
| 53 | from transformers import AutoTokenizer |
| 54 | |
| 55 | TOKENIZER = AutoTokenizer.from_pretrained( |
| 56 | "Snowflake/snowflake-arctic-embed-m-v1.5" |
| 57 | ) |
| 58 | return TOKENIZER |
| 59 | |
| 60 | |
| 61 | def chunk_text_by_tokens( |
no outgoing calls
no test coverage detected