| 73 | |
| 74 | |
| 75 | def build_tokenizer( |
| 76 | om_tokenizer_config: DictConfig, |
| 77 | ) -> Tokenizer: |
| 78 | os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "1" |
| 79 | os.environ["TOKENIZERS_PARALLELISM"] = "false" |
| 80 | |
| 81 | resolved_om_tokenizer_config = om.to_container(om_tokenizer_config, resolve=True) |
| 82 | tokenizer_kwargs = resolved_om_tokenizer_config.get( # type: ignore |
| 83 | "kwargs", {} |
| 84 | ) |
| 85 | tokenizer_name = resolved_om_tokenizer_config["name"] # type: ignore |
| 86 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, **tokenizer_kwargs) |
| 87 | |
| 88 | # HuggingFace does not respect the model_max_length kwarg, and overrides it with |
| 89 | # min(kwargs['model_max_length'], original_config['model_max_length']), so we |
| 90 | # explicitly set it here |
| 91 | tokenizer.model_max_length = tokenizer_kwargs.get( |
| 92 | "model_max_length", |
| 93 | int(1e30), |
| 94 | ) |
| 95 | |
| 96 | return tokenizer |
| 97 | |
| 98 | |
| 99 | class StreamingTextDataset(StreamingDataset): |