(examples)
| 146 | tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base") |
| 147 | |
| 148 | def tokenize_function(examples): |
| 149 | with CaptureLogger(tok_logger) as cl: |
| 150 | output = tokenizer(examples[text_column_name], return_token_type_ids=False) |
| 151 | # clm input could be much much longer than block_size |
| 152 | if "Token indices sequence length is longer than the" in cl.out: |
| 153 | tok_logger.warning( |
| 154 | "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits" |
| 155 | " before being passed to the model." |
| 156 | ) |
| 157 | return output |
| 158 | |
| 159 | with training_args.main_process_first(desc="dataset map tokenization"): |
| 160 | tokenized_datasets = raw_datasets.map( |
nothing calls this directly
no outgoing calls
no test coverage detected