(examples)
| 75 | def tokenize_and_group_text(raw_datasets, tokenizer, block_size, stride, padding_index, num_process): |
| 76 | |
| 77 | def tokenize_function(examples): |
| 78 | output = tokenizer(examples["text"]) |
| 79 | return output |
| 80 | |
| 81 | def group_texts(examples): |
| 82 | # Concatenate all texts. |
nothing calls this directly
no outgoing calls
no test coverage detected