(self, config, tokenizer, text_processor)
| 260 | return config |
| 261 | |
| 262 | def __init__(self, config, tokenizer, text_processor): |
| 263 | self.config = self.get_default_config(config) |
| 264 | name = self.config.name if self.config.name != '' else None |
| 265 | split = self.config.split if self.config.split != '' else None |
| 266 | self._tokenizer = tokenizer |
| 267 | self._text_processor = text_processor |
| 268 | self._dataset = load_dataset( |
| 269 | self.config.path, name, split=split, streaming=self.config.streaming |
| 270 | ) |
| 271 | |
| 272 | def __iter__(self): |
| 273 | chunk_size = self.config.batch_size * self.config.seq_length |
nothing calls this directly
no test coverage detected