Get an iterator that yields batches of data from the given dataset. Args: dataset (~fairseq.data.FairseqDataset): dataset to batch max_tokens (int, optional): max number of tokens in each batch (default: None). max_sentences (int,
(
self,
dataset,
max_tokens=None,
max_sentences=None,
max_positions=None,
ignore_invalid_inputs=False,
required_batch_size_multiple=1,
seed=1,
num_shards=1,
shard_id=0,
num_workers=0,
epoch=1,
data_buffer_size=0,
disable_iterator_cache=False,
)
| 209 | return getattr(dataset, "can_reuse_epoch_itr_across_epochs", False) |
| 210 | |
| 211 | def get_batch_iterator( |
| 212 | self, |
| 213 | dataset, |
| 214 | max_tokens=None, |
| 215 | max_sentences=None, |
| 216 | max_positions=None, |
| 217 | ignore_invalid_inputs=False, |
| 218 | required_batch_size_multiple=1, |
| 219 | seed=1, |
| 220 | num_shards=1, |
| 221 | shard_id=0, |
| 222 | num_workers=0, |
| 223 | epoch=1, |
| 224 | data_buffer_size=0, |
| 225 | disable_iterator_cache=False, |
| 226 | ): |
| 227 | """ |
| 228 | Get an iterator that yields batches of data from the given dataset. |
| 229 | |
| 230 | Args: |
| 231 | dataset (~fairseq.data.FairseqDataset): dataset to batch |
| 232 | max_tokens (int, optional): max number of tokens in each batch |
| 233 | (default: None). |
| 234 | max_sentences (int, optional): max number of sentences in each |
| 235 | batch (default: None). |
| 236 | max_positions (optional): max sentence length supported by the |
| 237 | model (default: None). |
| 238 | ignore_invalid_inputs (bool, optional): don't raise Exception for |
| 239 | sentences that are too long (default: False). |
| 240 | required_batch_size_multiple (int, optional): require batch size to |
| 241 | be a multiple of N (default: 1). |
| 242 | seed (int, optional): seed for random number generator for |
| 243 | reproducibility (default: 1). |
| 244 | num_shards (int, optional): shard the data iterator into N |
| 245 | shards (default: 1). |
| 246 | shard_id (int, optional): which shard of the data iterator to |
| 247 | return (default: 0). |
| 248 | num_workers (int, optional): how many subprocesses to use for data |
| 249 | loading. 0 means the data will be loaded in the main process |
| 250 | (default: 0). |
| 251 | epoch (int, optional): the epoch to start the iterator from |
| 252 | (default: 1). |
| 253 | data_buffer_size (int, optional): number of batches to |
| 254 | preload (default: 0). |
| 255 | disable_iterator_cache (bool, optional): don't cache the |
| 256 | EpochBatchIterator (ignores `FairseqTask::can_reuse_epoch_itr`) |
| 257 | (default: False). |
| 258 | Returns: |
| 259 | ~fairseq.iterators.EpochBatchIterator: a batched iterator over the |
| 260 | given dataset split |
| 261 | """ |
| 262 | can_reuse_epoch_itr = not disable_iterator_cache and self.can_reuse_epoch_itr( |
| 263 | dataset |
| 264 | ) |
| 265 | if can_reuse_epoch_itr and dataset in self.dataset_to_epoch_iter: |
| 266 | logger.debug("reusing EpochBatchIterator for epoch {}".format(epoch)) |
| 267 | return self.dataset_to_epoch_iter[dataset] |
| 268 |
no test coverage detected