Args: batch_size: batch size drop_last: whether to drop the last few data that cannot form a batch (recommanded True). shuffle: whether to shuffle the data slightly (according to their length of course).
(
self,
batch_size,
drop_last,
shuffle,
bucket_boundaries=None,
max_batch_combined_size=-1,
)
| 51 | """ |
| 52 | |
| 53 | def __init__( |
| 54 | self, |
| 55 | batch_size, |
| 56 | drop_last, |
| 57 | shuffle, |
| 58 | bucket_boundaries=None, |
| 59 | max_batch_combined_size=-1, |
| 60 | ): |
| 61 | """ |
| 62 | Args: |
| 63 | batch_size: |
| 64 | batch size |
| 65 | drop_last: |
| 66 | whether to drop the last few data that cannot form a batch (recommanded True). |
| 67 | shuffle: |
| 68 | whether to shuffle the data slightly (according to their length of course). |
| 69 | bucket_boundaries: |
| 70 | int (number of bins) or a list (containing the edges in ascending order, |
| 71 | excluding two outmost boundaries). |
| 72 | |
| 73 | edge 0 1 2 |
| 74 | len ___|___|___|___ |
| 75 | max_batch_combined_size: |
| 76 | limitation on batch_size * seq_len. -1: ignored |
| 77 | """ |
| 78 | self.batch_size = batch_size |
| 79 | self.drop_last = drop_last |
| 80 | self.shuffle = shuffle |
| 81 | self.bucket_boundaries = np.array(bucket_boundaries) if bucket_boundaries is not None else None |
| 82 | self.max_batch_combined_size = max_batch_combined_size |
| 83 | self.dataset_lengths = None |
| 84 | self.total_batches = None |
| 85 | |
| 86 | def set_bucket_boundaries(self, bucket_boundaries): |
| 87 | """ |
nothing calls this directly
no outgoing calls
no test coverage detected