Method
__init__
(
self,
template,
dataset,
num_proc: int = 1,
*,
packing_interval: int = 128,
packing_length: Optional[int] = None,
strict: bool = False,
cyclic: bool = False,
**kwargs,
)
Source from the content-addressed store, hash-verified
| 113 | class IterablePackingDataset(IterableDataset): |
| 114 | |
| 115 | def __init__( |
| 116 | self, |
| 117 | template, |
| 118 | dataset, |
| 119 | num_proc: int = 1, |
| 120 | *, |
| 121 | packing_interval: int = 128, |
| 122 | packing_length: Optional[int] = None, |
| 123 | strict: bool = False, |
| 124 | cyclic: bool = False, |
| 125 | **kwargs, |
| 126 | ): |
| 127 | template.packing = True |
| 128 | template.padding_free = True # TODO: remove |
| 129 | self.template = template |
| 130 | self.dataset = dataset |
| 131 | self.num_proc = num_proc |
| 132 | self.strict = strict |
| 133 | self.packing_length = packing_length or self.template.max_length |
| 134 | |
| 135 | self.packing_interval = packing_interval |
| 136 | self._in_queue = mp.Queue() |
| 137 | self._out_queue = mp.Queue() |
| 138 | self.workers = [] |
| 139 | self.cyclic = cyclic |
| 140 | for _ in range(self.num_proc): |
| 141 | worker = mp.Process(target=self._processor, daemon=True) |
| 142 | worker.start() |
| 143 | self.workers.append(worker) |
| 144 | |
| 145 | def _processor(self): |
| 146 | while True: |
Callers
nothing calls this directly
Tested by
no test coverage detected