r""" wrap reader so that it can be used with multiprocessing. Otherwise, pickling of records fails.
()
| 65 | reader = Reader(schema=[("text", str)]) |
| 66 | |
| 67 | def _wrap_reader(): |
| 68 | r""" wrap reader so that it can be used with multiprocessing. |
| 69 | Otherwise, pickling of records fails. """ |
| 70 | for record in reader.read( |
| 71 | uri="file://" + datafile, |
| 72 | max_samples=self._num_samples, |
| 73 | return_idx=False |
| 74 | ): |
| 75 | yield record.text |
| 76 | |
| 77 | global_dist = np.zeros(self._buckets, dtype=np.int64) |
| 78 |