(self, chunk_size)
| 49 | params = [chunk_sizes] |
| 50 | |
| 51 | def setup(self, chunk_size): |
| 52 | # Note we're careful to stream different chunks instead of |
| 53 | # streaming N times the same chunk, so that we avoid operating |
| 54 | # entirely out of L1/L2. |
| 55 | chunks = generate_chunks(self.total_size, |
| 56 | nchunks=self.total_size // chunk_size, |
| 57 | ncols=self.ncols) |
| 58 | batches = [pa.RecordBatch.from_pandas(df) |
| 59 | for df in chunks] |
| 60 | schema = batches[0].schema |
| 61 | sink = pa.BufferOutputStream() |
| 62 | stream_writer = pa.RecordBatchStreamWriter(sink, schema) |
| 63 | for batch in batches: |
| 64 | stream_writer.write_batch(batch) |
| 65 | self.source = sink.getvalue() |
| 66 | |
| 67 | def time_read_to_dataframe(self, *args): |
| 68 | reader = pa.RecordBatchStreamReader(self.source) |
nothing calls this directly
no test coverage detected