(table_data_generator, mb_per_reducer)
| 104 | |
| 105 | |
| 106 | def estimate_rows_per_reducer(table_data_generator, mb_per_reducer): |
| 107 | bytes_per_reducer = mb_per_reducer * 1024 ** 2 |
| 108 | bytes_per_row = estimate_bytes_per_row(table_data_generator, 1) |
| 109 | if bytes_per_row >= bytes_per_reducer: |
| 110 | return 1 |
| 111 | rows_per_reducer = bytes_per_reducer // bytes_per_row |
| 112 | bytes_per_row = estimate_bytes_per_row(table_data_generator, |
| 113 | max(int(rows_per_reducer * 0.001), 1)) |
| 114 | return max(bytes_per_reducer // bytes_per_row, 1) |
no test coverage detected