Partition the list of files into `n` groups
(mut self, n: usize)
| 452 | |
| 453 | /// Partition the list of files into `n` groups |
| 454 | pub fn split_files(mut self, n: usize) -> Vec<FileGroup> { |
| 455 | if self.is_empty() { |
| 456 | return vec![]; |
| 457 | } |
| 458 | |
| 459 | // ObjectStore::list does not guarantee any consistent order and for some |
| 460 | // implementations such as LocalFileSystem, it may be inconsistent. Thus |
| 461 | // Sort files by path to ensure consistent plans when run more than once. |
| 462 | self.files.sort_by(|a, b| a.path().cmp(b.path())); |
| 463 | |
| 464 | // effectively this is div with rounding up instead of truncating |
| 465 | let chunk_size = self.len().div_ceil(n); |
| 466 | let mut chunks = Vec::with_capacity(n); |
| 467 | let mut current_chunk = Vec::with_capacity(chunk_size); |
| 468 | for file in self.files.drain(..) { |
| 469 | current_chunk.push(file); |
| 470 | if current_chunk.len() == chunk_size { |
| 471 | let full_chunk = FileGroup::new(mem::replace( |
| 472 | &mut current_chunk, |
| 473 | Vec::with_capacity(chunk_size), |
| 474 | )); |
| 475 | chunks.push(full_chunk); |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | if !current_chunk.is_empty() { |
| 480 | chunks.push(FileGroup::new(current_chunk)) |
| 481 | } |
| 482 | |
| 483 | chunks |
| 484 | } |
| 485 | |
| 486 | /// Groups files by their partition values, ensuring all files with same |
| 487 | /// partition values are in the same group. |