Merge a distributed analysis set from a FileHandler. Parameters ---------- set_path : str of pathlib.Path Path to distributed analysis set folder cleanup : bool, optional Delete distributed files after merging (default: False)
(set_path, cleanup=False)
| 189 | |
| 190 | |
| 191 | def merge_distributed_set(set_path, cleanup=False): |
| 192 | """ |
| 193 | Merge a distributed analysis set from a FileHandler. |
| 194 | |
| 195 | Parameters |
| 196 | ---------- |
| 197 | set_path : str of pathlib.Path |
| 198 | Path to distributed analysis set folder |
| 199 | cleanup : bool, optional |
| 200 | Delete distributed files after merging (default: False) |
| 201 | |
| 202 | """ |
| 203 | set_path = pathlib.Path(set_path) |
| 204 | logger.info("Merging set {}".format(set_path)) |
| 205 | |
| 206 | set_stem = set_path.stem |
| 207 | proc_paths = set_path.glob("{}_p*.h5".format(set_stem)) |
| 208 | proc_paths = natural_sort(proc_paths) |
| 209 | joint_path = set_path.parent.joinpath("{}.h5".format(set_stem)) |
| 210 | |
| 211 | # Create joint file, overwriting if it already exists |
| 212 | with h5py.File(str(joint_path), mode='w') as joint_file: |
| 213 | # Setup joint file based on first process file (arbitrary) |
| 214 | merge_setup(joint_file, proc_paths) |
| 215 | # Merge data from all process files |
| 216 | for proc_path in proc_paths: |
| 217 | merge_data(joint_file, proc_path) |
| 218 | # Cleanup after completed merge, if directed |
| 219 | if cleanup: |
| 220 | for proc_path in proc_paths: |
| 221 | proc_path.unlink() |
| 222 | set_path.rmdir() |
| 223 | |
| 224 | |
| 225 | def merge_setup(joint_file, proc_paths, virtual=False): |
no test coverage detected