Load and concatenate numpy arrays according to pattern. Args: pattern: pattern of array filenames to be loaded. stride: stride to traverse paths to be loaded. offset: starting offset into paths to be loaded. Returns: Loaded and concatenated array.
(
pattern: Union[str, Sequence[str]],
stride: int,
offset: int,
)
| 338 | |
| 339 | |
| 340 | def load_sharded_array( |
| 341 | pattern: Union[str, Sequence[str]], |
| 342 | stride: int, |
| 343 | offset: int, |
| 344 | ) -> np.ndarray: |
| 345 | """Load and concatenate numpy arrays according to pattern. |
| 346 | |
| 347 | Args: |
| 348 | pattern: pattern of array filenames to be loaded. |
| 349 | stride: stride to traverse paths to be loaded. |
| 350 | offset: starting offset into paths to be loaded. |
| 351 | |
| 352 | Returns: |
| 353 | Loaded and concatenated array. |
| 354 | """ |
| 355 | paths = tf.io.gfile.glob(pattern) |
| 356 | array_list = [] |
| 357 | for path in paths[offset::stride]: |
| 358 | logging.info('Loading %s on to process %d', path, jax.process_index()) |
| 359 | with tf.io.gfile.GFile(path, 'rb') as f: |
| 360 | array_list.append(np.load(f)) |
| 361 | |
| 362 | array = np.concatenate(array_list, axis=0) |
| 363 | return array |
| 364 | |
| 365 | |
| 366 | def save_sharded_array(array: np.ndarray, prefix: str, num_shards: int, |
nothing calls this directly
no test coverage detected
searching dependent graphs…