Store arrays in HDF5 file This saves several dask arrays into several datapaths in an HDF5 file. It creates the necessary datasets and handles clean file opening/closing. Parameters ---------- chunks: tuple or ``True`` Chunk shape, or ``True`` to pass the chunks from th
(filename, *args, chunks=True, **kwargs)
| 5707 | |
| 5708 | |
| 5709 | def to_hdf5(filename, *args, chunks=True, **kwargs): |
| 5710 | """Store arrays in HDF5 file |
| 5711 | |
| 5712 | This saves several dask arrays into several datapaths in an HDF5 file. |
| 5713 | It creates the necessary datasets and handles clean file opening/closing. |
| 5714 | |
| 5715 | Parameters |
| 5716 | ---------- |
| 5717 | chunks: tuple or ``True`` |
| 5718 | Chunk shape, or ``True`` to pass the chunks from the dask array. |
| 5719 | Defaults to ``True``. |
| 5720 | |
| 5721 | Examples |
| 5722 | -------- |
| 5723 | |
| 5724 | >>> da.to_hdf5('myfile.hdf5', '/x', x) # doctest: +SKIP |
| 5725 | |
| 5726 | or |
| 5727 | |
| 5728 | >>> da.to_hdf5('myfile.hdf5', {'/x': x, '/y': y}) # doctest: +SKIP |
| 5729 | |
| 5730 | Optionally provide arguments as though to ``h5py.File.create_dataset`` |
| 5731 | |
| 5732 | >>> da.to_hdf5('myfile.hdf5', '/x', x, compression='lzf', shuffle=True) # doctest: +SKIP |
| 5733 | |
| 5734 | >>> da.to_hdf5('myfile.hdf5', '/x', x, chunks=(10,20,30)) # doctest: +SKIP |
| 5735 | |
| 5736 | This can also be used as a method on a single Array |
| 5737 | |
| 5738 | >>> x.to_hdf5('myfile.hdf5', '/x') # doctest: +SKIP |
| 5739 | |
| 5740 | See Also |
| 5741 | -------- |
| 5742 | da.store |
| 5743 | h5py.File.create_dataset |
| 5744 | """ |
| 5745 | if len(args) == 1 and isinstance(args[0], dict): |
| 5746 | data = args[0] |
| 5747 | elif len(args) == 2 and isinstance(args[0], str) and isinstance(args[1], Array): |
| 5748 | data = {args[0]: args[1]} |
| 5749 | else: |
| 5750 | raise ValueError("Please provide {'/data/path': array} dictionary") |
| 5751 | |
| 5752 | import h5py |
| 5753 | |
| 5754 | with h5py.File(filename, mode="a") as f: |
| 5755 | dsets = [ |
| 5756 | f.require_dataset( |
| 5757 | dp, |
| 5758 | shape=x.shape, |
| 5759 | dtype=x.dtype, |
| 5760 | chunks=tuple(c[0] for c in x.chunks) if chunks is True else chunks, |
| 5761 | **kwargs, |
| 5762 | ) |
| 5763 | for dp, x in data.items() |
| 5764 | ] |
| 5765 | store(list(data.values()), dsets) |
| 5766 |