Method write_numpy

python/ray/data/dataset.py:5094–5191

Writes a column of the :class:`~ray.data.Dataset` to .npy files. This is only supported for columns in the datasets that can be converted to NumPy arrays. The number of files is determined by the number of blocks in the dataset. To control the number of number of bl

(
        self,
        path: str,
        *,
        column: str,
        filesystem: Optional["pyarrow.fs.FileSystem"] = None,
        try_create_dir: bool = True,
        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
        filename_provider: Optional[FilenameProvider] = None,
        min_rows_per_file: Optional[int] = None,
        ray_remote_args: Dict[str, Any] = None,
        concurrency: Optional[int] = None,
        num_rows_per_file: Optional[int] = None,
        mode: SaveMode = SaveMode.APPEND,
    )

    @ConsumptionAPI
    @PublicAPI(api_group=IOC_API_GROUP)
    def write_numpy(
        self,
        path: str,
        *,
        column: str,
        filesystem: Optional["pyarrow.fs.FileSystem"] = None,
        try_create_dir: bool = True,
        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
        filename_provider: Optional[FilenameProvider] = None,
        min_rows_per_file: Optional[int] = None,
        ray_remote_args: Dict[str, Any] = None,
        concurrency: Optional[int] = None,
        num_rows_per_file: Optional[int] = None,
        mode: SaveMode = SaveMode.APPEND,
    ) -> None:
        """Writes a column of the :class:`~ray.data.Dataset` to .npy files.

        This is only supported for columns in the datasets that can be converted to
        NumPy arrays.

        The number of files is determined by the number of blocks in the dataset.
        To control the number of number of blocks, call
        :meth:`~ray.data.Dataset.repartition`.


        By default, the format of the output files is ``{uuid}_{block_idx}.npy``,
        where ``uuid`` is a unique id for the dataset. To modify this behavior,
        implement a custom :class:`~ray.data.datasource.FilenameProvider`
        and pass it in as the ``filename_provider`` argument.

        Examples:
            >>> import ray
            >>> ds = ray.data.range(100)
            >>> ds.write_numpy("local:///tmp/data/", column="id")

        Time complexity: O(dataset size / parallelism)

        Args:
            path: The path to the destination root directory, where
                the npy files are written to.
            column: The name of the column that contains the data to
                be written.
            filesystem: The pyarrow filesystem implementation to write to.
                These filesystems are specified in the
                `pyarrow docs <https://arrow.apache.org/docs\
                /python/api/filesystems.html#filesystem-implementations>`_.
                Specify this if you need to provide specific configurations to the
                filesystem. By default, the filesystem is automatically selected based
                on the scheme of the paths. For example, if the path begins with
                ``s3://``, the ``S3FileSystem`` is used.
            try_create_dir: If ``True``, attempts to create all directories in
                destination path. Does nothing if all directories already
                exist. Defaults to ``True``.
            arrow_open_stream_args: kwargs passed to
                `pyarrow.fs.FileSystem.open_output_stream <https://arrow.apache.org\
                /docs/python/generated/pyarrow.fs.FileSystem.html\
                #pyarrow.fs.FileSystem.open_output_stream>`_, which is used when
                opening the file to write to.