Method write_webdataset

python/ray/data/dataset.py:4990–5090

Writes the dataset to `WebDataset `_ files. The `TFRecord `_ files will contain `tf.train.Example `_ # noqa: E501

(
        self,
        path: str,
        *,
        filesystem: Optional["pyarrow.fs.FileSystem"] = None,
        try_create_dir: bool = True,
        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
        filename_provider: Optional[FilenameProvider] = None,
        min_rows_per_file: Optional[int] = None,
        ray_remote_args: Dict[str, Any] = None,
        encoder: Optional[Union[bool, str, callable, list]] = True,
        concurrency: Optional[int] = None,
        num_rows_per_file: Optional[int] = None,
        mode: SaveMode = SaveMode.APPEND,
    )

    @ConsumptionAPI
    @PublicAPI(stability="alpha", api_group=IOC_API_GROUP)
    def write_webdataset(
        self,
        path: str,
        *,
        filesystem: Optional["pyarrow.fs.FileSystem"] = None,
        try_create_dir: bool = True,
        arrow_open_stream_args: Optional[Dict[str, Any]] = None,
        filename_provider: Optional[FilenameProvider] = None,
        min_rows_per_file: Optional[int] = None,
        ray_remote_args: Dict[str, Any] = None,
        encoder: Optional[Union[bool, str, callable, list]] = True,
        concurrency: Optional[int] = None,
        num_rows_per_file: Optional[int] = None,
        mode: SaveMode = SaveMode.APPEND,
    ) -> None:
        """Writes the dataset to `WebDataset <https://github.com/webdataset/webdataset>`_ files.

        The `TFRecord <https://www.tensorflow.org/tutorials/load_data/tfrecord>`_
        files will contain
        `tf.train.Example <https://www.tensorflow.org/api_docs/python/tf/train/Example>`_ # noqa: E501
        records, with one Example record for each row in the dataset.

        .. warning::
            tf.train.Feature only natively stores ints, floats, and bytes,
            so this function only supports datasets with these data types,
            and will error if the dataset contains unsupported types.

        This is only supported for datasets convertible to Arrow records.
        To control the number of files, use :meth:`Dataset.repartition`.

        Unless a custom filename provider is given, the format of the output
        files is ``{uuid}_{block_idx}.tfrecords``, where ``uuid`` is a unique id
        for the dataset.

        Examples:

            .. testcode::
                :skipif: True

                import ray

                ds = ray.data.range(100)
                ds.write_webdataset("s3://bucket/folder/")

        Time complexity: O(dataset size / parallelism)

        Args:
            path: The path to the destination root directory, where tfrecords
                files are written to.
            filesystem: The filesystem implementation to write to.
            try_create_dir: If ``True``, attempts to create all
                directories in the destination path. Does nothing if all directories
                already exist. Defaults to ``True``.
            arrow_open_stream_args: kwargs passed to
                ``pyarrow.fs.FileSystem.open_output_stream``
            filename_provider: A :class:`~ray.data.datasource.FilenameProvider`
                implementation. Use this parameter to customize what your filenames
                look like.