Function read

python/pathway/io/fs/__init__.py:32–269

Reads a table from one or several files with the specified format. In case the format is ``"plaintext"``, the table will consist of a single column ``data`` with each cell containing a single line from the file. In case the format is one of ``"plaintext_by_file"`` or ``"binary"`` the t

(
    path: str | PathLike,
    format: Literal[
        "csv", "json", "plaintext", "plaintext_by_file", "binary", "only_metadata"
    ],
    *,
    schema: type[Schema] | None = None,
    mode: Literal["streaming", "static"] = "streaming",
    csv_settings: CsvParserSettings | None = None,
    json_field_paths: dict[str, str] | None = None,
    object_pattern: str = "*",
    with_metadata: bool = False,
    name: str | None = None,
    autocommit_duration_ms: int | None = 1500,
    max_backlog_size: int | None = None,
    debug_data: Any = None,
    _stacklevel: int = 1,
    **kwargs,
)

@check_arg_types
@trace_user_frame
def read(
    path: str | PathLike,
    format: Literal[
        "csv", "json", "plaintext", "plaintext_by_file", "binary", "only_metadata"
    ],
    *,
    schema: type[Schema] | None = None,
    mode: Literal["streaming", "static"] = "streaming",
    csv_settings: CsvParserSettings | None = None,
    json_field_paths: dict[str, str] | None = None,
    object_pattern: str = "*",
    with_metadata: bool = False,
    name: str | None = None,
    autocommit_duration_ms: int | None = 1500,
    max_backlog_size: int | None = None,
    debug_data: Any = None,
    _stacklevel: int = 1,
    **kwargs,
) -> Table:
    """Reads a table from one or several files with the specified format.

    In case the format is ``"plaintext"``, the table will consist of a single column
    ``data`` with each cell containing a single line from the file.

    In case the format is one of ``"plaintext_by_file"`` or ``"binary"`` the table will
    consist of a single column ``data`` with each cell containing contents of the whole file.

    If the format is ``"only_metadata"``, only the metadata column will be read, without
    opening and without reading the contents of the files. The metadata is then available
    in the ``_metadata`` column.

    Args:
        path: Path to the file or to the folder with files or
            `glob <https://en.wikipedia.org/wiki/Glob_(programming)>`_ pattern for the
            objects to be read. The connector will read the contents of all matching files as well
            as recursively read the contents of all matching folders.
        format: Format of data to be read. Currently ``"csv"``, ``"json"``, ``"plaintext"``,
            ``"plaintext_by_file"``, ``"binary"``, and ``"only_metadata"`` formats are
            supported. The difference between ``"plaintext"`` and ``"plaintext_by_file"`` is
            how the input is tokenized: if the ``"plaintext"`` option is chosen, it&#x27;s split
            by the newlines. Otherwise, the files are split in full and one row will
            correspond to one file. In case the ``"binary"`` format is specified,
            the data is read as raw bytes without UTF-8 parsing. Finally, if ``"only_metadata"``
            is chosen, the connector only scans the filesystem for file additions,
            changes, modifications, and provides them in the metadata column.
        schema: Schema of the resulting table.
        mode: Denotes how the engine polls the new data from the source. Currently
            ``"streaming"`` and ``"static"`` are supported. If set to ``"streaming"`` the engine will wait for
            the updates in the specified directory. It will track file additions, deletions, and
            modifications and reflect these events in the state. For example, if a file was deleted,
            ``"streaming"`` mode will also remove rows obtained by reading this file from the table. On
            the other hand, the ``"static"`` mode will only consider the available data and ingest all
            of it in one commit. The default value is ``"streaming"``.
        csv_settings: Settings for the CSV parser. This parameter is used only in case
            the specified format is ``"csv"``.
        json_field_paths: If the format is ``"json"``, this field allows to map field names
            into path in the read json object. For the field which require such mapping,
            it should be given in the format ``<field_name>: <path to be mapped>``,