Class Loader

python/graphscope/framework/loader.py:94–287

Generic data source wrapper. Loader can take various data sources, and assemble necessary information into a AttrValue.

class Loader(object):
    """Generic data source wrapper.
    Loader can take various data sources, and assemble necessary information into a AttrValue.
    """

    def __init__(
        self, source, delimiter=",", sep=",", header_row=True, filetype="CSV", **kwargs
    ):
        """Initialize a loader with configurable options.
        Note: Loader cannot be reused since it may change inner state when constructing
        information for loading a graph.

        Args:
            source (str or value):
                The data source to be load, which could be one of the followings:

                    * local file: specified by URL :code:`file://...`
                    * oss file: specified by URL :code:`oss://...`
                    * hdfs file: specified by URL :code:`hdfs://...`
                    * s3 file: specified by URL :code:`s3://...`
                    * numpy ndarray, in CSR format
                    * pandas dataframe

                Ordinary data sources can be loaded using vineyard stream as well, a :code:`vineyard://`
                prefix can be used in the URL then the local file, oss object or HDFS file will be loaded
                into a vineyard stream first, then GraphScope's fragment will be built upon those streams
                in vineyard.

                Once the stream IO in vineyard reaches a stable state, it will be the default mode to
                load data sources and construct fragments in GraphScope.

            delimiter (char, optional): Column delimiter. Defaults to ','

            header_row (bool, optional): Whether source have a header. If true, column names
                will be read from the first row of source, else they are named by 'f0', 'f1', ....
                Defaults to True.

            filetype (str, optional): Specify the type of files to load, can be "CSV", "ORC", and
                "PARQUET". Default is "CSV".

        Notes:
            Data is resolved by drivers in `vineyard <https://github.com/v6d-io/v6d>`_ .
            See more additional info in `Loading Graph` section of Docs, and implementations in `vineyard`.
        """
        self.protocol = ""
        # For numpy or pandas, source is the serialized raw bytes
        # For files, it's the location
        # For vineyard, it's the ID or name
        self.source = ""
        # options for data source is csv
        self.options = CSVOptions()
        check_argument(
            isinstance(delimiter, str) and len(delimiter) == 1,
            "The delimiter must be a single character, cannot be '%s'" % delimiter,
        )
        self.options.delimiter = delimiter
        self.options.header_row = header_row
        self.options.filetype = filetype