Function rest_api_source

dlt/sources/rest_api/__init__.py:78–147

Creates and configures a REST API source for data extraction. Args: config (RESTAPIConfig): Configuration for the REST API source. name (str, optional): Name of the source. section (str, optional): Section of the configuration file. max_table_nesting (int, option

(
    config: RESTAPIConfig,
    name: str = None,
    section: str = None,
    max_table_nesting: int = None,
    root_key: bool = None,
    schema: Schema = None,
    schema_contract: TSchemaContract = None,
    parallelized: bool = False,
)

def rest_api_source(
    config: RESTAPIConfig,
    name: str = None,
    section: str = None,
    max_table_nesting: int = None,
    root_key: bool = None,
    schema: Schema = None,
    schema_contract: TSchemaContract = None,
    parallelized: bool = False,
) -> DltSource:
    """Creates and configures a REST API source for data extraction.

    Args:
        config (RESTAPIConfig): Configuration for the REST API source.
        name (str, optional): Name of the source.
        section (str, optional): Section of the configuration file.
        max_table_nesting (int, optional): Maximum depth of nested table above which
            the remaining nodes are loaded as structs or JSON.
        root_key (bool, optional): Enables merging on all resources by propagating
            root foreign key to child tables. This option is most useful if you
            plan to change write disposition of a resource to disable/enable merge.
            Defaults to False.
        schema (Schema, optional): An explicit `Schema` instance to be associated
            with the source. If not present, `dlt` creates a new `Schema` object
            with provided `name`. If such `Schema` already exists in the same
            folder as the module containing the decorated function, such schema
            will be loaded from file.
        schema_contract (TSchemaContract, optional): Schema contract settings
            that will be applied to this resource.
        parallelized (bool, optional): If `True`, resource generators will be
            extracted in parallel with other resources. Transformers that return items are also parallelized.
            Non-eligible resources are ignored. Defaults to `False` which preserves resource settings.

    Returns:
        DltSource: A configured dlt source.

    Example:
        pokemon_source = rest_api_source({
            "client": {
                "base_url": "https://pokeapi.co/api/v2/",
                "paginator": "json_link",
            },
            "resources": [
                {
                    "name": "pokemon",
                    "endpoint": {
                        "path": "pokemon",
                        "params": {
                            "limit": 100,
                        },
                    },
                    "primary_key": "id",
                }
            ]
        })
    """
    # TODO: this must be removed when TypedDicts are supported by resolve_configuration
    #   so secrets values are bound BEFORE validation. validation will happen during the resolve process