Function loadUrlListToDB

AskAgent/python/data_loading/db_load.py:910–1060

Process a file containing a list of URLs, fetch each URL, and load the content into the database. Each line in the file should be a single URL pointing to RSS/XML or JSON content. The file itself can be a local file or a URL. Args: file_path: Path to the file containing URL

(file_path: str, site: str, batch_size: int = 100, delete_existing: bool = False, force_recompute: bool = False, database: str | None = None)

                pass

async def loadUrlListToDB(file_path: str, site: str, batch_size: int = 100, delete_existing: bool = False, force_recompute: bool = False, database: str | None = None):
    """
    Process a file containing a list of URLs, fetch each URL, and load the content into the database.
    Each line in the file should be a single URL pointing to RSS/XML or JSON content.
    The file itself can be a local file or a URL.

    Args:
        file_path: Path to the file containing URLs, one per line, or a URL to such a file
        site: Site identifier
        batch_size: Number of documents to process and upload in each batch
        delete_existing: Whether to delete existing entries for this site before loading
        force_recompute: Whether to force recomputation of embeddings
        database: Specific database endpoint to use (if None, uses preferred endpoint)

    Returns:
        Total number of documents loaded
    """
    # Use specified database or fall back to preferred endpoint
    endpoint_name = database or CONFIG.write_endpoint

    # Check if the file_path is a URL
    is_url_list_remote = await is_url(file_path)
    temp_path = None

    try:
        # If the file is a URL, fetch it first
        if is_url_list_remote:
            print(f"URL list file is a remote URL. Fetching: {file_path}")
            content, _ = await fetch_url(file_path)

            # Save to temporary file
            with tempfile.NamedTemporaryFile(suffix='.txt', delete=False, mode='w', encoding='utf-8') as temp:
                temp.write(content)
                temp_path = temp.name

            print(f"Saved URL list to temporary file: {temp_path}")
            # Use the temp path for further processing
            file_path = temp_path

        # Read all lines from the file, each line should be a URL
        lines = await read_file_lines(file_path)
        total_urls = len(lines)

        # Filter out empty lines and comments
        urls = [line.strip() for line in lines if line.strip() and not line.strip().startswith('#')]
        total_valid_urls = len(urls)

        print(f"Found {total_valid_urls} valid URLs out of {total_urls} lines in the file")

        # Delete existing entries for this site if requested (do this only once)
        if delete_existing:
            await delete_site_from_database(site, endpoint_name)

        # Get client directly from the factory function, using query_params for development mode override
        query_params = {"db": database} if database else None
        get_vector_db_client(query_params=query_params)

        # Process each URL