Function process_normal_path

AskAgent/python/data_loading/db_load.py:1073–1112

(input_file_path: str, site: str, batch_size: int = 100, delete_site: bool = False, force_recompute: bool = False, database: str | None = None)

    print(f"Deleted {count} entries for site '{site}'")

async def process_normal_path(input_file_path: str, site: str, batch_size: int = 100, delete_site: bool = False, force_recompute: bool = False, database: str | None = None):
    # Check if file exists at the specified path
    if not await is_url(input_file_path) and not os.path.exists(input_file_path):
        print(f"Warning: File not found at '{input_file_path}'. Will try to resolve or download it.")

    # Determine if the file is a URL
    is_url_path = await is_url(input_file_path)
    temp_path = None
    file_path = input_file_path

    # If it's a URL, fetch the content
    if is_url_path:
        print(f"Fetching content from URL: {input_file_path}")
        temp_path, _ = await save_url_content(input_file_path)
        file_path = temp_path

    try:
        # Detect file type and if it contains embeddings
        if os.path.exists(file_path):
            file_type, has_embeddings = await detect_file_type(file_path)
            print(f"Detected file type: {file_type}, contains embeddings: {'Yes' if has_embeddings else 'No'}")

            # Process based on whether the file has embeddings
            if has_embeddings and not force_recompute:
                print("File already contains embeddings, loading directly...")
                await loadJsonWithEmbeddingsToDB(file_path, site, batch_size, delete_site, database)
            else:
                print("Computing embeddings for file...")
                await loadJsonToDB(file_path, site, batch_size, delete_site, force_recompute, database)
        else:
            print(f"Error: File not found at '{file_path}'")
            sys.exit(1)
    finally:
        # Clean up temporary file if needed
        if temp_path and os.path.exists(temp_path):
            try:
                os.unlink(temp_path)
                print(f"Cleaned up temporary file: {temp_path}")
            except Exception:
                pass


async def main():