Class SqlStorageClient

src/crawlee/storage_clients/_sql/_storage_client.py:35–305

SQL implementation of the storage client. This storage client provides access to datasets, key-value stores, and request queues that persist data to a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for records. The client accepts eithe

@docs_group('Storage clients')
class SqlStorageClient(StorageClient):
    """SQL implementation of the storage client.

    This storage client provides access to datasets, key-value stores, and request queues that persist data
    to a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for
    records.

    The client accepts either a database connection string or a pre-configured AsyncEngine. If neither is
    provided, it creates a default SQLite database 'crawlee.db' in the storage directory.

    Database schema is automatically created during initialization. SQLite databases receive performance
    optimizations including WAL mode and increased cache size.

    Warning:
        This is an experimental feature. The behavior and interface may change in future versions.
    """

    _DEFAULT_DB_NAME = 'crawlee.db'
    """Default database name if not specified in connection string."""

    _SUPPORTED_DIALECTS: ClassVar[set[str]] = {'sqlite', 'postgresql', 'mysql', 'mariadb'}

    def __init__(
        self,
        *,
        connection_string: str | None = None,
        engine: AsyncEngine | None = None,
    ) -> None:
        """Initialize the SQL storage client.

        Args:
            connection_string: Database connection string (e.g., "sqlite+aiosqlite:///crawlee.db").
                If not provided, defaults to SQLite database in the storage directory.
            engine: Pre-configured AsyncEngine instance. If provided, connection_string is ignored.
        """
        if engine is not None and connection_string is not None:
            raise ValueError('Either connection_string or engine must be provided, not both.')

        self._connection_string = connection_string
        self._engine = engine
        self._initialized = False
        self.session_maker: None | async_sessionmaker[AsyncSession] = None

        self._listeners_registered = False
        self._dialect_name: str | None = None

        # Call the notification only once
        warnings.warn(
            'The SqlStorageClient is experimental and may change or be removed in future releases.',
            category=UserWarning,
            stacklevel=2,
        )

    async def __aenter__(self) -> SqlStorageClient:
        """Async context manager entry."""
        return self

    async def __aexit__(