Function test_read_not_materialized_table

tests/load/test_read_interfaces.py:1470–1525

(destination_config: DestinationTestConfiguration)

@pytest.mark.essential
def test_read_not_materialized_table(destination_config: DestinationTestConfiguration):
    # TODO lancedb destination faills unreliably on the 2nd `pipeline.run()` because
    # of the dltSentinel table. The test parallelization PR should solve this issue
    # and upgrading LanceDB destination with `lance-namespace` should avoid it
    # entirely.
    # TODO reenable test after fix
    if destination_config.destination_type == "lancedb":
        pytest.skip("lancedb has race conditions for dltSentinel table")

    @dlt.source
    def two_tables():
        @dlt.resource(
            columns=[{"name": "id", "data_type": "bigint", "nullable": True, "primary_key": True}],
            write_disposition="append",
            table_format=destination_config.table_format,
        )
        def table_1():
            yield {"id": 1}

        @dlt.resource(
            columns=[{"name": "id", "data_type": "bigint", "nullable": True}],
            write_disposition="replace",
            table_format=destination_config.table_format,
        )
        def table_3(make_data=False):
            return
            yield

        return table_1, table_3

    pipeline = destination_config.setup_pipeline(
        "test_pipeline_upfront_tables_two_loads",
        dataset_name="test_pipeline_upfront_tables_two_loads",
        dev_mode=True,
    )

    # create table without any data in it and try to access it. destination should not know this table
    # expected behavior is that table is not found
    schema = two_tables().discover_schema()

    # now we use this schema but load just one resource
    source = two_tables()
    # push state, table 3 not created
    pipeline.run(source.table_3, schema=schema, **destination_config.run_kwargs)

    with pytest.raises(DestinationUndefinedEntity):
        pipeline.dataset().table_3.fetchall()

    # now set table_3 so it has seen data
    with pipeline.dataset() as dataset_:
        schema = dataset_.schema
        schema.tables["table_3"]["x-normalizer"] = {"seen-data": True}

    # forces sql_client to map views. but data does not exist so must raise same exceptions
    with pytest.raises(DestinationUndefinedEntity):
        pipeline.dataset().table_3.fetchall()