Method generate

src/Storages/ObjectStorage/StorageObjectStorageSource.cpp:378–608

}

Chunk StorageObjectStorageSource::generate()
{
    lazyInitialize();

    while (true)
    {
        if (isCancelled() || !reader)
        {
            if (reader)
                reader->cancel();
            break;
        }

        Chunk chunk;
        if (reader->pull(chunk))
        {
            UInt64 num_rows = chunk.getNumRows();
            total_rows_in_file += num_rows;

            size_t chunk_size = 0;
            if (const auto * input_format = reader.getInputFormat())
                chunk_size = input_format->getApproxBytesReadForChunk();

            progress(num_rows, chunk_size ? chunk_size : chunk.bytes());

            const auto & object_info = reader.getObjectInfo();
            const auto & filename = object_info->getFileName();
            std::string full_path = object_info->getPath();

            const auto reading_path = configuration->getPathForRead().path;

            if (!full_path.starts_with(reading_path))
                full_path = fs::path(reading_path) / object_info->getPath();

            auto object_metadata = object_info->getObjectMetadata();

            chassert(object_metadata);

            const auto path = getUniqueStoragePathIdentifier(*configuration, *object_info, false);

            /// The order is important, hive partition columns must be added before virtual columns
            /// because they are part of the schema
            if (!read_from_format_info.hive_partition_columns_to_read_from_file_path.empty())
            {
                HivePartitioningUtils::addPartitionColumnsToChunk(
                    chunk,
                    read_from_format_info.hive_partition_columns_to_read_from_file_path,
                    path,
                    format_settings,
                    read_context);
            }

            const String * iceberg_metadata_file_path = nullptr;
#if USE_AVRO
            if (const auto * iceberg_info = dynamic_cast<const IcebergDataObjectInfo *>(object_info.get()))
                iceberg_metadata_file_path = &iceberg_info->info.data_object_file_path_key.serialize();
#endif