| 43 | } |
| 44 | |
| 45 | QueryPipelineBuilderPtr UnionStep::updatePipeline(QueryPipelineBuilders pipelines, const BuildQueryPipelineSettings & settings) |
| 46 | { |
| 47 | auto pipeline = std::make_unique<QueryPipelineBuilder>(); |
| 48 | |
| 49 | if (pipelines.empty()) |
| 50 | { |
| 51 | QueryPipelineProcessorsCollector collector(*pipeline, this); |
| 52 | pipeline->init(Pipe(std::make_shared<NullSource>(output_header))); |
| 53 | processors = collector.detachProcessors(); |
| 54 | return pipeline; |
| 55 | } |
| 56 | |
| 57 | size_t new_max_threads = max_threads ? max_threads : settings.max_threads; |
| 58 | |
| 59 | for (auto & cur_pipeline : pipelines) |
| 60 | { |
| 61 | /// Headers for union must be equal. |
| 62 | /// But, just in case, convert it to the same header if not. |
| 63 | /// This can happen when PREWHERE optimization adds extra pass-through columns |
| 64 | /// to ReadFromMergeTree output that are not consumed by the expression DAG above, |
| 65 | /// causing plan headers and pipeline headers to diverge. |
| 66 | if (!blocksHaveEqualStructure(cur_pipeline->getHeader(), *getOutputHeader())) |
| 67 | { |
| 68 | QueryPipelineProcessorsCollector collector(*cur_pipeline, this); |
| 69 | auto converting_dag = ActionsDAG::makeConvertingActions( |
| 70 | cur_pipeline->getHeader().getColumnsWithTypeAndName(), |
| 71 | getOutputHeader()->getColumnsWithTypeAndName(), |
| 72 | ActionsDAG::MatchColumnsMode::Name, |
| 73 | nullptr); |
| 74 | |
| 75 | auto converting_actions = std::make_shared<ExpressionActions>(std::move(converting_dag)); |
| 76 | cur_pipeline->addSimpleTransform([&](const SharedHeader & cur_header) |
| 77 | { |
| 78 | return std::make_shared<ExpressionTransform>(cur_header, converting_actions); |
| 79 | }); |
| 80 | |
| 81 | auto added_processors = collector.detachProcessors(); |
| 82 | processors.insert(processors.end(), added_processors.begin(), added_processors.end()); |
| 83 | } |
| 84 | |
| 85 | #if defined(DEBUG_OR_SANITIZER_BUILD) |
| 86 | assertCompatibleHeader(cur_pipeline->getHeader(), *getOutputHeader(), "UnionStep"); |
| 87 | #endif |
| 88 | } |
| 89 | |
| 90 | *pipeline = QueryPipelineBuilder::unitePipelines(std::move(pipelines), new_max_threads, &processors); |
| 91 | |
| 92 | /// The `max_streams_for_union_step*` cap only applies to steps built for SQL |
| 93 | /// `UNION ALL` / `UNION DISTINCT`, and only while no downstream step relies on |
| 94 | /// per-stream sortedness of the union output. For all other cases the narrowing |
| 95 | /// must be skipped: shuffling streams through `ConcatProcessor` would break the |
| 96 | /// ordering invariants of `GroupingAggregatedTransform` (memory-efficient |
| 97 | /// distributed aggregation), `MergingSortedTransform`, and similar order-sensitive |
| 98 | /// consumers. We still validate the ratio so misconfiguration is reported on every |
| 99 | /// query rather than only when a narrowable `UNION` happens to be present. |
| 100 | const double max_streams_ratio = settings.max_streams_for_union_step_to_max_threads_ratio; |
| 101 | if (!isFinite(max_streams_ratio) || max_streams_ratio < 0) |
| 102 | throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, |
nothing calls this directly
no test coverage detected