MCPcopy Create free account
hub / github.com/apache/impala / CalculateDataFiles

Method CalculateDataFiles

be/src/exec/iceberg-delete-builder.cc:117–208  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

115IcebergDeleteBuilder::~IcebergDeleteBuilder() {}
116
117Status IcebergDeleteBuilder::CalculateDataFiles() {
118 auto& fragment_state_map = runtime_state_->query_state()->FragmentStateMap();
119 auto fragment_it = fragment_state_map.end();
120 PlanNode* delete_scan_node = nullptr;
121 bool found = false;
122 std::queue<const PlanNode*> q;
123 for (auto it = fragment_state_map.begin(); !found && it != fragment_state_map.end();
124 it++) {
125 q.push(it->second->plan_tree());
126 while (!q.empty()) {
127 auto* current = q.front();
128 q.pop();
129 if (current->tnode_->node_id == join_node_id_) {
130 fragment_it = it;
131 // Tuple caching can place a TupleCacheNode above the scan node. Look past
132 // a TupleCacheNode to get to the scan node.
133 delete_scan_node = PlanNode::LookPastTupleCache(current->children_[0]);
134 DCHECK_EQ(delete_scan_node->tnode_->node_type, TPlanNodeType::HDFS_SCAN_NODE)
135 << "Failed to calculate delete files: "
136 << Substitute("Unexpected type for plan node $0: $1",
137 delete_scan_node->tnode_->node_id, delete_scan_node->tnode_->node_type);
138 found = true;
139 while (!q.empty()) q.pop();
140 break;
141 }
142 for (auto* child : current->children_) {
143 q.push(child);
144 }
145 }
146 }
147
148 const vector<const PlanFragmentInstanceCtxPB*>& instance_ctx_pbs =
149 fragment_it->second->instance_ctx_pbs();
150 for (auto ctx : instance_ctx_pbs) {
151 auto ranges = ctx->per_node_scan_ranges().find(delete_scan_node->tnode_->node_id);
152 if (ranges == ctx->per_node_scan_ranges().end()) continue;
153
154 auto tuple_id = delete_scan_node->tnode_->hdfs_scan_node.tuple_id;
155 auto tuple_desc = runtime_state_->desc_tbl().GetTupleDescriptor(tuple_id);
156 DCHECK(tuple_desc->table_desc() != nullptr);
157 auto hdfs_table = static_cast<const HdfsTableDescriptor*>(tuple_desc->table_desc());
158 DCHECK(hdfs_table->IsIcebergTable());
159
160 for (const ScanRangeParamsPB& params : ranges->second.scan_ranges()) {
161 DCHECK(params.scan_range().has_hdfs_file_split());
162 const HdfsFileSplitPB& split = params.scan_range().hdfs_file_split();
163
164 HdfsPartitionDescriptor* partition_desc =
165 hdfs_table->GetPartition(split.partition_id());
166
167 std::filesystem::path file_path;
168 if (split.relative_path().empty()) {
169 file_path.append(split.absolute_path());
170 } else {
171 file_path.append(partition_desc->location()).append(split.relative_path());
172 }
173 auto& file_path_str = file_path.native();
174 char* ptr_copy =

Callers

nothing calls this directly

Calls 15

SubstituteFunction · 0.85
moveFunction · 0.85
OKFunction · 0.85
pushMethod · 0.80
plan_treeMethod · 0.80
frontMethod · 0.80
GetTupleDescriptorMethod · 0.80
IsIcebergTableMethod · 0.80
GetPartitionMethod · 0.80
locationMethod · 0.80
pathMethod · 0.80
StatusClass · 0.70

Tested by

no test coverage detected