MCPcopy Create free account
hub / github.com/apache/datafusion / execute

Method execute

datafusion/physical-plan/src/repartition/mod.rs:1210–1351  ·  view source on GitHub ↗
(
        &self,
        partition: usize,
        context: Arc<TaskContext>,
    )

Source from the content-addressed store, hash-verified

1208 }
1209
1210 fn execute(
1211 &self,
1212 partition: usize,
1213 context: Arc<TaskContext>,
1214 ) -> Result<SendableRecordBatchStream> {
1215 trace!(
1216 "Start {}::execute for partition: {}",
1217 self.name(),
1218 partition
1219 );
1220
1221 let spill_metrics = SpillMetrics::new(&self.metrics, partition);
1222
1223 let input = Arc::clone(&self.input);
1224 let partitioning = self.partitioning().clone();
1225 let metrics = self.metrics.clone();
1226 let preserve_order = self.sort_exprs().is_some();
1227 let name = self.name().to_owned();
1228 let schema = self.schema();
1229 let schema_captured = Arc::clone(&schema);
1230
1231 let spill_manager = SpillManager::new(
1232 Arc::clone(&context.runtime_env()),
1233 spill_metrics,
1234 input.schema(),
1235 );
1236
1237 // Get existing ordering to use for merging
1238 let sort_exprs = self.sort_exprs().cloned();
1239
1240 let state = Arc::clone(&self.state);
1241 if let Some(mut state) = state.try_lock() {
1242 state.ensure_input_streams_initialized(
1243 &input,
1244 &metrics,
1245 partitioning.partition_count(),
1246 &context,
1247 )?;
1248 }
1249
1250 let num_input_partitions = input.output_partitioning().partition_count();
1251
1252 let stream = futures::stream::once(async move {
1253 // lock scope
1254 let (rx, reservation, spill_readers, abort_helper) = {
1255 // lock mutexes
1256 let mut state = state.lock();
1257 let state = state.consume_input_streams(
1258 &input,
1259 &metrics,
1260 &partitioning,
1261 preserve_order,
1262 &name,
1263 &context,
1264 spill_manager.clone(),
1265 )?;
1266
1267 // now return stream for the specified *output* partition which will

Calls 15

newFunction · 0.85
partitioningMethod · 0.80
sort_exprsMethod · 0.80
partition_countMethod · 0.80
consume_input_streamsMethod · 0.80
memory_poolMethod · 0.80
with_spill_managerMethod · 0.80
with_expressionsMethod · 0.80
with_streamsMethod · 0.80
session_configMethod · 0.80
cloneMethod · 0.45