MCPcopy Create free account
hub / github.com/apache/datafusion / experiment

Function experiment

datafusion/core/src/datasource/memory_test.rs:266–310  ·  view source on GitHub ↗
(
        schema: SchemaRef,
        initial_data: Vec<Vec<RecordBatch>>,
        inserted_data: Vec<Vec<RecordBatch>>,
    )

Source from the content-addressed store, hash-verified

264 }
265
266 async fn experiment(
267 schema: SchemaRef,
268 initial_data: Vec<Vec<RecordBatch>>,
269 inserted_data: Vec<Vec<RecordBatch>>,
270 ) -> Result<Vec<Vec<RecordBatch>>> {
271 let expected_count: u64 = inserted_data
272 .iter()
273 .flat_map(|batches| batches.iter().map(|batch| batch.num_rows() as u64))
274 .sum();
275
276 // Create a new session context
277 let session_ctx = SessionContext::new();
278 // Create and register the initial table with the provided schema and data
279 let initial_table = Arc::new(MemTable::try_new(schema.clone(), initial_data)?);
280 session_ctx.register_table("t", initial_table.clone())?;
281 let target = Arc::new(DefaultTableSource::new(initial_table.clone()));
282 // Create and register the source table with the provided schema and inserted data
283 let source_table = Arc::new(MemTable::try_new(schema.clone(), inserted_data)?);
284 session_ctx.register_table("source", source_table.clone())?;
285 // Convert the source table into a provider so that it can be used in a query
286 let source = provider_as_source(source_table);
287 // Create a table scan logical plan to read from the source table
288 let scan_plan = LogicalPlanBuilder::scan("source", source, None)?.build()?;
289 // Create an insert plan to insert the source data into the initial table
290 let insert_into_table =
291 LogicalPlanBuilder::insert_into(scan_plan, "t", target, InsertOp::Append)?
292 .build()?;
293 // Create a physical plan from the insert plan
294 let plan = session_ctx
295 .state()
296 .create_physical_plan(&insert_into_table)
297 .await?;
298
299 // Execute the physical plan and collect the results
300 let res = collect(plan, session_ctx.task_ctx()).await?;
301 assert_eq!(extract_count(res), expected_count);
302
303 // Read the data from the initial table and store it in a vector of partitions
304 let mut partitions = vec![];
305 for partition in initial_table.batches.iter() {
306 let part = partition.read().await.clone();
307 partitions.push(part);
308 }
309 Ok(partitions)
310 }
311
312 /// Returns the value of results. For example, returns 6 given the following
313 ///

Calls 15

newFunction · 0.85
provider_as_sourceFunction · 0.85
sumMethod · 0.80
readMethod · 0.80
scanFunction · 0.50
collectFunction · 0.50
iterMethod · 0.45
mapMethod · 0.45
cloneMethod · 0.45
register_tableMethod · 0.45
buildMethod · 0.45
create_physical_planMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…