MCPcopy Create free account
hub / github.com/apache/datafusion / generate_dict

Function generate_dict

benchmarks/src/dict.rs:259–311  ·  view source on GitHub ↗
(
    value_type: DictValueType,
    cardinality: Cardinality,
    null_percent: NullPercent,
    size: usize,
)

Source from the content-addressed store, hash-verified

257];
258
259pub fn generate_dict(
260 value_type: DictValueType,
261 cardinality: Cardinality,
262 null_percent: NullPercent,
263 size: usize,
264) -> ArrayRef {
265 let num_distinct = cardinality.num_distinct(size);
266
267 let dict_values: ArrayRef = match value_type {
268 DictValueType::Utf8 => {
269 let strings: StringArray = (0..num_distinct)
270 .map(|i| format!("value_{i}"))
271 .collect::<Vec<_>>()
272 .into();
273 Arc::new(strings)
274 }
275 DictValueType::ListUtf8 => {
276 let flat: StringArray = (0..num_distinct)
277 .flat_map(|i| (0..ITEMS_PER_VALUE).map(move |j| format!("value_{i}_{j}")))
278 .collect::<Vec<_>>()
279 .into();
280
281 let offsets: Vec<i32> = (0..=(num_distinct * ITEMS_PER_VALUE))
282 .step_by(ITEMS_PER_VALUE)
283 .map(|o| o as i32)
284 .collect();
285
286 Arc::new(ListArray::new(
287 Arc::new(Field::new("item", DataType::Utf8, false)),
288 OffsetBuffer::new(offsets.into()),
289 Arc::new(flat),
290 None,
291 ))
292 }
293 };
294
295 let null_every = null_percent.null_every();
296
297 let mut key_builder = Int32Array::builder(size);
298 for i in 0..size {
299 if null_every.is_some_and(|n| i % n == 0) {
300 key_builder.append_null();
301 } else {
302 key_builder.append_value((i % num_distinct) as i32);
303 }
304 }
305 let keys = key_builder.finish();
306
307 Arc::new(
308 DictionaryArray::<Int32Type>::try_new(keys, dict_values)
309 .expect("valid dictionary array"),
310 )
311}
312impl RunOpt {
313 pub async fn run(self) -> Result<()> {
314 println!(

Callers 1

make_record_batchMethod · 0.85

Calls 9

newFunction · 0.85
num_distinctMethod · 0.80
collectMethod · 0.80
null_everyMethod · 0.80
intoMethod · 0.45
mapMethod · 0.45
append_nullMethod · 0.45
append_valueMethod · 0.45
finishMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…