Function array_data

datafusion/functions/benches/substr_index.rs:104–147

(
    batch_size: usize,
    single_char_delimiter: bool,
)

}

fn array_data(
    batch_size: usize,
    single_char_delimiter: bool,
) -> (Vec<String>, Vec<String>, Vec<i64>) {
    let count_dist = Filter {
        dist: Uniform::new(-4, 5).expect("valid count distribution"),
        test: |x: &i64| x != &0,
    };
    let mut rng = StdRng::seed_from_u64(ARRAY_DATA_SEED);
    let mut strings = Vec::with_capacity(batch_size);
    let mut delimiters = Vec::with_capacity(batch_size);
    let mut counts = Vec::with_capacity(batch_size);

    for _ in 0..batch_size {
        let length = rng.random_range(20..50);
        let base = random_token(&mut rng, length);

        let (string_value, delimiter) = if single_char_delimiter {
            let char_idx = rng.random_range(0..base.chars().count());
            let delimiter = base.chars().nth(char_idx).unwrap().to_string();
            (base, delimiter)
        } else {
            let long_delimiters = ["|||", "***", "&&&", "###", "@@@", "$$$"];
            let delimiter =
                long_delimiters[rng.random_range(0..long_delimiters.len())].to_string();

            let delimiter_count = rng.random_range(1..4);
            let mut result = String::new();
            for i in 0..delimiter_count {
                result.push_str(&base);
                if i < delimiter_count - 1 {
                    result.push_str(&delimiter);
                }
            }
            (result, delimiter)
        };

        strings.push(string_value);
        delimiters.push(delimiter);
        counts.push(count_dist.sample(&mut rng));
    }

    (strings, delimiters, counts)
}

fn scalar_data(batch_size: usize, delimiter: &str) -> Vec<String> {
    let mut rng = StdRng::seed_from_u64(SCALAR_DATA_SEED);