MCPcopy Create free account
hub / github.com/apache/arrow / GenerateTestCase

Method GenerateTestCase

cpp/src/arrow/json/reader_test.cc:537–587  ·  view source on GitHub ↗

Creates a test case from valid JSON objects with a human-readable index field and a struct field of random data. `block_size_multiplier` is applied to the largest generated row length to determine the target block_size. i.e - higher multiplier means fewer batches

Source from the content-addressed store, hash-verified

535 // generated row length to determine the target block_size. i.e - higher multiplier
536 // means fewer batches
537 static TestCase GenerateTestCase(int num_rows, double block_size_multiplier = 3.0) {
538 FieldVector data_fields = {field("s", utf8()), field("f", float64()),
539 field("b", boolean())};
540 FieldVector fields = {field("i", int64()), field("d", struct_({data_fields}))};
541 TestCase out;
542 out.schema = schema(fields);
543 out.num_rows = num_rows;
544
545 constexpr int kSeed = 0x432432;
546 std::default_random_engine engine(kSeed);
547 std::vector<std::string> rows(num_rows);
548 size_t max_row_size = 1;
549
550 auto options = GenerateOptions::Defaults();
551 options.null_probability = 0;
552 for (int i = 0; i < num_rows; ++i) {
553 Writer writer;
554 ABORT_NOT_OK(Generate(data_fields, engine, &writer, options));
555
556 std::string json(writer.GetString().ValueOrDie());
557
558 rows[i] = Join({"{\"i\":", std::to_string(i), ",\"d\":", json, "}\n"});
559 max_row_size = std::max(max_row_size, rows[i].size());
560 }
561
562 auto block_size = static_cast<size_t>(max_row_size * block_size_multiplier);
563 // Deduce the expected record batches from the target block size.
564 std::vector<std::string> batch_rows;
565 size_t pos = 0;
566 for (const auto& row : rows) {
567 pos += row.size();
568 if (pos > block_size) {
569 out.batches.push_back(
570 RecordBatchFromJSON(out.schema, Join({"[", Join(batch_rows, ","), "]"})));
571 batch_rows.clear();
572 pos -= block_size;
573 }
574 batch_rows.push_back(row);
575 out.json += row;
576 }
577 if (!batch_rows.empty()) {
578 out.batches.push_back(
579 RecordBatchFromJSON(out.schema, Join({"[", Join(batch_rows, ","), "]"})));
580 }
581
582 out.json_size = static_cast<int>(out.json.size());
583 out.block_size = static_cast<int>(block_size);
584 out.num_batches = static_cast<int>(out.batches.size());
585
586 return out;
587 }
588
589 static std::string Join(const std::vector<std::string>& strings,
590 const std::string& delim = "", bool trailing_delim = false) {

Callers

nothing calls this directly

Calls 14

struct_Function · 0.85
GenerateFunction · 0.85
JoinFunction · 0.85
to_stringFunction · 0.85
RecordBatchFromJSONFunction · 0.85
ValueOrDieMethod · 0.80
push_backMethod · 0.80
fieldFunction · 0.50
schemaFunction · 0.50
DefaultsFunction · 0.50
GetStringMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected