MCPcopy Create free account
hub / github.com/apache/arrow / RandomUtf8String

Function RandomUtf8String

cpp/src/arrow/testing/random.cc:1487–1562  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1485}
1486
1487std::string RandomUtf8String(random::SeedType seed, int num_chars) {
1488 arrow::random::pcg32 gen(seed);
1489 std::string s;
1490 s.reserve(num_chars * 3); // Reserve for average 3 bytes per codepoint
1491
1492 std::uniform_int_distribution<uint32_t> plane_dist(0, 3);
1493 std::bernoulli_distribution bmp_range_dist(0.5);
1494 std::uniform_int_distribution<uint32_t> bmp_lower_dist(0x0020, 0xD7FF);
1495 std::uniform_int_distribution<uint32_t> bmp_upper_dist(0xE000, 0xFFFD);
1496 std::uniform_int_distribution<uint32_t> smp_dist(0x10000, 0x1FFFF);
1497 std::uniform_int_distribution<uint32_t> sip_dist(0x20000, 0x2FFFF);
1498 std::uniform_int_distribution<uint32_t> high_plane_dist(0x30000, 0x10FFFF);
1499
1500 for (int i = 0; i < num_chars; ++i) {
1501 uint32_t codepoint;
1502 uint32_t plane = plane_dist(gen);
1503
1504 if (plane == 0) {
1505 // Basic Multilingual Plane (BMP): U+0000 to U+FFFF
1506 // Exclude surrogate code points (U+D800 to U+DFFF)
1507 // https://www.unicode.org/versions/Unicode15.1.0/ch03.pdf (Section 3.8, D71)
1508 // Exclude control chars below U+0020 for readability
1509 // Generate from two ranges with equal probability (overrepresents the smaller
1510 // upper range):
1511 // - Lower: U+0020 to U+D7FF (55,776 values, 50% selection probability)
1512 // - Upper: U+E000 to U+FFFD (8,190 values, 50% selection probability)
1513 if (bmp_range_dist(gen)) {
1514 // Lower range: U+0020 to U+D7FF (before surrogate range)
1515 codepoint = bmp_lower_dist(gen);
1516 } else {
1517 // Upper range: U+E000 to U+FFFD (after surrogate range)
1518 // Note: Stops at U+FFFD to exclude noncharacters U+FFFE and U+FFFF
1519 // Other noncharacters (U+FDD0-U+FDEF, plane-ending pairs) are included
1520 // as they are valid Unicode scalar values per the Unicode Standard
1521 codepoint = bmp_upper_dist(gen);
1522 }
1523 } else if (plane == 1) {
1524 // Supplementary Multilingual Plane (SMP): U+10000 to U+1FFFF
1525 // https://www.unicode.org/roadmaps/smp/
1526 codepoint = smp_dist(gen);
1527 } else if (plane == 2) {
1528 // Supplementary Ideographic Plane (SIP): U+20000 to U+2FFFF
1529 // https://www.unicode.org/roadmaps/sip/
1530 codepoint = sip_dist(gen);
1531 } else {
1532 // Planes 3–16: U+30000–U+10FFFF
1533 // Includes TIP, SSP, PUA-A, PUA-B, and unassigned planes: U+30000 to U+10FFFF
1534 // Max valid Unicode codepoint is U+10FFFF per the Standard
1535 // https://www.unicode.org/versions/Unicode15.1.0/ch03.pdf (Section 3.4, D9)
1536 codepoint = high_plane_dist(gen);
1537 }
1538
1539 // Encode as UTF-8 per RFC 3629 (Section 3: UTF-8 definition)
1540 // https://www.rfc-editor.org/rfc/rfc3629.html#section-3
1541 if (codepoint <= 0x7F) {
1542 // 1-byte sequence: 0xxxxxxx
1543 s.push_back(static_cast<char>(codepoint));
1544 } else if (codepoint <= 0x7FF) {

Callers 1

GenerateUtf8Method · 0.85

Calls 2

push_backMethod · 0.80
reserveMethod · 0.45

Tested by 1

GenerateUtf8Method · 0.68