| 1485 | } |
| 1486 | |
| 1487 | std::string RandomUtf8String(random::SeedType seed, int num_chars) { |
| 1488 | arrow::random::pcg32 gen(seed); |
| 1489 | std::string s; |
| 1490 | s.reserve(num_chars * 3); // Reserve for average 3 bytes per codepoint |
| 1491 | |
| 1492 | std::uniform_int_distribution<uint32_t> plane_dist(0, 3); |
| 1493 | std::bernoulli_distribution bmp_range_dist(0.5); |
| 1494 | std::uniform_int_distribution<uint32_t> bmp_lower_dist(0x0020, 0xD7FF); |
| 1495 | std::uniform_int_distribution<uint32_t> bmp_upper_dist(0xE000, 0xFFFD); |
| 1496 | std::uniform_int_distribution<uint32_t> smp_dist(0x10000, 0x1FFFF); |
| 1497 | std::uniform_int_distribution<uint32_t> sip_dist(0x20000, 0x2FFFF); |
| 1498 | std::uniform_int_distribution<uint32_t> high_plane_dist(0x30000, 0x10FFFF); |
| 1499 | |
| 1500 | for (int i = 0; i < num_chars; ++i) { |
| 1501 | uint32_t codepoint; |
| 1502 | uint32_t plane = plane_dist(gen); |
| 1503 | |
| 1504 | if (plane == 0) { |
| 1505 | // Basic Multilingual Plane (BMP): U+0000 to U+FFFF |
| 1506 | // Exclude surrogate code points (U+D800 to U+DFFF) |
| 1507 | // https://www.unicode.org/versions/Unicode15.1.0/ch03.pdf (Section 3.8, D71) |
| 1508 | // Exclude control chars below U+0020 for readability |
| 1509 | // Generate from two ranges with equal probability (overrepresents the smaller |
| 1510 | // upper range): |
| 1511 | // - Lower: U+0020 to U+D7FF (55,776 values, 50% selection probability) |
| 1512 | // - Upper: U+E000 to U+FFFD (8,190 values, 50% selection probability) |
| 1513 | if (bmp_range_dist(gen)) { |
| 1514 | // Lower range: U+0020 to U+D7FF (before surrogate range) |
| 1515 | codepoint = bmp_lower_dist(gen); |
| 1516 | } else { |
| 1517 | // Upper range: U+E000 to U+FFFD (after surrogate range) |
| 1518 | // Note: Stops at U+FFFD to exclude noncharacters U+FFFE and U+FFFF |
| 1519 | // Other noncharacters (U+FDD0-U+FDEF, plane-ending pairs) are included |
| 1520 | // as they are valid Unicode scalar values per the Unicode Standard |
| 1521 | codepoint = bmp_upper_dist(gen); |
| 1522 | } |
| 1523 | } else if (plane == 1) { |
| 1524 | // Supplementary Multilingual Plane (SMP): U+10000 to U+1FFFF |
| 1525 | // https://www.unicode.org/roadmaps/smp/ |
| 1526 | codepoint = smp_dist(gen); |
| 1527 | } else if (plane == 2) { |
| 1528 | // Supplementary Ideographic Plane (SIP): U+20000 to U+2FFFF |
| 1529 | // https://www.unicode.org/roadmaps/sip/ |
| 1530 | codepoint = sip_dist(gen); |
| 1531 | } else { |
| 1532 | // Planes 3–16: U+30000–U+10FFFF |
| 1533 | // Includes TIP, SSP, PUA-A, PUA-B, and unassigned planes: U+30000 to U+10FFFF |
| 1534 | // Max valid Unicode codepoint is U+10FFFF per the Standard |
| 1535 | // https://www.unicode.org/versions/Unicode15.1.0/ch03.pdf (Section 3.4, D9) |
| 1536 | codepoint = high_plane_dist(gen); |
| 1537 | } |
| 1538 | |
| 1539 | // Encode as UTF-8 per RFC 3629 (Section 3: UTF-8 definition) |
| 1540 | // https://www.rfc-editor.org/rfc/rfc3629.html#section-3 |
| 1541 | if (codepoint <= 0x7F) { |
| 1542 | // 1-byte sequence: 0xxxxxxx |
| 1543 | s.push_back(static_cast<char>(codepoint)); |
| 1544 | } else if (codepoint <= 0x7FF) { |