Function test_utf8_case_fold_fuzz

scripts/test_stringzilla.cpp:679–757

* @brief Exhaustive fuzz test for UTF-8 case folding using all Unicode codepoints. * * - First run: Tests all valid codepoints in order (0x0 to 0x10FFFF). * - Subsequent runs: Shuffles the codepoints to create random sequences. * - Checks for both length and content matches between serial and SIMD implementations. */

 *  - Checks for both length and content matches between serial and SIMD implementations.
 */
void test_utf8_case_fold_fuzz(sz_utf8_case_fold_t fold_base, sz_utf8_case_fold_t fold_simd,
                              std::size_t iterations = scale_iterations(100)) {
    std::printf("  - testing case folding fuzz (%zu iterations + ordered check)...\n", iterations);

    // 1. Generate all valid codepoints (ordered initially)
    std::vector<sz_rune_t> all_runes;
    all_runes.reserve(0x10FFFF);
    for (sz_rune_t cp = 0; cp <= 0x10FFFF; ++cp) {
        if (cp >= 0xD800 && cp <= 0xDFFF) continue; // Skip surrogates
        all_runes.push_back(cp);
    }

    // 2. Prepare buffers
    // Max UTF-8 size is 4 bytes per rune.
    std::vector<char> input_buffer(all_runes.size() * 4);
    // Max expansion is handled by the folding functions, usually 3x is safe for worst cases (like µ)
    std::vector<char> output_base(input_buffer.size() * 3 + 64);
    std::vector<char> output_simd(input_buffer.size() * 3 + 64);

    auto &rng = global_random_generator();

    // Iterate: 0 = Ordered, 1..N = Shuffled
    for (std::size_t it = 0; it <= iterations; ++it) {
        if (it > 0) std::shuffle(all_runes.begin(), all_runes.end(), rng);

        // Convert to UTF-8
        char *data_ptr = input_buffer.data();
        for (sz_rune_t cp : all_runes) data_ptr += sz_rune_export(cp, (sz_u8_t *)data_ptr);
        sz_size_t input_len = data_ptr - input_buffer.data();

        // Run tests
        sz_size_t len_base = fold_base(input_buffer.data(), input_len, output_base.data());
        sz_size_t len_simd = fold_simd(input_buffer.data(), input_len, output_simd.data());

        // Validations
        if (len_base != len_simd) {
            std::fprintf(stderr, "Iteration %zu: Length mismatch base=%zu simd=%zu\n", it, len_base, len_simd);
            assert(false);
        }

        // Optimize: Memcmp first, then diagnose
        if (std::memcmp(output_base.data(), output_simd.data(), len_base) != 0) {
            std::fprintf(stderr, "Iteration %zu: Content mismatch\n", it);
            // Find first mismatch for debug
            for (std::size_t i = 0; i < len_base; ++i) {
                if (output_base[i] != output_simd[i]) {
                    std::fprintf(stderr, "Mismatch at byte %zu: 0x%02X vs 0x%02X\n", i, (unsigned char)output_base[i],
                                 (unsigned char)output_simd[i]);

                    // Show context (16 bytes before/after)
                    sz_size_t start = (i > 16) ? i - 16 : 0;
                    sz_size_t end = (i + 16 < len_base) ? i + 16 : len_base;

                    std::fprintf(stderr, "Context (Base): ");
                    for (sz_size_t j = start; j < end; ++j)
                        std::fprintf(stderr, "%02X ", (unsigned char)output_base[j]);
                    std::fprintf(stderr, "\n");