* @brief Tests UTF-8 functions across different SIMD backends against the serial implementation. * * Generates random strings containing: * - ASCII content (1-byte) * - Multi-byte UTF-8 characters (2, 3, 4-byte) - correct and broken ones * - All 25 Unicode White_Space characters (including all newlines) + CRLF sequences - correct and partial ones * * For each generated string, compar
| 401 | * - sz_utf8_find_whitespace: whitespace detection (position and matched length) |
| 402 | */ |
| 403 | void test_utf8_equivalence( // |
| 404 | sz_utf8_count_t count_base, sz_utf8_count_t count_simd, // |
| 405 | sz_utf8_find_boundary_t newline_base, // |
| 406 | sz_utf8_find_boundary_t newline_simd, // |
| 407 | sz_utf8_find_boundary_t whitespace_base, // |
| 408 | sz_utf8_find_boundary_t whitespace_simd, // |
| 409 | std::size_t min_text_length = 4000, std::size_t min_iterations = scale_iterations(10000)) { |
| 410 | |
| 411 | auto check = [&](std::string const &text) { |
| 412 | sz_cptr_t data = text.data(); |
| 413 | sz_size_t len = text.size(); |
| 414 | |
| 415 | // Test `sz_utf8_count` equivalence |
| 416 | sz_size_t count_result_base = count_base(data, len); |
| 417 | sz_size_t count_result_simd = count_simd(data, len); |
| 418 | assert(count_result_base == count_result_simd); |
| 419 | |
| 420 | // Test `sz_utf8_find_newline` equivalence by scanning the entire string |
| 421 | sz_cptr_t pos = data; |
| 422 | sz_size_t remaining = len; |
| 423 | while (remaining > 0) { |
| 424 | sz_size_t matched_base = 0, matched_simd = 0; |
| 425 | sz_cptr_t found_base = newline_base(pos, remaining, &matched_base); |
| 426 | sz_cptr_t found_simd = newline_simd(pos, remaining, &matched_simd); |
| 427 | assert(found_base == found_simd && "Mismatch in newline detection"); |
| 428 | if (found_base == SZ_NULL_CHAR) break; |
| 429 | assert(matched_base == matched_simd); |
| 430 | sz_size_t offset = (found_base - pos) + matched_base; |
| 431 | pos += offset; |
| 432 | remaining -= offset; |
| 433 | } |
| 434 | |
| 435 | // Test `sz_utf8_find_whitespace` equivalence by scanning the entire string |
| 436 | pos = data; |
| 437 | remaining = len; |
| 438 | while (remaining > 0) { |
| 439 | sz_size_t matched_base = 0, matched_simd = 0; |
| 440 | sz_cptr_t found_base = whitespace_base(pos, remaining, &matched_base); |
| 441 | sz_cptr_t found_simd = whitespace_simd(pos, remaining, &matched_simd); |
| 442 | assert(found_base == found_simd && "Mismatched position in whitespace detection"); |
| 443 | if (found_base == SZ_NULL_CHAR) break; |
| 444 | assert(matched_base == matched_simd); |
| 445 | sz_size_t offset = (found_base - pos) + matched_base; |
| 446 | pos += offset; |
| 447 | remaining -= offset; |
| 448 | } |
| 449 | }; |
| 450 | |
| 451 | // Strings that shouldn't affect the control flow |
| 452 | static char const *utf8_content[] = { |
| 453 | // Various ASCII strings |
| 454 | "", |
| 455 | "a", |
| 456 | "hello", |
| 457 | "012", |
| 458 | "3456789", |
| 459 | // 2-byte Cyrillic П (U+041F), Armenian Ս (U+054D), and Greek Pi π (U+03C0) |
| 460 | "\xD0\x9F", |
no test coverage detected
searching dependent graphs…