Function test_unicode

tests/peg-parser/test-unicode.cpp:26–446

}

void test_unicode(testing &t) {
    struct test_case {
        std::string input;
        std::string expected_text;
        common_peg_parse_result_type expected_result;
    };

    t.test("any", [](testing &t) {
        std::vector<test_case> test_cases {
            // Valid UTF-8 sequences
            {"Hello", "Hello", COMMON_PEG_PARSE_RESULT_SUCCESS},
            {std::string("Caf\xC3\xA9"), std::string("Caf\xC3\xA9"), COMMON_PEG_PARSE_RESULT_SUCCESS},
            {std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), std::string("\xE4\xBD\xA0\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS},
            {std::string("\xF0\x9F\x9A\x80"), std::string("\xF0\x9F\x9A\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS},

            // Incomplete UTF-8 sequences (partial bytes at end)
            {std::string("Caf\xC3"), "Caf", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
            {std::string("\xE4\xBD"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},
            {std::string("\xF0\x9F\x9A"), "", COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT},

            // Invalid/malformed UTF-8 sequences
            {std::string("\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL},
            {std::string("Hello\x80World"), "Hello", COMMON_PEG_PARSE_RESULT_FAIL},
            {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
        };

        auto parser = build_peg_parser([](common_peg_parser_builder& p) {
            return p.sequence({p.one_or_more(p.any()), p.end()});
        });

        for (size_t i = 0; i < test_cases.size(); i++) {
            const auto & tc = test_cases[i];
            std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);

            t.test(test_name, [&](testing &t) {
                common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
                auto result = parser.parse(ctx);

                // Assert result type matches
                assert_result_equal(t, tc.expected_result, result.type);

                // Assert matched text if success or need_more_input
                if (result.success() || result.need_more_input()) {
                    std::string matched = tc.input.substr(result.start, result.end - result.start);
                    t.assert_equal(tc.expected_text, matched);
                }
            });
        }
    });

    t.test("char classes", [](testing &t) {
        t.test("unicode range U+4E00-U+9FFF (CJK)", [](testing &t) {
            std::vector<test_case> test_cases {
                // Within range - CJK Unified Ideographs
                {std::string("\xE4\xB8\x80"), std::string("\xE4\xB8\x80"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4E00
                {std::string("\xE4\xBD\xA0"), std::string("\xE4\xBD\xA0"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+4F60
                {std::string("\xE5\xA5\xBD"), std::string("\xE5\xA5\xBD"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+597D
                {std::string("\xE9\xBF\xBF"), std::string("\xE9\xBF\xBF"), COMMON_PEG_PARSE_RESULT_SUCCESS}, // U+9FFF