| 952 | static bool g_force_reconstruction_test = false; |
| 953 | |
| 954 | static void test_peg_parser(common_chat_templates * tmpls, |
| 955 | const std::function<void(peg_test_case &)> & init, |
| 956 | bool detailed_debug) { |
| 957 | // UTF-8-safe truncation helper (same as in test_parser_with_streaming) |
| 958 | constexpr auto utf8_truncate_safe_len = [](const std::string_view s) -> size_t { |
| 959 | auto len = s.size(); |
| 960 | if (len == 0) { |
| 961 | return 0; |
| 962 | } |
| 963 | auto i = len; |
| 964 | for (size_t back = 0; back < 4 && i > 0; ++back) { |
| 965 | --i; |
| 966 | unsigned char c = s[i]; |
| 967 | if ((c & 0x80) == 0) { |
| 968 | return len; |
| 969 | } |
| 970 | if ((c & 0xC0) == 0xC0) { |
| 971 | size_t expected_len = 0; |
| 972 | if ((c & 0xE0) == 0xC0) { |
| 973 | expected_len = 2; |
| 974 | } else if ((c & 0xF0) == 0xE0) { |
| 975 | expected_len = 3; |
| 976 | } else if ((c & 0xF8) == 0xF0) { |
| 977 | expected_len = 4; |
| 978 | } else { |
| 979 | return i; |
| 980 | } |
| 981 | if (len - i >= expected_len) { |
| 982 | return len; |
| 983 | } |
| 984 | return i; |
| 985 | } |
| 986 | } |
| 987 | return len - std::min(len, size_t(3)); |
| 988 | }; |
| 989 | |
| 990 | peg_test_case tc; |
| 991 | init(tc); |
| 992 | if (tc.params.messages.empty()) { |
| 993 | tc.params.messages = { message_user }; |
| 994 | } |
| 995 | if (tc.expect.role.empty()) { |
| 996 | tc.expect.role = "assistant"; |
| 997 | } |
| 998 | |
| 999 | auto parser = make_peg_parser(tmpls, tc.params, detailed_debug); |
| 1000 | if (detailed_debug) { |
| 1001 | LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str()); |
| 1002 | LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str()); |
| 1003 | } |
| 1004 | |
| 1005 | common_chat_msg msg_accum; |
| 1006 | common_chat_msg msg_prev; |
| 1007 | msg_accum.role = msg_prev.role = "assistant"; |
| 1008 | |
| 1009 | for (size_t i = 1; i <= tc.input.size(); ++i) { |
| 1010 | auto is_partial = i < tc.input.size() || tc.is_partial; |
| 1011 | // Use UTF-8 safe truncation to avoid corrupting multi-byte characters |
no test coverage detected