| 947 | } |
| 948 | |
| 949 | bool process_token(completion_token_output &result, llama_client_slot &slot) { |
| 950 | // remember which tokens were sampled - used for repetition penalties during sampling |
| 951 | const std::string token_str = llama_token_to_piece(ctx, result.tok); |
| 952 | slot.sampled = result.tok; |
| 953 | |
| 954 | // search stop word and delete it |
| 955 | slot.generated_text += token_str; |
| 956 | slot.has_next_token = true; |
| 957 | |
| 958 | if (slot.multibyte_pending > 0) |
| 959 | { |
| 960 | slot.multibyte_pending -= token_str.size(); |
| 961 | } |
| 962 | else if (token_str.size() == 1) |
| 963 | { |
| 964 | const char c = token_str[0]; |
| 965 | // 2-byte characters: 110xxxxx 10xxxxxx |
| 966 | if ((c & 0xE0) == 0xC0) |
| 967 | { |
| 968 | slot.multibyte_pending = 1; |
| 969 | // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx |
| 970 | } |
| 971 | else if ((c & 0xF0) == 0xE0) |
| 972 | { |
| 973 | slot.multibyte_pending = 2; |
| 974 | // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx |
| 975 | } |
| 976 | else if ((c & 0xF8) == 0xF0) |
| 977 | { |
| 978 | slot.multibyte_pending = 3; |
| 979 | } |
| 980 | else |
| 981 | { |
| 982 | slot.multibyte_pending = 0; |
| 983 | } |
| 984 | } |
| 985 | |
| 986 | if (slot.multibyte_pending == 0) |
| 987 | { |
| 988 | size_t pos = std::min(slot.sent_count, slot.generated_text.size()); |
| 989 | const std::string str_test = slot.generated_text.substr(pos); |
| 990 | bool is_stop_full = false; |
| 991 | size_t stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_FULL, slot); |
| 992 | if (stop_pos != std::string::npos) |
| 993 | { |
| 994 | is_stop_full = true; |
| 995 | slot.generated_text.erase( |
| 996 | slot.generated_text.begin() + pos + stop_pos, |
| 997 | slot.generated_text.end()); |
| 998 | pos = std::min(slot.sent_count, slot.generated_text.size()); |
| 999 | } |
| 1000 | else |
| 1001 | { |
| 1002 | is_stop_full = false; |
| 1003 | stop_pos = find_stopping_strings(str_test, token_str.size(), STOP_PARTIAL, slot); |
| 1004 | } |
| 1005 | |
| 1006 | // check if there is any token to predict |
nothing calls this directly
no test coverage detected