| 1058 | } |
| 1059 | |
| 1060 | std::string llama_detokenize_spm(llama_context * ctx, const std::vector<llama_token> & tokens) { |
| 1061 | const llama_token bos_id = llama_token_bos(llama_get_model(ctx)); |
| 1062 | |
| 1063 | std::string piece; |
| 1064 | std::string result; |
| 1065 | |
| 1066 | for (size_t i = 0; i < tokens.size(); ++i) { |
| 1067 | piece = llama_token_to_piece(ctx, tokens[i]); |
| 1068 | |
| 1069 | // remove the leading space of the first non-BOS token |
| 1070 | if (((tokens[0] == bos_id && i == 1) || (tokens[0] != bos_id && i == 0)) && piece[0] == ' ') { |
| 1071 | piece = piece.substr(1); |
| 1072 | } |
| 1073 | |
| 1074 | result += piece; |
| 1075 | } |
| 1076 | |
| 1077 | return result; |
| 1078 | } |
| 1079 | |
| 1080 | std::string llama_detokenize_bpe(llama_context * ctx, const std::vector<llama_token> & tokens) { |
| 1081 | std::string piece; |