| 1078 | } |
| 1079 | |
| 1080 | std::string llama_detokenize_bpe(llama_context * ctx, const std::vector<llama_token> & tokens) { |
| 1081 | std::string piece; |
| 1082 | std::string result; |
| 1083 | |
| 1084 | for (size_t i = 0; i < tokens.size(); ++i) { |
| 1085 | piece = llama_token_to_piece(ctx, tokens[i]); |
| 1086 | |
| 1087 | result += piece; |
| 1088 | } |
| 1089 | |
| 1090 | // NOTE: the original tokenizer decodes bytes after collecting the pieces. |
| 1091 | return result; |
| 1092 | } |
| 1093 | |
| 1094 | // |
| 1095 | // YAML utils |