| 437 | } |
| 438 | |
| 439 | static std::vector<llama_token> prepare_guide_tokens(const llama_vocab * vocab, const std::string & str, const outetts_version tts_version = OUTETTS_V0_2) { |
| 440 | const std::string& delimiter = (tts_version == OUTETTS_V0_3 ? "<|space|>" : "<|text_sep|>"); |
| 441 | |
| 442 | std::vector<llama_token> result; |
| 443 | size_t start = 0; |
| 444 | size_t end = str.find(delimiter); |
| 445 | |
| 446 | //first token is always a newline, as it was not previously added |
| 447 | result.push_back(common_tokenize(vocab, "\n", false, true)[0]); |
| 448 | |
| 449 | while (end != std::string::npos) { |
| 450 | std::string current_word = str.substr(start, end - start); |
| 451 | auto tmp = common_tokenize(vocab, current_word, false, true); |
| 452 | result.push_back(tmp[0]); |
| 453 | start = end + delimiter.length(); |
| 454 | end = str.find(delimiter, start); |
| 455 | } |
| 456 | |
| 457 | // Add the last part |
| 458 | std::string current_word = str.substr(start); |
| 459 | auto tmp = common_tokenize(vocab, current_word, false, true); |
| 460 | if (tmp.size() > 0) { |
| 461 | result.push_back(tmp[0]); |
| 462 | } |
| 463 | return result; |
| 464 | } |
| 465 | |
| 466 | static json speaker_from_file(const std::string & speaker_file) { |
| 467 | std::ifstream file(speaker_file); |