| 450 | } |
| 451 | |
| 452 | int32_t tokenize(mtmd_input_chunks * output) { |
| 453 | cur.entries.clear(); |
| 454 | std::vector<std::string> parts = split_text(input_text, ctx->media_marker); |
| 455 | size_t i_bm = 0; // index of the current bitmap |
| 456 | for (auto & part : parts) { |
| 457 | if (part == ctx->media_marker) { |
| 458 | // this is a marker, we should add the next bitmap |
| 459 | if (i_bm >= bitmaps.size()) { |
| 460 | LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n", |
| 461 | __func__, bitmaps.size(), parts.size() - 1); |
| 462 | return 1; |
| 463 | } |
| 464 | const mtmd_bitmap * bitmap = bitmaps[i_bm++]; |
| 465 | int32_t res = add_media(bitmap); |
| 466 | if (res != 0) { |
| 467 | return res; |
| 468 | } |
| 469 | } else { |
| 470 | // this is a text part, we should add it as text |
| 471 | add_text(part, parse_special); |
| 472 | } |
| 473 | } |
| 474 | |
| 475 | if (add_special && llama_vocab_get_add_bos(vocab)) { |
| 476 | // if first chunk is text, we add BOS token to first text chunk |
| 477 | // otherwise, create a new text chunk with BOS token |
| 478 | if (!cur.entries.empty() && cur.entries[0].type == MTMD_INPUT_CHUNK_TYPE_TEXT) { |
| 479 | // add BOS token to the beginning of first text chunk |
| 480 | cur.entries[0].tokens_text.insert(cur.entries[0].tokens_text.begin(), llama_vocab_bos(vocab)); |
| 481 | } else { |
| 482 | // create a new text chunk with BOS token at the beginning |
| 483 | mtmd_input_chunk bos_chunk{ |
| 484 | MTMD_INPUT_CHUNK_TYPE_TEXT, |
| 485 | {llama_vocab_bos(vocab)}, |
| 486 | nullptr, // image tokens |
| 487 | nullptr, // audio tokens |
| 488 | }; |
| 489 | cur.entries.insert(cur.entries.begin(), std::move(bos_chunk)); |
| 490 | } |
| 491 | } |
| 492 | |
| 493 | if (add_special && llama_vocab_get_add_eos(vocab)) { |
| 494 | // if last chunk is text, we add EOS token to it |
| 495 | add_text({llama_vocab_eos(vocab)}); |
| 496 | } |
| 497 | |
| 498 | if (i_bm != bitmaps.size()) { |
| 499 | LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n", |
| 500 | __func__, bitmaps.size(), parts.size() - 1); |
| 501 | return 1; |
| 502 | } |
| 503 | |
| 504 | *output = std::move(cur); |
| 505 | |
| 506 | return 0; |
| 507 | } |
| 508 | |
| 509 | void add_text(const std::string & txt, bool parse_special) { |
no test coverage detected