Method encode

src/tokenizer.cpp:57–94

}

std::vector<int> Tokenizer::encode(const std::string& text, bool encode_bos) const {
  std::vector<int> out_tokens;
  if (encode_bos) {
    out_tokens.push_back(bos_id);
  }

  for (size_t i = 0; i < text.size();) {
    size_t l = 0;
    size_t valid_l = 0;
    const TokenTrie* p = &vocab_trie;
    const TokenTrie* valid_p = nullptr;
    while (i + l < text.size()) {
      char c = text[i+l];
      if (p->children.count(c)) {
        p = p->children.at(c).get();
        l += 1;
        if (p->token_id >= 0) {
          valid_p = p;
          valid_l = l;
        }
      } else {
        break;
      }
    }
    if (!valid_p) {
      // No substring starting from `i` matches any vocab words, use byte fallback
      if (byte_fallback_start >= 0) {
        out_tokens.push_back((unsigned char)text[i] + byte_fallback_start);
      }
      i += 1;
    } else {
      out_tokens.push_back(valid_p->token_id);
      i += valid_l;
    }
  }

  return out_tokens;
}

std::string Tokenizer::encoding_to_debug_string(const std::vector<int>& encoding) const {
  std::string token_encoding_debug_str = "";