Method decode_one

src/tokenizer.cpp:44–55

}

std::string Tokenizer::decode_one(int prev_token, int token) const {
  const std::string& piece = vocab[token];
  // if following BOS token, sentencepiece decoder strips any leading whitespace
  if (prev_token == bos_id && piece[0] == ' ') {
    return piece.substr(1);
  }
  // return byte piece for byte fallback tokens (<0x00>, <0x01>, ..., <0xFF>)
  if (byte_fallback_start >= 0 && token >= byte_fallback_start && (token - byte_fallback_start) < 256) {
    return byte_pieces[token - byte_fallback_start];
  }
  return piece;
}

std::vector<int> Tokenizer::encode(const std::string& text, bool encode_bos) const {
  std::vector<int> out_tokens;