Method decode

core/llm/llamaTokenizer.js:387–408

(tokenIds, add_bos_token = true, add_preceding_space = true)

  }

  decode(tokenIds, add_bos_token = true, add_preceding_space = true) {
    const utf8byteVals = [];
    const startIndex = add_bos_token ? 1 : 0;
    for (let i = startIndex; i < tokenIds.length; i++) {
      const tokenId = tokenIds[i];
      const tokenString = this.vocabById[tokenId];
      if (tokenString.startsWith("<0x") && tokenString.endsWith(">")) {
        // Special case
        const utf8byte = hexToUtf8Byte(tokenString);
        utf8byteVals.push(utf8byte);
      } else {
        // Typical case
        const utf8bytes = this.utf8Encoder.encode(tokenString);
        utf8bytes.forEach((utf8Byte) => utf8byteVals.push(utf8Byte));
      }
    }
    const uint8Array = new Uint8Array(utf8byteVals);
    const decodedString = this.utf8Decoder.decode(uint8Array);
    const spacesFixed = decodedString.replaceAll(this.vocabById[29871], " ");
    // Note that preceding space must be removed here at string level, not earlier at token level, because multiple consecutive spaces are represented as single token.
    return add_preceding_space ? spacesFixed.slice(1) : spacesFixed;
  }

  defaultTests(tokenizer) {
    function isEqual(arr1, arr2) {