Method testCase

core/llm/llamaTokenizer.js:420–428

(inputString, expectedTokenIds)

    }

    function testCase(inputString, expectedTokenIds) {
      const actualTokens = tokenizer.encode(inputString, true, true, true);
      if (!isEqual(actualTokens, expectedTokenIds)) {
        throw `Test failed. LLaMA Tokenizer Encoder returned unexpected result: expected tokenize(${inputString}) === ${expectedTokenIds}, actual was: ${actualTokens}`;
      }
      if (inputString !== tokenizer.decode(actualTokens)) {
        throw `Test failed. LLaMA Tokenizer Decoder returned unexpected result: expected decode(${actualTokens}) === ${inputString}, actual was: ${decode(actualTokens)}`;
      }
    }

    // Simple test case
    testCase("grabbed", [1, 2646, 1327, 287]);