Function encode

core/llm/tiktokenWorkerPool.mjs:6–8

(text)

const tiktokenEncoding = _encodingForModel("gpt-4");

function encode(text) {
  return tiktokenEncoding.encode(text, "all", []);
}

function decode(tokens) {
  return tiktokenEncoding.decode(tokens);