File tokenizer.worker.ts

src/node/utils/main/tokenizer.worker.ts:None–None

import assert from "node:assert";
import { parentPort } from "node:worker_threads";
import { Tokenizer, models } from "ai-tokenizer";
import type { ModelName } from "ai-tokenizer";