File tokenizer.test.ts

src/test/suite/tokenizer.test.ts:None–None

import * as assert from "assert";
import { tokenize } from "../../core/tokenizer";
import { flatten, range } from "lodash";