Function tokenize

src/esprima.ts:94–120

(code: string, options, delegate)

}

export function tokenize(code: string, options, delegate) {
    const tokenizer = new Tokenizer(code, options);

    let tokens;
    tokens = [];

    try {
        while (true) {
            let token = tokenizer.getNextToken();
            if (!token) {
                break;
            }
            if (delegate) {
                token = delegate(token);
            }
            tokens.push(token);
        }
    } catch (e) {
        tokenizer.errorHandler.tolerate(e);
    }

    if (tokenizer.errorHandler.tolerant) {
        tokens.errors = tokenizer.errors();
    }

    return tokens;
}

export { Syntax } from './syntax';