| 9798 | } |
| 9799 | |
| 9800 | function tokenize(code, options) { |
| 9801 | var toString, |
| 9802 | token, |
| 9803 | tokens; |
| 9804 | |
| 9805 | toString = String; |
| 9806 | if (typeof code !== 'string' && !(code instanceof String)) { |
| 9807 | code = toString(code); |
| 9808 | } |
| 9809 | |
| 9810 | delegate = SyntaxTreeDelegate; |
| 9811 | source = code; |
| 9812 | index = 0; |
| 9813 | lineNumber = (source.length > 0) ? 1 : 0; |
| 9814 | lineStart = 0; |
| 9815 | length = source.length; |
| 9816 | lookahead = null; |
| 9817 | state = { |
| 9818 | allowKeyword: true, |
| 9819 | allowIn: true, |
| 9820 | labelSet: new StringMap(), |
| 9821 | inFunctionBody: false, |
| 9822 | inIteration: false, |
| 9823 | inSwitch: false, |
| 9824 | lastCommentStart: -1 |
| 9825 | }; |
| 9826 | |
| 9827 | extra = {}; |
| 9828 | |
| 9829 | // Options matching. |
| 9830 | options = options || {}; |
| 9831 | |
| 9832 | // Of course we collect tokens here. |
| 9833 | options.tokens = true; |
| 9834 | extra.tokens = []; |
| 9835 | extra.tokenize = true; |
| 9836 | // The following two fields are necessary to compute the Regex tokens. |
| 9837 | extra.openParenToken = -1; |
| 9838 | extra.openCurlyToken = -1; |
| 9839 | |
| 9840 | extra.range = (typeof options.range === 'boolean') && options.range; |
| 9841 | extra.loc = (typeof options.loc === 'boolean') && options.loc; |
| 9842 | |
| 9843 | if (typeof options.comment === 'boolean' && options.comment) { |
| 9844 | extra.comments = []; |
| 9845 | } |
| 9846 | if (typeof options.tolerant === 'boolean' && options.tolerant) { |
| 9847 | extra.errors = []; |
| 9848 | } |
| 9849 | |
| 9850 | patch(); |
| 9851 | |
| 9852 | try { |
| 9853 | peek(); |
| 9854 | if (lookahead.type === Token.EOF) { |
| 9855 | return extra.tokens; |
| 9856 | } |
| 9857 | |