(stream, state)
| 93 | |
| 94 | // Main function |
| 95 | function tokenize(stream, state) { |
| 96 | // Finally advance the stream |
| 97 | var ch = stream.next(); |
| 98 | |
| 99 | // BLOCKCOMMENT |
| 100 | if (ch === '/' && stream.eat('*')) { |
| 101 | state.continueComment = true; |
| 102 | return "comment"; |
| 103 | } else if (state.continueComment === true) { // in comment block |
| 104 | //comment ends at the beginning of the line |
| 105 | if (ch === '*' && stream.peek() === '/') { |
| 106 | stream.next(); |
| 107 | state.continueComment = false; |
| 108 | } else if (stream.skipTo('*')) { //comment is potentially later in line |
| 109 | stream.skipTo('*'); |
| 110 | stream.next(); |
| 111 | if (stream.eat('/')) |
| 112 | state.continueComment = false; |
| 113 | } else { |
| 114 | stream.skipToEnd(); |
| 115 | } |
| 116 | return "comment"; |
| 117 | } |
| 118 | |
| 119 | if (ch == "*" && stream.column() == stream.indentation()) { |
| 120 | stream.skipToEnd() |
| 121 | return "comment" |
| 122 | } |
| 123 | |
| 124 | // DoubleOperator match |
| 125 | var doubleOperator = ch + stream.peek(); |
| 126 | |
| 127 | if ((ch === '"' || ch === "'") && !state.continueString) { |
| 128 | state.continueString = ch |
| 129 | return "string" |
| 130 | } else if (state.continueString) { |
| 131 | if (state.continueString == ch) { |
| 132 | state.continueString = null; |
| 133 | } else if (stream.skipTo(state.continueString)) { |
| 134 | // quote found on this line |
| 135 | stream.next(); |
| 136 | state.continueString = null; |
| 137 | } else { |
| 138 | stream.skipToEnd(); |
| 139 | } |
| 140 | return "string"; |
| 141 | } else if (state.continueString !== null && stream.eol()) { |
| 142 | stream.skipTo(state.continueString) || stream.skipToEnd(); |
| 143 | return "string"; |
| 144 | } else if (/[\d\.]/.test(ch)) { //find numbers |
| 145 | if (ch === ".") |
| 146 | stream.match(/^[0-9]+([eE][\-+]?[0-9]+)?/); |
| 147 | else if (ch === "0") |
| 148 | stream.match(/^[xX][0-9a-fA-F]+/) || stream.match(/^0[0-7]+/); |
| 149 | else |
| 150 | stream.match(/^[0-9]*\.?[0-9]*([eE][\-+]?[0-9]+)?/); |
| 151 | return "number"; |
| 152 | } else if (isDoubleOperatorChar.test(ch + stream.peek())) { // TWO SYMBOL TOKENS |
no test coverage detected