(stream, state)
| 133 | } |
| 134 | |
| 135 | function tokenize(stream, state) { |
| 136 | if (stream.eatSpace()) { |
| 137 | return rval(state,stream,"whitespace"); |
| 138 | } |
| 139 | |
| 140 | // attributes and type specs |
| 141 | if ((peekToken(state).token == "" || peekToken(state).token == ".") && |
| 142 | stream.peek() == '-') { |
| 143 | stream.next(); |
| 144 | if (stream.eat(smallRE) && stream.eatWhile(anumRE)) { |
| 145 | if (isMember(stream.current(),typeWords)) { |
| 146 | return rval(state,stream,"type"); |
| 147 | }else{ |
| 148 | return rval(state,stream,"attribute"); |
| 149 | } |
| 150 | } |
| 151 | stream.backUp(1); |
| 152 | } |
| 153 | |
| 154 | var ch = stream.next(); |
| 155 | |
| 156 | // comment |
| 157 | if (ch == '%') { |
| 158 | stream.skipToEnd(); |
| 159 | return rval(state,stream,"comment"); |
| 160 | } |
| 161 | |
| 162 | // macro |
| 163 | if (ch == '?') { |
| 164 | stream.eatWhile(anumRE); |
| 165 | return rval(state,stream,"macro"); |
| 166 | } |
| 167 | |
| 168 | // record |
| 169 | if ( ch == "#") { |
| 170 | stream.eatWhile(anumRE); |
| 171 | return rval(state,stream,"record"); |
| 172 | } |
| 173 | |
| 174 | // char |
| 175 | if ( ch == "$") { |
| 176 | if (stream.next() == "\\") { |
| 177 | if (!stream.eatWhile(octitRE)) { |
| 178 | stream.next(); |
| 179 | } |
| 180 | } |
| 181 | return rval(state,stream,"string"); |
| 182 | } |
| 183 | |
| 184 | // quoted atom |
| 185 | if (ch == '\'') { |
| 186 | if (singleQuote(stream)) { |
| 187 | return rval(state,stream,"atom"); |
| 188 | }else{ |
| 189 | return rval(state,stream,"error"); |
| 190 | } |
| 191 | } |
| 192 |
no test coverage detected