(stream, state)
| 24 | } |
| 25 | |
| 26 | function tokenBase(stream, state) { |
| 27 | curPunc = null; |
| 28 | if (stream.sol() && stream.match("=begin") && stream.eol()) { |
| 29 | state.tokenize.push(readBlockComment); |
| 30 | return "comment"; |
| 31 | } |
| 32 | if (stream.eatSpace()) return null; |
| 33 | var ch = stream.next(), m; |
| 34 | if (ch == "`" || ch == "'" || ch == '"') { |
| 35 | return chain(readQuoted(ch, "string", ch == '"' || ch == "`"), stream, state); |
| 36 | } else if (ch == "/" && !stream.eol() && stream.peek() != " ") { |
| 37 | return chain(readQuoted(ch, "string-2", true), stream, state); |
| 38 | } else if (ch == "%") { |
| 39 | var style = "string", embed = true; |
| 40 | if (stream.eat("s")) style = "atom"; |
| 41 | else if (stream.eat(/[WQ]/)) style = "string"; |
| 42 | else if (stream.eat(/[r]/)) style = "string-2"; |
| 43 | else if (stream.eat(/[wxq]/)) { style = "string"; embed = false; } |
| 44 | var delim = stream.eat(/[^\w\s]/); |
| 45 | if (!delim) return "operator"; |
| 46 | if (matching.propertyIsEnumerable(delim)) delim = matching[delim]; |
| 47 | return chain(readQuoted(delim, style, embed, true), stream, state); |
| 48 | } else if (ch == "#") { |
| 49 | stream.skipToEnd(); |
| 50 | return "comment"; |
| 51 | } else if (ch == "<" && (m = stream.match(/^<-?[\`\"\']?([a-zA-Z_?]\w*)[\`\"\']?(?:;|$)/))) { |
| 52 | return chain(readHereDoc(m[1]), stream, state); |
| 53 | } else if (ch == "0") { |
| 54 | if (stream.eat("x")) stream.eatWhile(/[\da-fA-F]/); |
| 55 | else if (stream.eat("b")) stream.eatWhile(/[01]/); |
| 56 | else stream.eatWhile(/[0-7]/); |
| 57 | return "number"; |
| 58 | } else if (/\d/.test(ch)) { |
| 59 | stream.match(/^[\d_]*(?:\.[\d_]+)?(?:[eE][+\-]?[\d_]+)?/); |
| 60 | return "number"; |
| 61 | } else if (ch == "?") { |
| 62 | while (stream.match(/^\\[CM]-/)) {} |
| 63 | if (stream.eat("\\")) stream.eatWhile(/\w/); |
| 64 | else stream.next(); |
| 65 | return "string"; |
| 66 | } else if (ch == ":") { |
| 67 | if (stream.eat("'")) return chain(readQuoted("'", "atom", false), stream, state); |
| 68 | if (stream.eat('"')) return chain(readQuoted('"', "atom", true), stream, state); |
| 69 | |
| 70 | // :> :>> :< :<< are valid symbols |
| 71 | if (stream.eat(/[\<\>]/)) { |
| 72 | stream.eat(/[\<\>]/); |
| 73 | return "atom"; |
| 74 | } |
| 75 | |
| 76 | // :+ :- :/ :* :| :& :! are valid symbols |
| 77 | if (stream.eat(/[\+\-\*\/\&\|\:\!]/)) { |
| 78 | return "atom"; |
| 79 | } |
| 80 | |
| 81 | // Symbols can't start by a digit |
| 82 | if (stream.eat(/[a-zA-Z$@_]/)) { |
| 83 | stream.eatWhile(/[\w]/); |
no test coverage detected