(stream, state)
| 46 | } |
| 47 | // tokenizers |
| 48 | function tokenBase(stream, state) { |
| 49 | if (stream.eatSpace()) { |
| 50 | return null; |
| 51 | } |
| 52 | |
| 53 | var ch = stream.peek(); |
| 54 | |
| 55 | // Handle Comments |
| 56 | if (ch === "'") { |
| 57 | stream.skipToEnd(); |
| 58 | return 'comment'; |
| 59 | } |
| 60 | |
| 61 | |
| 62 | // Handle Number Literals |
| 63 | if (stream.match(/^((&H)|(&O))?[0-9\.a-f]/i, false)) { |
| 64 | var floatLiteral = false; |
| 65 | // Floats |
| 66 | if (stream.match(/^\d*\.\d+F?/i)) { floatLiteral = true; } |
| 67 | else if (stream.match(/^\d+\.\d*F?/)) { floatLiteral = true; } |
| 68 | else if (stream.match(/^\.\d+F?/)) { floatLiteral = true; } |
| 69 | |
| 70 | if (floatLiteral) { |
| 71 | // Float literals may be "imaginary" |
| 72 | stream.eat(/J/i); |
| 73 | return 'number'; |
| 74 | } |
| 75 | // Integers |
| 76 | var intLiteral = false; |
| 77 | // Hex |
| 78 | if (stream.match(/^&H[0-9a-f]+/i)) { intLiteral = true; } |
| 79 | // Octal |
| 80 | else if (stream.match(/^&O[0-7]+/i)) { intLiteral = true; } |
| 81 | // Decimal |
| 82 | else if (stream.match(/^[1-9]\d*F?/)) { |
| 83 | // Decimal literals may be "imaginary" |
| 84 | stream.eat(/J/i); |
| 85 | // TODO - Can you have imaginary longs? |
| 86 | intLiteral = true; |
| 87 | } |
| 88 | // Zero by itself with no other piece of number. |
| 89 | else if (stream.match(/^0(?![\dx])/i)) { intLiteral = true; } |
| 90 | if (intLiteral) { |
| 91 | // Integer literals may be "long" |
| 92 | stream.eat(/L/i); |
| 93 | return 'number'; |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | // Handle Strings |
| 98 | if (stream.match(stringPrefixes)) { |
| 99 | state.tokenize = tokenStringFactory(stream.current()); |
| 100 | return state.tokenize(stream, state); |
| 101 | } |
| 102 | |
| 103 | // Handle operators and Delimiters |
| 104 | if (stream.match(tripleDelimiters) || stream.match(doubleDelimiters)) { |
| 105 | return null; |
nothing calls this directly
no test coverage detected