| 36 | } |
| 37 | |
| 38 | function TokenStream(input){ |
| 39 | var current=null; |
| 40 | var pykeywords=['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] |
| 41 | return { |
| 42 | next : next, |
| 43 | peek : peek, |
| 44 | eof : eof, |
| 45 | croak : input.croak |
| 46 | }; |
| 47 | function is_keyword(x) { |
| 48 | return pykeywords.indexOf(x) >= 0; |
| 49 | } |
| 50 | function is_digit(ch) { |
| 51 | return /[0-9]/i.test(ch); |
| 52 | } |
| 53 | function is_id_start(ch) { |
| 54 | return /[a-zA-Z$_@]/i.test(ch); |
| 55 | } |
| 56 | function is_id(ch) { |
| 57 | return is_id_start(ch) || "0123456789".indexOf(ch) >= 0; |
| 58 | } |
| 59 | function is_op_char(ch) { |
| 60 | return "+-*/%=&|<>!^\\".indexOf(ch) >= 0;// \ for 正则 |
| 61 | } |
| 62 | function is_punc(ch) { |
| 63 | return "(){}[]".indexOf(ch) >= 0; |
| 64 | } |
| 65 | function is_whitespace(ch) { |
| 66 | return " \t".indexOf(ch) >= 0; |
| 67 | } |
| 68 | |
| 69 | function is_split(ch){ |
| 70 | return ",;.:?".indexOf(ch)>=0; |
| 71 | } |
| 72 | |
| 73 | function read_while(predicate) { |
| 74 | var str = ""; |
| 75 | while (!input.eof() && predicate(input.peek())) |
| 76 | str += input.next(); |
| 77 | return str; |
| 78 | } |
| 79 | function read_number() { |
| 80 | var has_dot = false; |
| 81 | var number = read_while(function(ch){ |
| 82 | if (ch == ".") { |
| 83 | if (has_dot) return false; |
| 84 | has_dot = true; |
| 85 | return true; |
| 86 | } |
| 87 | return is_digit(ch); |
| 88 | }); |
| 89 | return { |
| 90 | type: "num", |
| 91 | value: parseFloat(number), |
| 92 | poi:{ |
| 93 | line:start.line, |
| 94 | start:start.column, |
| 95 | end:input.poi().column |