Tokenize Lexical analysis
(str string)
| 40 | |
| 41 | // Tokenize Lexical analysis |
| 42 | func Tokenize(str string) []*TokenType { |
| 43 | bytes := []byte(str) |
| 44 | var result []*TokenType |
| 45 | var values []byte |
| 46 | status := Init |
| 47 | for _, b := range bytes { |
| 48 | switch status { |
| 49 | case Init: |
| 50 | status, values = InitStatus(b, values) |
| 51 | break |
| 52 | case Identifier: |
| 53 | if isLetter(b) { |
| 54 | values = append(values, b) |
| 55 | } else { |
| 56 | t := &TokenType{ |
| 57 | tokenType: Identifier, |
| 58 | value: string(values), |
| 59 | } |
| 60 | result = append(result, t) |
| 61 | values = nil |
| 62 | |
| 63 | status, values = InitStatus(b, values) |
| 64 | break |
| 65 | } |
| 66 | case Int1: |
| 67 | if b == 'n' { |
| 68 | status = Int2 |
| 69 | values = append(values, b) |
| 70 | } else if isLetter(b) || isDigit(b) { |
| 71 | status = Identifier |
| 72 | values = append(values, b) |
| 73 | } else { |
| 74 | status, values = InitStatus(b, values) |
| 75 | } |
| 76 | case Int2: |
| 77 | if b == 't' { |
| 78 | status = Int3 |
| 79 | values = append(values, b) |
| 80 | } else if isLetter(b) || isDigit(b) { |
| 81 | status = Identifier |
| 82 | values = append(values, b) |
| 83 | } else { |
| 84 | status, values = InitStatus(b, values) |
| 85 | } |
| 86 | case Int3: |
| 87 | if b == ' ' { |
| 88 | status = Int |
| 89 | } else { |
| 90 | status = Identifier |
| 91 | values = append(values, b) |
| 92 | } |
| 93 | case Int: |
| 94 | t := &TokenType{ |
| 95 | tokenType: Int, |
| 96 | value: string(values), |
| 97 | } |
| 98 | result = append(result, t) |
| 99 | values = nil |