scan returns the next token and position from the underlying reader. Also returns the literal text read for strings, numbers, and duration tokens since these token types can have different literal representations.
()
| 44 | // Also returns the literal text read for strings, numbers, and duration tokens |
| 45 | // since these token types can have different literal representations. |
| 46 | func (s *scanner) scan() (tok token, pos int, lit string) { |
| 47 | // Read next code point. |
| 48 | ch0, pos := s.r.read() |
| 49 | |
| 50 | // if we see whitespace then consume all contiguous whitespace. |
| 51 | // if we see a letter, or certain acceptable special characters, then consume |
| 52 | // as an ident or reserved word. |
| 53 | if isWhitespace(ch0) { |
| 54 | return s.scanWhitespace() |
| 55 | } else if isLetter(ch0) || ch0 == '_' { |
| 56 | s.r.unread() |
| 57 | return s.scanIdent() |
| 58 | } else if isDigit(ch0) { |
| 59 | return s.scanNumber() |
| 60 | } |
| 61 | |
| 62 | // Otherwise, parse individual characters. |
| 63 | switch ch0 { |
| 64 | case eof: |
| 65 | return EOF, pos, "" |
| 66 | case '"': |
| 67 | s.r.unread() |
| 68 | return s.scanIdent() |
| 69 | case '\'': |
| 70 | return s.scanString() |
| 71 | case '.': |
| 72 | ch1, _ := s.r.read() |
| 73 | s.r.unread() |
| 74 | if isDigit(ch1) { |
| 75 | return s.scanNumber() |
| 76 | } |
| 77 | return Dot, pos, "" |
| 78 | case '=': |
| 79 | return Eq, pos, "" |
| 80 | case '~': |
| 81 | if ch1, _ := s.r.read(); ch1 == '=' { |
| 82 | return IEq, pos, "" |
| 83 | } |
| 84 | s.r.unread() |
| 85 | case '!': |
| 86 | if ch1, _ := s.r.read(); ch1 == '=' { |
| 87 | return Neq, pos, "" |
| 88 | } |
| 89 | s.r.unread() |
| 90 | case '>': |
| 91 | if ch1, _ := s.r.read(); ch1 == '=' { |
| 92 | return Gte, pos, "" |
| 93 | } |
| 94 | s.r.unread() |
| 95 | return Gt, pos, "" |
| 96 | case '<': |
| 97 | if ch1, _ := s.r.read(); ch1 == '=' { |
| 98 | return Lte, pos, "" |
| 99 | } else if ch1 == '>' { |
| 100 | return Neq, pos, "" |
| 101 | } |
| 102 | s.r.unread() |
| 103 | return Lt, pos, "" |
nothing calls this directly
no test coverage detected