Function lex

utils/vscode/src/parser/parser.go:354–392

lex returns all the tokens and diagnostics after lexing source.

(source string)

// lex returns all the tokens and diagnostics after lexing source.
func lex(source string) ([]*Token, []Diagnostic, error) {
	l := lexer{source: source, lexerState: lexerState{pos: Position{1, 1}}}

	lastPos := Position{}
	for l.e == nil {
		// Integrity check that the parser is making progress
		if l.pos == lastPos {
			log.Panicf("Parsing stuck at %v", l.pos)
		}
		lastPos = l.pos

		s := l.save()
		r := l.next()
		switch {
		case r == '%':
			l.restore(s)
			l.pident()
		case r == '+' || r == '-' || r == '_' || isAlphaNumeric(r):
			l.restore(s)
			l.numberOrIdent()
		case r == '"':
			l.restore(s)
			l.string()
		case r == '=', r == '|':
			l.restore(s)
			l.operator()
		case r == ';':
			l.restore(s)
			l.lineComment()
		case r == '\n':
			l.restore(s)
			l.newline()
		}
	}
	if l.e != nil && l.e != io.EOF {
		return nil, nil, l.e
	}
	return l.toks, l.diags, nil
}

func isNumeric(r rune) bool      { return unicode.IsDigit(r) }
func isAlpha(r rune) bool        { return unicode.IsLetter(r) }