Function Lex

parser/lexer/lexer.go:16–31

Lex will buffer and return the tokens of a disposable *[Lexer].

(source file.Source)

// Lex will buffer and return the tokens of a disposable *[Lexer].
func Lex(source file.Source) ([]Token, error) {
	tokens := make([]Token, 0, ringChunkSize)
	l := New()
	l.Reset(source)
	for {
		t, err := l.Next()
		switch err {
		case nil:
			tokens = append(tokens, t)
		case io.EOF:
			return tokens, nil
		default:
			return nil, err
		}
	}
}

// New returns a reusable lexer.
func New() *Lexer {