Method Tokenize

pkg/tokenizer/tokenizer.go:44–59

Tokenize converts entire source to tokens

()

// Tokenize converts entire source to tokens
func (t *Tokenizer) Tokenize() ([]Token, error) {
	t.tokens = nil

	for {
		tok, err := t.nextToken()
		if err != nil {
			return nil, err
		}
		t.tokens = append(t.tokens, tok)
		if tok.Kind == EOF {
			break
		}
	}

	return t.tokens, nil
}

// nextToken scans and returns the next token
func (t *Tokenizer) nextToken() (Token, error) {