Method setToken

tokenizer/tokenizer.go:133–136

setToken adds token to the list of this Tokenizer's tokens.

(token *Token)

// setToken adds token to the list of this Tokenizer's tokens.
func (t *Tokenizer) setToken(token *Token) *Token {
	t.tokens = append(t.tokens, token)
	return token
}

// getPreviousToken returns the token that was most-recently read.
func (t *Tokenizer) getPreviousToken() *Token {