Function NewFromString

tools/tokenizer/tokenizer.go:27–29

NewFromString creates new Tokenizer from the provided string.

(str string)

// NewFromString creates new Tokenizer from the provided string.
func NewFromString(str string) *Tokenizer {
	return New(strings.NewReader(str))
}

// NewFromBytes creates new Tokenizer from the provided bytes slice.
func NewFromBytes(b []byte) *Tokenizer {