Struct TermTokenizer

tok/tok.go:365–367

TermTokenizer generates term tokens from string data.

// TermTokenizer generates term tokens from string data.
type TermTokenizer struct {
	lang string
}

func (t TermTokenizer) Name() string { return "term" }
func (t TermTokenizer) Type() string { return "string" }