Function BuildTokens

tok/tok.go:110–120

BuildTokens tokenizes a value, creating strings that can be used to create index keys.

(val interface{}, t Tokenizer)

// BuildTokens tokenizes a value, creating strings that can be used to create
// index keys.
func BuildTokens(val interface{}, t Tokenizer) ([]string, error) {
	tokens, err := t.Tokens(val)
	if err != nil {
		return nil, err
	}
	id := t.Identifier()
	for i := range tokens {
		tokens[i] = encodeToken(tokens[i], id)
	}
	return tokens, nil
}

func BuildNGramQueryTokens(val interface{}, t NGramTokenizer) ([]string, error) {
	tokens, err := t.QueryTokens(val)