Method QueryTokens

tok/tok.go:494–520

(v interface{})

}

func (t NGramTokenizer) QueryTokens(v interface{}) ([]string, error) {
	tokens, err := t.processTokens(v)
	if err != nil || tokens == nil {
		return []string{}, err
	}

	return t.generateNGrams(tokens, func(tokens analysis.TokenStream, addToRes func(string)) {
		n := len(tokens)
		gram := 3
		if n < 3 {
			gram = n
		}

		for i := 0; i < n; i++ {
			if i+gram <= n {
				var builder strings.Builder
				for j := 0; j < gram; j++ {
					builder.Write(tokens[i+j].Term)
					if j != (gram - 1) {
						builder.Write([]byte(" "))
					}
				}
				addToRes(builder.String())
			}
		}
	}), nil
}

func (t NGramTokenizer) Name() string { return "ngram" }
func (t NGramTokenizer) Type() string { return "string" }