(val interface{}, t NGramTokenizer)
| 120 | } |
| 121 | |
| 122 | func BuildNGramQueryTokens(val interface{}, t NGramTokenizer) ([]string, error) { |
| 123 | tokens, err := t.QueryTokens(val) |
| 124 | if err != nil { |
| 125 | return nil, err |
| 126 | } |
| 127 | id := t.Identifier() |
| 128 | for i := range tokens { |
| 129 | tokens[i] = encodeToken(tokens[i], id) |
| 130 | } |
| 131 | return tokens, nil |
| 132 | } |
| 133 | |
| 134 | // LoadCustomTokenizer reads and loads a custom tokenizer from the given file. |
| 135 | func LoadCustomTokenizer(soFile string) { |