(value types.Val, filter *stringFilter)
| 90 | } |
| 91 | |
| 92 | func tokenizeValue(value types.Val, filter *stringFilter) []string { |
| 93 | tokenizer, found := tok.GetTokenizer(filter.tokName) |
| 94 | // tokenizer was used in previous stages of query processing, it has to be available |
| 95 | x.AssertTrue(found) |
| 96 | |
| 97 | tokens, err := tok.BuildTokens(value.Value, tok.GetTokenizerForLang(tokenizer, filter.lang)) |
| 98 | if err != nil { |
| 99 | glog.Errorf("Error while building tokens: %s", err) |
| 100 | return []string{} |
| 101 | } |
| 102 | return tokens |
| 103 | } |
no test coverage detected