filterStopwords filters stop words using an existing filter, imported here. If the lang filter is found, the we will forward requests to it. Returns filtered tokens if filter is found, otherwise returns tokens unmodified.
(lang string, input analysis.TokenStream)
| 73 | // If the lang filter is found, the we will forward requests to it. |
| 74 | // Returns filtered tokens if filter is found, otherwise returns tokens unmodified. |
| 75 | func filterStopwords(lang string, input analysis.TokenStream) analysis.TokenStream { |
| 76 | if len(input) == 0 { |
| 77 | return input |
| 78 | } |
| 79 | // check if we have stop words filter for this lang. |
| 80 | name, ok := langStops[lang] |
| 81 | if !ok { |
| 82 | return input |
| 83 | } |
| 84 | // get filter from concurrent cache so we dont recreate. |
| 85 | filter, err := bleveCache.TokenFilterNamed(name) |
| 86 | if err != nil { |
| 87 | glog.Errorf("Error while filtering %q stop words: %s", lang, err) |
| 88 | return input |
| 89 | } |
| 90 | if filter != nil { |
| 91 | return filter.Filter(input) |
| 92 | } |
| 93 | return input |
| 94 | } |