Caches and returns the unique query tokens for the given search mode. @param tokens token list @return token set
(final TokenList tokens)
| 350 | * @return token set |
| 351 | */ |
| 352 | private TokenSet unique(final TokenList tokens) { |
| 353 | // cache all query tokens in a set (duplicates are removed) |
| 354 | final TokenSet ts = new TokenSet(); |
| 355 | switch(mode) { |
| 356 | case ALL: |
| 357 | case ANY: |
| 358 | for(final byte[] token : tokens) ts.add(token); |
| 359 | break; |
| 360 | case ALL_WORDS: |
| 361 | case ANY_WORD: |
| 362 | final FTLexer lexer = new FTLexer(ftOpt); |
| 363 | for(final byte[] token : tokens) { |
| 364 | lexer.init(token); |
| 365 | while(lexer.hasNext()) ts.add(lexer.nextToken()); |
| 366 | } |
| 367 | break; |
| 368 | case PHRASE: |
| 369 | final TokenBuilder tb = new TokenBuilder(); |
| 370 | for(final byte[] token : tokens) tb.add(token).add(' '); |
| 371 | ts.add(tb.trim().finish()); |
| 372 | } |
| 373 | return ts; |
| 374 | } |
| 375 | |
| 376 | /** |
| 377 | * Returns a full-text tokenizer instance. |