Abstract tokenizer. @author BaseX Team, BSD License @author Jens Erat
| 9 | * @author Jens Erat |
| 10 | */ |
| 11 | public abstract class Tokenizer extends LanguageImpl { |
| 12 | /** List of available tokenizers. */ |
| 13 | static final ArrayList<Tokenizer> IMPL = new ArrayList<>(); |
| 14 | |
| 15 | /** Return original tokens. */ |
| 16 | boolean original; |
| 17 | /** Return all tokens. */ |
| 18 | boolean all; |
| 19 | |
| 20 | /* Load tokenizer classes and order them by precedence. */ |
| 21 | static { |
| 22 | IMPL.add(new WesternTokenizer(null)); |
| 23 | if(JapaneseTokenizer.available()) IMPL.add(new JapaneseTokenizer(null)); |
| 24 | Collections.sort(IMPL); |
| 25 | } |
| 26 | |
| 27 | /** |
| 28 | * Checks if the language is supported by the available tokenizers. |
| 29 | * @param language language to be found |
| 30 | * @return result of check |
| 31 | */ |
| 32 | public static boolean supportFor(final Language language) { |
| 33 | for(final Tokenizer impl : IMPL) { |
| 34 | if(impl.supports(language)) return true; |
| 35 | } |
| 36 | return false; |
| 37 | } |
| 38 | |
| 39 | /** |
| 40 | * Factory method. |
| 41 | * @param f full-text options |
| 42 | * @return tokenizer |
| 43 | */ |
| 44 | abstract Tokenizer get(FTOpt f); |
| 45 | |
| 46 | /** |
| 47 | * Gets full-text info for the specified token. |
| 48 | * Needed for visualizations; does not have to be implemented by all tokenizers. |
| 49 | * <ul> |
| 50 | * <li> int[0]: length of each token</li> |
| 51 | * <li> int[1]: sentence info, length of each sentence</li> |
| 52 | * <li> int[2]: paragraph info, length of each paragraph</li> |
| 53 | * <li> int[3]: each token as int[]</li> |
| 54 | * <li> int[4]: punctuation marks of each sentence</li> |
| 55 | * </ul> |
| 56 | * @return int arrays or empty array if not implemented |
| 57 | */ |
| 58 | int[][] info() { |
| 59 | return new int[0][]; |
| 60 | } |
| 61 | |
| 62 | /** |
| 63 | * Checks if current token is a paragraph. |
| 64 | * Needed for visualizations; Does not have to be implemented by all tokenizers. |
| 65 | * @return whether current token is a paragraph, or {@code false} if not implemented |
| 66 | */ |
| 67 | boolean paragraph() { |
| 68 | return false; |