Returns a token without diacritics. @param token token to be normalized @return resulting token
(final byte[] token)
| 37 | * @return resulting token |
| 38 | */ |
| 39 | public static byte[] noDiacritics(final byte[] token) { |
| 40 | final int tl = token.length; |
| 41 | final TokenBuilder tb = new TokenBuilder(tl); |
| 42 | Token.forEachCp(token, cp -> { |
| 43 | if(cp < 0x80) { |
| 44 | tb.add(cp); |
| 45 | } else if(!isCombining(Character.getType(cp))) { |
| 46 | tb.add(noDiacritics(cp)); |
| 47 | } |
| 48 | }); |
| 49 | return tb.finish(); |
| 50 | } |
| 51 | |
| 52 | /** |
| 53 | * Checks if a character is a letter or digit. |