Method distinctTokens

basex-core/src/main/java/org/basex/util/Token.java:1068–1080

Normalizes the specified input and returns its distinct tokens. Optimized for small number of tokens. @param token token @return distinct tokens

(final byte[] token)

   * @return distinct tokens
   */
  public static byte[][] distinctTokens(final byte[] token) {
    final byte[][] tokens = split(normalize(token), ' ');
    int tl = tokens.length;
    for(int i = 0; i < tl - 1; i++) {
      for(int j = i + 1; j < tl; j++) {
        if(eq(tokens[i], tokens[j])) {
          Array.remove(tokens, j, 1, tl);
          j--; tl--;
        }
      }
    }
    return Array.copyOf(tokens, tl);
  }

  /**
   * Checks if the specified token has only whitespace.