Function hash_trigrams

src/simhash/minhash.c:221–247

}

static int hash_trigrams(const char **tokens, int token_count, cbm_minhash_t *out) {
    for (int k = 0; k < CBM_MINHASH_K; k++) {
        out->values[k] = UINT32_MAX;
    }

    uniq_trig_set_t uniq;
    uniq_trig_init(&uniq);
    char trigram_buf[TRIGRAM_BUF_LEN];

    for (int i = 0; i + TRIGRAM_WINDOW < token_count; i++) {
        int w =
            trigram_structural_weight(tokens[i], tokens[i + SKIP_ONE], tokens[i + TRIGRAM_WINDOW]);
        if (w == 0) {
            continue;
        }

        int len = snprintf(trigram_buf, sizeof(trigram_buf), "%s|%s|%s", tokens[i],
                           tokens[i + SKIP_ONE], tokens[i + TRIGRAM_WINDOW]);
        if (len <= 0 || (size_t)len >= sizeof(trigram_buf)) {
            continue;
        }

        uniq_trig_insert(&uniq, XXH3_64bits(trigram_buf, (size_t)len));
        weighted_minhash_update(out, trigram_buf, len, w);
    }
    return uniq.count;
}

bool cbm_minhash_compute(TSNode func_body, const char *source, int language, cbm_minhash_t *out) {
    (void)source;