Function main

tests/test_tokenizer.cpp:31–87

static const int N_CASES = sizeof(CASES) / sizeof(CASES[0]);

int main(int argc, char ** argv) {
    if (argc < 2) {
        fprintf(stderr, "Usage: %s <model.ggml>\n", argv[0]);
        return 1;
    }

    if (!sam3_test_load_tokenizer(argv[1])) {
        fprintf(stderr, "FAIL: could not load tokenizer from '%s'\n", argv[1]);
        return 1;
    }

    int n_pass = 0;
    for (int i = 0; i < N_CASES; ++i) {
        auto tokens = sam3_test_tokenize(CASES[i].text);
        const auto & exp = CASES[i].expected;
        bool pass = true;

        // Must be exactly 32 tokens (context length)
        if ((int)tokens.size() != 32) {
            fprintf(stderr, "FAIL [%d]: expected 32 tokens, got %d\n", i, (int)tokens.size());
            pass = false;
        }

        // Check non-padding prefix matches expected exactly
        for (size_t j = 0; j < exp.size() && j < tokens.size(); ++j) {
            if (tokens[j] != exp[j]) {
                fprintf(stderr, "FAIL [%d]: token[%zu] = %d, expected %d\n",
                        i, j, tokens[j], exp[j]);
                pass = false;
            }
        }

        // Check remaining positions are zero-padded
        for (size_t j = exp.size(); j < tokens.size(); ++j) {
            if (tokens[j] != 0) {
                fprintf(stderr, "FAIL [%d]: expected 0 at position %zu, got %d\n",
                        i, j, tokens[j]);
                pass = false;
                break;
            }
        }

        // Print result
        fprintf(stderr, "%s [%d] \"%s\" → [", pass ? "PASS" : "FAIL", i, CASES[i].text);
        for (size_t j = 0; j < tokens.size(); ++j) {
            if (j > 0) fprintf(stderr, ", ");
            fprintf(stderr, "%d", tokens[j]);
            if (tokens[j] == 0 && j > 1) { fprintf(stderr, ", ..."); break; }
        }
        fprintf(stderr, "]\n");

        if (pass) n_pass++;
    }

    fprintf(stderr, "\n%d/%d tests passed\n", n_pass, N_CASES);
    return (n_pass == N_CASES) ? 0 : 1;
}