| 351 | } |
| 352 | |
| 353 | void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ |
| 354 | std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test); |
| 355 | |
| 356 | size_t n_fails = 0; |
| 357 | |
| 358 | for (const auto & test : tests) { |
| 359 | std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first); |
| 360 | |
| 361 | if (tokens != test.second){ |
| 362 | n_fails++; |
| 363 | |
| 364 | // print out failure cases |
| 365 | fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str()); |
| 366 | fprintf(stderr, "%s : tokens in hf: ", __func__); |
| 367 | for (const auto & t : test.second) { |
| 368 | fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); |
| 369 | } |
| 370 | fprintf(stderr, "\n"); |
| 371 | fprintf(stderr, "%s : tokens in ggml: ", __func__); |
| 372 | for (const auto & t : tokens) { |
| 373 | fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); |
| 374 | } |
| 375 | fprintf(stderr, "\n"); |
| 376 | } |
| 377 | } |
| 378 | |
| 379 | fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size()); |
| 380 | } |
| 381 | |
| 382 | bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) { |
| 383 | printf("%s: loading vocab from '%s'\n", __func__, fname.c_str()); |