| 439 | } |
| 440 | |
| 441 | static results_perplexity perplexity(llama_context * ctx, const common_params & params, const int32_t n_ctx) { |
| 442 | if (params.ppl_stride > 0) { |
| 443 | return perplexity_v2(ctx, params); |
| 444 | } |
| 445 | |
| 446 | // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip |
| 447 | // Run `./llama-perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw` |
| 448 | // Output: `perplexity: 13.5106 [114/114]` |
| 449 | // BOS tokens will be added for each chunk before eval |
| 450 | |
| 451 | const llama_model * model = llama_get_model(ctx); |
| 452 | const llama_vocab * vocab = llama_model_get_vocab(model); |
| 453 | |
| 454 | const bool add_bos = llama_vocab_get_add_bos(vocab); |
| 455 | GGML_ASSERT(!llama_vocab_get_add_eos(vocab)); |
| 456 | |
| 457 | std::ofstream logits_stream; |
| 458 | if (!params.logits_file.empty()) { |
| 459 | logits_stream.open(params.logits_file.c_str(), std::ios::binary); |
| 460 | if (!logits_stream.is_open()) { |
| 461 | LOG_ERR("%s: failed to open %s for writing\n", __func__, params.logits_file.c_str()); |
| 462 | return {}; |
| 463 | } |
| 464 | LOG_INF("%s: saving all logits to %s\n", __func__, params.logits_file.c_str()); |
| 465 | logits_stream.write("_logits_", 8); |
| 466 | logits_stream.write(reinterpret_cast<const char *>(&n_ctx), sizeof(n_ctx)); |
| 467 | } |
| 468 | |
| 469 | auto tim1 = std::chrono::high_resolution_clock::now(); |
| 470 | LOG_INF("%s: tokenizing the input ..\n", __func__); |
| 471 | |
| 472 | std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true); |
| 473 | |
| 474 | auto tim2 = std::chrono::high_resolution_clock::now(); |
| 475 | LOG_INF("%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count()); |
| 476 | |
| 477 | if (int(tokens.size()) < 2*n_ctx) { |
| 478 | LOG_ERR("%s: you need at least %d tokens to evaluate perplexity with a context of %d\n",__func__,2*n_ctx, |
| 479 | n_ctx); |
| 480 | LOG_ERR("%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size()); |
| 481 | return {std::move(tokens), 0., {}, {}}; |
| 482 | } |
| 483 | |
| 484 | std::vector<float> logit_history; |
| 485 | logit_history.resize(tokens.size()); |
| 486 | |
| 487 | std::vector<float> prob_history; |
| 488 | prob_history.resize(tokens.size()); |
| 489 | |
| 490 | const int n_chunk_max = tokens.size() / n_ctx; |
| 491 | |
| 492 | const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max); |
| 493 | const int n_batch = params.n_batch; |
| 494 | |
| 495 | const int n_vocab = llama_vocab_n_tokens(vocab); |
| 496 | |
| 497 | int count = 0; |
| 498 | double nll = 0.0; |
no test coverage detected