Function run_perplexity

src/main.cpp:371–431

};

void run_perplexity(
  Session& session,
  const std::vector<int>& encoding
) {
  Model& model = session.model;
  InferenceState& state = session.state;
  Sampler& sampler = session.sampler;

  std::cout << "Model active bytes with full context window: " << model.active_bytes(model.config->max_seq_len) << std::endl;

  {
    ProfileDisabledScope profile_disabled;
    std::cout << "Running warmup..." << std::endl;
    // Do one inference as warmup.
    // On CPU, this ensures all tensors are loaded into memory via mmap.
    model.forward(state, 0, 0);
    std::cout << "Warmup complete" << std::endl;
  }

  double sum_logprob = 0.0;
  double ss_logprob = 0.0;
  // Generates output logits for all tokens in the prompt and sum log probs to
  // compute perplexity.
  uint64_t start_ms = get_timestamp_ms();
  size_t read_bytes = 0;
  size_t N = encoding.size() - 1;
  for (size_t pos = 0; pos + 1 < encoding.size(); pos++) {
    std::cout << "\r Computing perplexity..." << pos + 1 << "/" << N << std::flush;
    
    int token_id = encoding[pos];
    model.forward(state, token_id, pos);
    read_bytes += model.active_bytes(pos);

    double logprob = std::log(sampler.sample_prob(encoding[pos + 1], state));
    sum_logprob += logprob;
    ss_logprob += logprob * logprob;
  }
  std::cout << std::endl;
  uint64_t end_ms = get_timestamp_ms();
  double elapsed_s = (end_ms - start_ms)/1000.0;
  double perplexity = std::exp(-sum_logprob / N);
  double perplexity_error = perplexity * std::sqrt(
    (ss_logprob - sum_logprob * sum_logprob / N) / N / N
  );
  std::cout << fmt::format(
    "Stats:\n"
    "  {} tokens\n"
    "  perplexity: {:.5} ± {:.5}\n"
    "  throughput: {:.5}tok/s\n"
    "  latency: {:.5}s/tok\n"
    "  bandwidth: {:.5}GB/s\n"
    "  total: {:.5}s\n",
    N,
    perplexity,
    perplexity_error,
    N / elapsed_s,
    elapsed_s / N,
    ((double)read_bytes / 1e9) / elapsed_s,