Method decode

cactus/models/model_needle.cpp:293–334

}

uint32_t NeedleModel::decode(const std::vector<uint32_t>& tokens,
                             float temperature, float top_p, size_t top_k,
                             const std::string& profile_file, float* out_entropy,
                             float /*min_p*/, float /*repetition_penalty*/) {
    if (temperature < 0) temperature = config_.default_temperature;
    if (top_p < 0) top_p = config_.default_top_p;
    if (top_k == 0) top_k = config_.default_top_k;

    auto* gb = static_cast<CactusGraph*>(graph_handle_);
    gb->soft_reset();
    reset_graph_side_cache_nodes();

    bool cold_start = kv_cache_.is_empty();
    std::vector<uint32_t> cold_tokens;
    const std::vector<uint32_t>* dec_tokens = &tokens;
    if (!encoder_ready_) {
        reset_cache(); gb->soft_reset();
        std::vector<uint32_t> enc(tokens.begin(), tokens.end() - 1);
        cold_tokens.assign(tokens.end() - 1, tokens.end());
        run_encoder(enc);
        encoder_ready_ = true;
        dec_tokens = &cold_tokens;
        cold_start = true;
    }

    size_t logits = cold_start
        ? run_decoder_step(*dec_tokens, false, false)
        : run_decoder_step(tokens, true, tokens.size() == 1);

    if (config_.final_logit_softcapping > 0.0f) {
        float inv = 1.0f / config_.final_logit_softcapping;
        logits = gb->scalar_multiply(gb->tanh(gb->scalar_multiply(logits, inv)), config_.final_logit_softcapping);
    }

    auto sampled = sample_token(gb, logits, temperature, top_p, top_k, 0.0f, 1.0f);
    if (!profile_file.empty()) gb->execute(profile_file); else gb->execute();

    compute_entropy(gb, logits, out_entropy);
    post_execute_updates(gb, tokens.size());
    update_kv_cache(gb, dec_tokens->size());
    return *static_cast<uint32_t*>(gb->get_output(sampled));
}

} // namespace engine
} // namespace cactus