Method decode

cactus/engine/engine_model.cpp:237–283

}

uint32_t Model::decode(const std::vector<uint32_t>& tokens, float temperature, float top_p,
                        size_t top_k, const std::string& profile_file, float* out_entropy,
                        float min_p, float repetition_penalty) {

    if (temperature < 0) {
        temperature = config_.default_temperature;
    }
    if (top_p < 0) {
        top_p = config_.default_top_p;
    }
    if (top_k == 0) {
        top_k = config_.default_top_k;
    }
    auto final_hidden = forward(tokens, true);

    auto* gb = static_cast<CactusGraph*>(graph_handle_);
    auto backend = config_.default_backend == Config::Backend::CPU
        ? ComputeBackend::CPU
        : ComputeBackend::NPU;

    auto last_hidden = gb->index(final_hidden, tokens.size() - 1, 0);
    const auto& last_hidden_buf = gb->get_output_buffer(last_hidden);
    size_t hidden_dim = last_hidden_buf.shape[0];
    last_hidden = gb->reshape(last_hidden, {1, hidden_dim});

    auto logits_node_id = gb->matmul(last_hidden, output_weight_node_id_, true, backend);

    if (config_.final_logit_softcapping > 0.0f) {
        float inv_cap = 1.0f / config_.final_logit_softcapping;
        logits_node_id = gb->scalar_multiply(logits_node_id, inv_cap);
        logits_node_id = gb->tanh(logits_node_id);
        logits_node_id = gb->scalar_multiply(logits_node_id, config_.final_logit_softcapping);
    }
    auto sampled_token_id = sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty);

    gb->execute(profile_file);

    compute_entropy(gb, logits_node_id, out_entropy);

    post_execute_updates(gb, tokens.size());
    update_kv_cache(gb, tokens.size());

    auto* output_ptr = gb->get_output(sampled_token_id);
    uint32_t result_token = *static_cast<uint32_t*>(output_ptr);
    record_sampled_token(result_token);
    return result_token;
}

size_t Model::sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k,
                           float min_p, float repetition_penalty,