Function gpt_example

examples/cpp/gpt/gpt_example.cc:142–497

template<typename T>
void gpt_example(const INIReader reader)
{
    const std::string model_name         = reader.Get("ft_instance_hyperparameter", "model_name");
    const size_t      max_batch_size     = reader.GetInteger("ft_instance_hyperparameter", "max_batch_size");
    const size_t      max_seq_len        = reader.GetInteger("ft_instance_hyperparameter", "max_seq_len");
    const size_t      beam_width         = reader.GetInteger("ft_instance_hyperparameter", "beam_width");
    const uint        top_k              = (uint)reader.GetInteger("ft_instance_hyperparameter", "top_k");
    const float       top_p              = reader.GetFloat("ft_instance_hyperparameter", "top_p");
    const float       temperature        = reader.GetFloat("ft_instance_hyperparameter", "temperature");
    const float       repetition_penalty = reader.GetFloat("ft_instance_hyperparameter", "repetition_penalty", 1.0f);
    const float       presence_penalty   = reader.GetFloat("ft_instance_hyperparameter", "presence_penalty", 0.0f);
    const int         min_length         = reader.GetInteger("ft_instance_hyperparameter", "min_length", 0);
    const std::string model_dir          = std::string(reader.Get("ft_instance_hyperparameter", "model_dir"));
    const bool        sparse             = static_cast<bool>(reader.GetInteger("ft_instance_hyperparameter", "sparse"));
    const float shared_contexts_ratio    = reader.GetFloat("ft_instance_hyperparameter", "shared_contexts_ratio", 1.0f);
    const float len_penalty              = reader.GetFloat("ft_instance_hyperparameter", "len_penalty");
    const float beam_search_diversity_rate =
        reader.GetFloat("ft_instance_hyperparameter", "beam_search_diversity_rate");
    const unsigned long long int random_seed = 0;

    FT_CHECK_WITH_INFO(
        repetition_penalty == 1.0f || presence_penalty == 0.0f,
        fmtstr("Found ambiguous parameters repetition_penalty (%f) and presence_penalty (%f) "
               "which are mutually exclusive. Please remove one of repetition_penalty or presence_penalty "
               "or set to a default value.",
               repetition_penalty,
               presence_penalty));

    const size_t head_num       = reader.GetInteger(model_name, "head_num");
    const size_t size_per_head  = reader.GetInteger(model_name, "size_per_head");
    const size_t vocab_size     = reader.GetInteger(model_name, "vocab_size");
    const size_t decoder_layers = reader.GetInteger(model_name, "decoder_layers");
    const size_t hidden_units   = head_num * size_per_head;
    const size_t inter_size     = 4 * hidden_units;

    const size_t request_batch_size = reader.GetInteger("request", "request_batch_size");
    // The length of tokens we hope this model to generate
    const int request_output_len = reader.GetInteger("request", "request_output_len");
    // Whether to return the log probabilities of outputs.
    const bool is_return_log_probs = reader.GetBoolean("request", "return_log_probs", false);
    // Whether to include input contexts in computing the cumulative log probabilities.
    const bool is_return_context_cum_log_probs = reader.GetBoolean("request", "context_log_probs", false);
    if (is_return_log_probs && !is_return_context_cum_log_probs) {
        FT_LOG_WARNING("context_log_probs will be ignored since return_log_probs is disabled.");
    }

    const int start_id = 50256;
    const int end_id   = 50256;

    const int rank = 0;

    // Read ids of request from file.
    int              max_input_len = -1;
    std::vector<int> v_start_lengths;
    std::vector<int> v_start_ids;
    read_start_ids(request_batch_size, &v_start_lengths, &v_start_ids, max_input_len, end_id, 1);

    int* d_input_ids;