Method _shared_insert

src/common/AutoModel/automodel.cpp:147–209

}

bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector<int>& tokens, std::function<bool()> is_cancelled, void* payload, int first_len_run) {

    // print token history
    // header_print("DEBUG", "Current token history: ");
    // for (size_t i = 0; i < this->token_history.size(); i++) {
    //     std::cout << this->token_history[i] << " ";
    // }
    // std::cout << std::endl;
    // // print tokens to insert
    // header_print("DEBUG", "Tokens to insert: ");
    // for (size_t i = 0; i < tokens.size(); i++) {
    //     std::cout << tokens[i] <<  " ";
    // }
    // std::cout << std::endl;

    // prefix check for tokens and token history to see if we can skip some tokens
    const size_t idx = this->token_history.size();
    size_t skip_count = 0;
    for (size_t i = 0; i < idx; i++) {
        if (tokens[i] == this->token_history[i]) {
            skip_count++;
        } 
        else {
            break;
        }
    }
    if (skip_count != idx) {
        clear_context();
        skip_count = 0;
    }
    tokens.erase(tokens.begin(), tokens.begin() + skip_count);


    if (this->total_tokens + tokens.size() >= this->MAX_L){
        header_print("WARNING", "Max length reached, stopping prefilling...");
        return false;
    }
    for (int token : tokens){
        this->token_history.push_back(token);
    }
    buffer<bf16> y;

    auto prefill_start_time = this->profiler_list[PREFILL_TIME].start();
    
    y = _chunked_insert(meta_info, tokens, is_cancelled, payload, first_len_run);

    auto prefill_end_time = this->profiler_list[PREFILL_TIME].stop(tokens.size());
    meta_info.prefill_duration = (uint64_t)time_utils::duration_ns(prefill_start_time, prefill_end_time).first;
    meta_info.prompt_tokens = tokens.size();

    if (meta_info.stop_reason == CANCEL_DETECTED) {
        return false;
    }

    this->total_tokens += tokens.size();
    if (this->total_tokens >= this->MAX_L){
        header_print("WARNING", "Max length reached, stopping prefilling...");
    }