Function main

examples/batched/batched.cpp:10–274

#include <vector>

int main(int argc, char ** argv) {
    gpt_params params;

    if (argc == 1 || argv[1][0] == '-') {
        printf("usage: %s MODEL_PATH [PROMPT] [PARALLEL] [LEN] [N_THREAD] [VRAM_BUDGET] [NGL]\n" , argv[0]);
        return 1 ;
    }

    // number of parallel batches
    int n_parallel = 1;

    // total length of the sequences including the prompt
    int n_len = 32;

    // number of layers to offload to the GPU
    int n_gpu_layers = 0;

    // vram budget in GiB
    double vram_budget = -1;

    if (argc >= 2) {
        params.model = argv[1];
    }

    if (argc >= 3) {
        params.prompt = argv[2];
    }

    if (argc >= 4) {
        n_parallel = std::atoi(argv[3]);
    }

    if (argc >= 5) {
        n_len = std::atoi(argv[4]);
    }

    if (argc >= 6) {
        params.n_threads = std::atoi(argv[5]);
    }

    if (argc >= 7) {
        vram_budget = std::atof(argv[6]);
    }

    if (argc >= 8) {
        n_gpu_layers = std::atoi(argv[7]);
    }

    printf("params: model = %s, prompt = %s, n_parallel = %d, n_len = %d, n_gpu_layers = %d, n_threads = %d, vram_budget = %.2f GiB, reset_gpu_index = true\n",
           params.model.c_str(), params.prompt.c_str(), n_parallel, n_len, n_gpu_layers, params.n_threads, vram_budget);

    if (params.prompt.empty()) {
        params.prompt = "Hello my name is";
    }

    // init LLM

    llama_backend_init(params.numa);