Function apply_lora

examples/export-lora/export-lora.cpp:321–369

}

static bool apply_lora(struct ggml_tensor * tensor, struct lora_data * lora, int n_threads) {
    if (lora->ctx == NULL) {
        return false;
    }
    std::string name = ggml_get_name(tensor);
    std::string name_a = name + std::string(".loraA");
    std::string name_b = name + std::string(".loraB");
    struct ggml_tensor * lora_a = ggml_get_tensor(lora->ctx, name_a.c_str());
    struct ggml_tensor * lora_b = ggml_get_tensor(lora->ctx, name_b.c_str());
    if (lora_a == NULL || lora_b == NULL) {
        return false;
    }

    float scaling = lora->info.scale * (float)lora->lora_alpha / (float)lora->lora_r;

    struct ggml_init_params params;
    params.mem_size   = GGML_OBJECT_SIZE + ggml_graph_overhead() + ggml_tensor_overhead()*4 + GGML_MEM_ALIGN*5;
    params.mem_buffer = NULL;
    params.no_alloc   = true;
    struct ggml_context * ctx = NULL;
    struct ggml_allocr * alloc = NULL;
    struct ggml_cgraph * gf = NULL;

    ctx   = ggml_init(params);
    alloc = ggml_allocr_new_measure(tensor_alignment);
    gf    = build_graph_lora(ctx, tensor, lora_a, lora_b, scaling);
    size_t alloc_size = ggml_allocr_alloc_graph(alloc, gf);
    ggml_allocr_free(alloc);
    ggml_free(ctx);

    static std::vector<uint8_t> data_compute;
    data_compute.resize(alloc_size + tensor_alignment);

    ctx   = ggml_init(params);
    alloc = ggml_allocr_new(data_compute.data(), data_compute.size(), tensor_alignment);
    gf    = build_graph_lora(ctx, tensor, lora_a, lora_b, scaling);
    ggml_allocr_alloc_graph(alloc, gf);
    ggml_allocr_free(alloc);

    struct ggml_cplan cplan = ggml_graph_plan(gf, n_threads);
    static std::vector<uint8_t> data_work;
    data_work.resize(cplan.work_size);
    cplan.work_data = data_work.data();

    ggml_graph_compute(gf, &cplan);

    ggml_free(ctx);
    return true;
}

static void export_lora(struct export_lora_params * params) {
    // load all loras