| 2113 | } |
| 2114 | |
| 2115 | ggml_status llama_context::graph_compute( |
| 2116 | ggml_cgraph * gf, |
| 2117 | bool batched) { |
| 2118 | int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads; |
| 2119 | ggml_threadpool_t tp = batched ? threadpool_batch : threadpool; |
| 2120 | |
| 2121 | if (backend_cpu != nullptr) { |
| 2122 | auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| 2123 | auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); |
| 2124 | if (set_threadpool_fn) { |
| 2125 | set_threadpool_fn(backend_cpu, tp); |
| 2126 | } |
| 2127 | } |
| 2128 | |
| 2129 | // set the number of threads for all the backends |
| 2130 | for (const auto & set_n_threads_fn : set_n_threads_fns) { |
| 2131 | set_n_threads_fn.second(set_n_threads_fn.first, n_threads); |
| 2132 | } |
| 2133 | |
| 2134 | auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); |
| 2135 | if (status != GGML_STATUS_SUCCESS) { |
| 2136 | LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); |
| 2137 | } |
| 2138 | |
| 2139 | // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched)); |
| 2140 | |
| 2141 | return status; |
| 2142 | } |
| 2143 | |
| 2144 | llm_graph_cb llama_context::graph_get_cb() const { |
| 2145 | return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) { |
no test coverage detected