| 1371 | } |
| 1372 | |
| 1373 | ggml_status llama_context::graph_compute( |
| 1374 | ggml_cgraph * gf, |
| 1375 | bool batched) { |
| 1376 | int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads; |
| 1377 | ggml_threadpool_t tp = batched ? threadpool_batch : threadpool; |
| 1378 | |
| 1379 | if (backend_cpu != nullptr) { |
| 1380 | auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| 1381 | auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); |
| 1382 | set_threadpool_fn(backend_cpu, tp); |
| 1383 | } |
| 1384 | |
| 1385 | // set the number of threads for all the backends |
| 1386 | for (const auto & set_n_threads_fn : set_n_threads_fns) { |
| 1387 | set_n_threads_fn.second(set_n_threads_fn.first, n_threads); |
| 1388 | } |
| 1389 | |
| 1390 | auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); |
| 1391 | if (status != GGML_STATUS_SUCCESS) { |
| 1392 | LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); |
| 1393 | } |
| 1394 | |
| 1395 | // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched)); |
| 1396 | |
| 1397 | return status; |
| 1398 | } |
| 1399 | |
| 1400 | llm_graph_cb llama_context::graph_get_cb() const { |
| 1401 | return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) { |
no test coverage detected