| 2135 | } |
| 2136 | |
| 2137 | void load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, llama_mlock * lmlock) { |
| 2138 | size_t size_data = 0; |
| 2139 | size_t size_lock = 0; |
| 2140 | size_t size_pref = 0; // prefetch |
| 2141 | |
| 2142 | for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { |
| 2143 | struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); |
| 2144 | size_data += ggml_nbytes(cur); |
| 2145 | if (cur->backend == GGML_BACKEND_CPU) { |
| 2146 | size_pref += ggml_nbytes(cur); |
| 2147 | } |
| 2148 | } |
| 2149 | |
| 2150 | if (use_mmap) { |
| 2151 | mapping.reset(new llama_mmap(&file, size_pref, ggml_is_numa())); |
| 2152 | if (lmlock) { |
| 2153 | lmlock->init(mapping->addr); |
| 2154 | } |
| 2155 | } |
| 2156 | |
| 2157 | size_t done_size = 0; |
| 2158 | for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) { |
| 2159 | struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i)); |
| 2160 | GGML_ASSERT(cur); // unused tensors should have been caught by load_data already |
| 2161 | |
| 2162 | if (progress_callback) { |
| 2163 | progress_callback((float) done_size / size_data, progress_callback_user_data); |
| 2164 | } |
| 2165 | |
| 2166 | // allocate temp buffer if not using mmap |
| 2167 | if (!use_mmap && cur->data == NULL) { |
| 2168 | GGML_ASSERT(cur->backend != GGML_BACKEND_CPU); |
| 2169 | #ifdef GGML_USE_CPU_HBM |
| 2170 | cur->data = (uint8_t*)hbw_malloc(ggml_nbytes(cur)); |
| 2171 | #else |
| 2172 | cur->data = (uint8_t*)malloc(ggml_nbytes(cur)); |
| 2173 | #endif |
| 2174 | } |
| 2175 | |
| 2176 | load_data_for(cur); |
| 2177 | |
| 2178 | switch (cur->backend) { |
| 2179 | case GGML_BACKEND_CPU: |
| 2180 | if (use_mmap && lmlock) { |
| 2181 | size_lock += ggml_nbytes(cur); |
| 2182 | lmlock->grow_to(size_lock); |
| 2183 | } |
| 2184 | break; |
| 2185 | #ifdef GGML_USE_CUBLAS |
| 2186 | case GGML_BACKEND_GPU: |
| 2187 | case GGML_BACKEND_GPU_SPLIT: |
| 2188 | // old code: |
| 2189 | //ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor); |
| 2190 | |
| 2191 | // TODO: test if this works !! |
| 2192 | ggml_cuda_transform_tensor(cur->data, cur); |
| 2193 | if (!use_mmap) { |
| 2194 | free(cur->data); |
no test coverage detected