| 140 | } |
| 141 | |
| 142 | static struct llama_model * llama_model_load_from_file_impl( |
| 143 | const std::string & path_model, |
| 144 | std::vector<std::string> & splits, |
| 145 | struct llama_model_params params) { |
| 146 | ggml_time_init(); |
| 147 | |
| 148 | if (!params.vocab_only && ggml_backend_reg_count() == 0) { |
| 149 | LLAMA_LOG_ERROR("%s: no backends are loaded. hint: use ggml_backend_load() or ggml_backend_load_all() to load a backend before calling this function\n", __func__); |
| 150 | return nullptr; |
| 151 | } |
| 152 | |
| 153 | unsigned cur_percentage = 0; |
| 154 | if (params.progress_callback == NULL) { |
| 155 | params.progress_callback_user_data = &cur_percentage; |
| 156 | params.progress_callback = [](float progress, void * ctx) { |
| 157 | unsigned * cur_percentage_p = (unsigned *) ctx; |
| 158 | unsigned percentage = (unsigned) (100 * progress); |
| 159 | while (percentage > *cur_percentage_p) { |
| 160 | *cur_percentage_p = percentage; |
| 161 | LLAMA_LOG_CONT("."); |
| 162 | if (percentage >= 100) { |
| 163 | LLAMA_LOG_CONT("\n"); |
| 164 | } |
| 165 | } |
| 166 | return true; |
| 167 | }; |
| 168 | } |
| 169 | |
| 170 | llama_model * model = new llama_model(params); |
| 171 | |
| 172 | // create list of devices to use with this model |
| 173 | if (params.devices) { |
| 174 | for (ggml_backend_dev_t * dev = params.devices; *dev; ++dev) { |
| 175 | model->devices.push_back(*dev); |
| 176 | } |
| 177 | } else { |
| 178 | std::vector<ggml_backend_dev_t> rpc_servers; |
| 179 | // use all available devices |
| 180 | for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { |
| 181 | ggml_backend_dev_t dev = ggml_backend_dev_get(i); |
| 182 | switch (ggml_backend_dev_type(dev)) { |
| 183 | case GGML_BACKEND_DEVICE_TYPE_CPU: |
| 184 | case GGML_BACKEND_DEVICE_TYPE_ACCEL: |
| 185 | // skip CPU backends since they are handled separately |
| 186 | break; |
| 187 | |
| 188 | case GGML_BACKEND_DEVICE_TYPE_GPU: |
| 189 | ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); |
| 190 | if (ggml_backend_reg_name(reg) == std::string("RPC")) { |
| 191 | rpc_servers.push_back(dev); |
| 192 | } else { |
| 193 | model->devices.push_back(dev); |
| 194 | } |
| 195 | break; |
| 196 | } |
| 197 | } |
| 198 | // add RPC servers at the front of the list |
| 199 | if (!rpc_servers.empty()) { |
no test coverage detected