| 1986 | } |
| 1987 | |
| 1988 | void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) { |
| 1989 | GGML_ASSERT(!opt_ctx); |
| 1990 | model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx(); |
| 1991 | const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train); |
| 1992 | const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch); |
| 1993 | GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0); |
| 1994 | GGML_ASSERT(n_batch % n_ubatch == 0); |
| 1995 | |
| 1996 | ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); |
| 1997 | opt_params.opt_period = n_batch / n_ubatch; |
| 1998 | opt_params.get_opt_pars = lopt_params.get_opt_pars; |
| 1999 | opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud; |
| 2000 | |
| 2001 | opt_ctx = ggml_opt_init(opt_params); |
| 2002 | |
| 2003 | llama_opt_param_filter param_filter = lopt_params.param_filter; |
| 2004 | void * param_filter_ud = lopt_params.param_filter_ud; |
| 2005 | |
| 2006 | //llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME |
| 2007 | llama_set_param(model->type_embd, param_filter, param_filter_ud); |
| 2008 | llama_set_param(model->pos_embd, param_filter, param_filter_ud); |
| 2009 | llama_set_param(model->tok_norm, param_filter, param_filter_ud); |
| 2010 | llama_set_param(model->tok_norm_b, param_filter, param_filter_ud); |
| 2011 | llama_set_param(model->output_norm, param_filter, param_filter_ud); |
| 2012 | llama_set_param(model->output_norm_b, param_filter, param_filter_ud); |
| 2013 | llama_set_param(model->output, param_filter, param_filter_ud); |
| 2014 | llama_set_param(model->output_b, param_filter, param_filter_ud); |
| 2015 | llama_set_param(model->output_norm_enc, param_filter, param_filter_ud); |
| 2016 | llama_set_param(model->cls, param_filter, param_filter_ud); |
| 2017 | llama_set_param(model->cls_b, param_filter, param_filter_ud); |
| 2018 | llama_set_param(model->cls_out, param_filter, param_filter_ud); |
| 2019 | llama_set_param(model->cls_out_b, param_filter, param_filter_ud); |
| 2020 | |
| 2021 | for (struct llama_layer & layer : model->layers) { |
| 2022 | for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) { |
| 2023 | llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud); |
| 2024 | } |
| 2025 | } |
| 2026 | } |
| 2027 | |
| 2028 | void llama_context::opt_epoch_iter( |
| 2029 | ggml_opt_dataset_t dataset, |
no test coverage detected