| 2716 | } |
| 2717 | |
| 2718 | void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) { |
| 2719 | GGML_ASSERT(!opt_ctx); |
| 2720 | model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx(); |
| 2721 | const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train); |
| 2722 | const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch); |
| 2723 | GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0); |
| 2724 | GGML_ASSERT(n_batch % n_ubatch == 0); |
| 2725 | |
| 2726 | ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); |
| 2727 | opt_params.opt_period = n_batch / n_ubatch; |
| 2728 | opt_params.get_opt_pars = lopt_params.get_opt_pars; |
| 2729 | opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud; |
| 2730 | opt_params.optimizer = lopt_params.optimizer_type; |
| 2731 | opt_ctx = ggml_opt_init(opt_params); |
| 2732 | |
| 2733 | llama_opt_param_filter param_filter = lopt_params.param_filter; |
| 2734 | void * param_filter_ud = lopt_params.param_filter_ud; |
| 2735 | |
| 2736 | //llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME |
| 2737 | llama_set_param(model->type_embd, param_filter, param_filter_ud); |
| 2738 | llama_set_param(model->pos_embd, param_filter, param_filter_ud); |
| 2739 | llama_set_param(model->tok_norm, param_filter, param_filter_ud); |
| 2740 | llama_set_param(model->tok_norm_b, param_filter, param_filter_ud); |
| 2741 | llama_set_param(model->output_norm, param_filter, param_filter_ud); |
| 2742 | llama_set_param(model->output_norm_b, param_filter, param_filter_ud); |
| 2743 | llama_set_param(model->output, param_filter, param_filter_ud); |
| 2744 | llama_set_param(model->output_b, param_filter, param_filter_ud); |
| 2745 | llama_set_param(model->output_norm_enc, param_filter, param_filter_ud); |
| 2746 | llama_set_param(model->cls, param_filter, param_filter_ud); |
| 2747 | llama_set_param(model->cls_b, param_filter, param_filter_ud); |
| 2748 | llama_set_param(model->cls_out, param_filter, param_filter_ud); |
| 2749 | llama_set_param(model->cls_out_b, param_filter, param_filter_ud); |
| 2750 | llama_set_param(model->cls_norm, param_filter, param_filter_ud); |
| 2751 | |
| 2752 | for (struct llama_layer & layer : model->layers) { |
| 2753 | for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) { |
| 2754 | llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud); |
| 2755 | } |
| 2756 | } |
| 2757 | } |
| 2758 | |
| 2759 | void llama_context::opt_epoch_iter( |
| 2760 | ggml_opt_dataset_t dataset, |
no test coverage detected