| 2748 | } |
| 2749 | |
| 2750 | void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) { |
| 2751 | GGML_ASSERT(!opt_ctx); |
| 2752 | model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx(); |
| 2753 | const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train); |
| 2754 | const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch); |
| 2755 | GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0); |
| 2756 | GGML_ASSERT(n_batch % n_ubatch == 0); |
| 2757 | |
| 2758 | ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); |
| 2759 | opt_params.opt_period = n_batch / n_ubatch; |
| 2760 | opt_params.get_opt_pars = lopt_params.get_opt_pars; |
| 2761 | opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud; |
| 2762 | opt_params.optimizer = lopt_params.optimizer_type; |
| 2763 | opt_ctx = ggml_opt_init(opt_params); |
| 2764 | |
| 2765 | llama_opt_param_filter param_filter = lopt_params.param_filter; |
| 2766 | void * param_filter_ud = lopt_params.param_filter_ud; |
| 2767 | |
| 2768 | //llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME |
| 2769 | llama_set_param(model->type_embd, param_filter, param_filter_ud); |
| 2770 | llama_set_param(model->pos_embd, param_filter, param_filter_ud); |
| 2771 | llama_set_param(model->tok_norm, param_filter, param_filter_ud); |
| 2772 | llama_set_param(model->tok_norm_b, param_filter, param_filter_ud); |
| 2773 | llama_set_param(model->output_norm, param_filter, param_filter_ud); |
| 2774 | llama_set_param(model->output_norm_b, param_filter, param_filter_ud); |
| 2775 | llama_set_param(model->output, param_filter, param_filter_ud); |
| 2776 | llama_set_param(model->output_b, param_filter, param_filter_ud); |
| 2777 | llama_set_param(model->output_norm_enc, param_filter, param_filter_ud); |
| 2778 | llama_set_param(model->cls, param_filter, param_filter_ud); |
| 2779 | llama_set_param(model->cls_b, param_filter, param_filter_ud); |
| 2780 | llama_set_param(model->cls_out, param_filter, param_filter_ud); |
| 2781 | llama_set_param(model->cls_out_b, param_filter, param_filter_ud); |
| 2782 | |
| 2783 | for (struct llama_layer & layer : model->layers) { |
| 2784 | for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) { |
| 2785 | llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud); |
| 2786 | } |
| 2787 | } |
| 2788 | } |
| 2789 | |
| 2790 | void llama_context::opt_epoch_iter( |
| 2791 | ggml_opt_dataset_t dataset, |
no test coverage detected