| 779 | } |
| 780 | |
| 781 | void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result) { |
| 782 | GGML_ASSERT(opt_ctx->eval_ready); |
| 783 | if (opt_ctx->allocated_graph == opt_ctx->gb_opt) { |
| 784 | const ggml_opt_optimizer_params & opt_pars = opt_ctx->get_opt_pars(opt_ctx->get_opt_pars_ud); |
| 785 | |
| 786 | switch (opt_ctx->optimizer) { |
| 787 | case GGML_OPT_OPTIMIZER_TYPE_ADAMW: { |
| 788 | GGML_ASSERT(opt_pars.adamw.alpha > 0.0f); |
| 789 | GGML_ASSERT(opt_pars.adamw.beta1 >= 0.0f); |
| 790 | GGML_ASSERT(opt_pars.adamw.beta1 <= 1.0f); |
| 791 | GGML_ASSERT(opt_pars.adamw.beta2 >= 0.0f); |
| 792 | GGML_ASSERT(opt_pars.adamw.beta2 <= 1.0f); |
| 793 | GGML_ASSERT(opt_pars.adamw.eps >= 0.0f); |
| 794 | GGML_ASSERT(opt_pars.adamw.wd >= 0.0f); |
| 795 | GGML_ASSERT(opt_pars.adamw.wd <= 1.0f); |
| 796 | |
| 797 | // beta1, beta2 after applying warmup |
| 798 | const float beta1h = 1.0f / (1.0f - powf(opt_pars.adamw.beta1, opt_ctx->iter)); |
| 799 | const float beta2h = 1.0f / (1.0f - powf(opt_pars.adamw.beta2, opt_ctx->iter)); |
| 800 | |
| 801 | float * adamw_par_data = ggml_get_data_f32(opt_ctx->opt_step_params); |
| 802 | adamw_par_data[0] = opt_pars.adamw.alpha; |
| 803 | adamw_par_data[1] = opt_pars.adamw.beta1; |
| 804 | adamw_par_data[2] = opt_pars.adamw.beta2; |
| 805 | adamw_par_data[3] = opt_pars.adamw.eps; |
| 806 | adamw_par_data[4] = opt_pars.adamw.wd; |
| 807 | adamw_par_data[5] = beta1h; |
| 808 | adamw_par_data[6] = beta2h; |
| 809 | } break; |
| 810 | case GGML_OPT_OPTIMIZER_TYPE_SGD: { |
| 811 | GGML_ASSERT(opt_pars.sgd.alpha > 0.0f); |
| 812 | GGML_ASSERT(opt_pars.sgd.wd >= 0.0f); |
| 813 | GGML_ASSERT(opt_pars.sgd.wd <= 1.0f); |
| 814 | float * sgd = ggml_get_data_f32(opt_ctx->opt_step_params); |
| 815 | sgd[0] = opt_pars.sgd.alpha; |
| 816 | sgd[1] = opt_pars.sgd.wd; |
| 817 | } break; |
| 818 | default: |
| 819 | GGML_ABORT("fatal error"); |
| 820 | } |
| 821 | } |
| 822 | |
| 823 | ggml_backend_sched_graph_compute(opt_ctx->backend_sched, opt_ctx->allocated_graph_copy); |
| 824 | opt_ctx->iter += opt_ctx->allocated_graph == opt_ctx->gb_opt; |
| 825 | opt_ctx->opt_i = (opt_ctx->opt_i + 1) % opt_ctx->opt_period; |
| 826 | |
| 827 | if (!opt_ctx->static_graphs) { |
| 828 | opt_ctx->gf = nullptr; |
| 829 | opt_ctx->gb_grad = nullptr; |
| 830 | opt_ctx->gb_opt = nullptr; |
| 831 | opt_ctx->allocated_graph = nullptr; |
| 832 | opt_ctx->allocated_graph_copy = nullptr; |
| 833 | } |
| 834 | |
| 835 | opt_ctx->eval_ready = false; |
| 836 | |
| 837 | if (!result) { |
| 838 | return; |