| 578 | } |
| 579 | |
| 580 | static std::pair<int, int> test_gradient_accumulation( |
| 581 | ggml_backend_sched_t backend_sched, ggml_backend_t backend, const int32_t nbatch_physical, const enum ggml_opt_loss_type loss_type) { |
| 582 | int ntest = 0; |
| 583 | int npass = 0; |
| 584 | |
| 585 | struct helper_ctx_data cd = helper_get_ctx_data( |
| 586 | backend_sched, backend, /*init_opt_ctx =*/ true, /*optimizer_defaults =*/ false, /*nbatch_logical =*/ 6, nbatch_physical, loss_type); |
| 587 | |
| 588 | std::vector<float> grad_history(ndata); |
| 589 | for (int64_t idata = 0; idata < ndata; ++idata) { |
| 590 | grad_history[idata] = NAN; |
| 591 | } |
| 592 | |
| 593 | for (int epoch = 1; epoch <= 4; ++epoch) { |
| 594 | if (nbatch_physical == 1) { |
| 595 | for (int idata = 0; idata < ndata; ++idata) { |
| 596 | const float idataf = idata; |
| 597 | ggml_opt_alloc(cd.opt_ctx, /*backward =*/ true); |
| 598 | ggml_backend_tensor_set(cd.inputs, &idataf, 0, 1*sizeof(float)); |
| 599 | ggml_opt_eval(cd.opt_ctx, cd.result); |
| 600 | ggml_backend_tensor_get(ggml_opt_grad_acc(cd.opt_ctx, cd.weights), grad_history.data() + idata, 0, 1*sizeof(float)); |
| 601 | } |
| 602 | } else if (nbatch_physical == 2) { |
| 603 | for (int idata = 0; idata < ndata; idata += 2) { |
| 604 | const float idataf[2] = {float(idata + 0), float(idata + 1)}; |
| 605 | ggml_opt_alloc(cd.opt_ctx, /*backward =*/ true); |
| 606 | ggml_backend_tensor_set(cd.inputs, idataf, 0, 2*sizeof(float)); |
| 607 | ggml_opt_eval(cd.opt_ctx, cd.result); |
| 608 | |
| 609 | grad_history[idata + 0] = 0.0f; |
| 610 | ggml_backend_tensor_get(ggml_opt_grad_acc(cd.opt_ctx, cd.weights), grad_history.data() + idata + 1, 0, 1*sizeof(float)); |
| 611 | } |
| 612 | } else { |
| 613 | GGML_ASSERT(false); |
| 614 | } |
| 615 | |
| 616 | { |
| 617 | GGML_ASSERT(ndata == 6); |
| 618 | constexpr double atol = 1e-6; |
| 619 | bool subtest_ok = true; |
| 620 | if (loss_type == GGML_OPT_LOSS_TYPE_SUM) { |
| 621 | if (nbatch_physical == 1) { |
| 622 | subtest_ok = subtest_ok && almost_equal(grad_history[0], 1.0, atol); |
| 623 | subtest_ok = subtest_ok && almost_equal(grad_history[2], 3.0, atol); |
| 624 | subtest_ok = subtest_ok && almost_equal(grad_history[4], 5.0, atol); |
| 625 | } else { |
| 626 | subtest_ok = subtest_ok && almost_equal(grad_history[0], 0.0, atol); |
| 627 | subtest_ok = subtest_ok && almost_equal(grad_history[2], 0.0, atol); |
| 628 | subtest_ok = subtest_ok && almost_equal(grad_history[4], 0.0, atol); |
| 629 | } |
| 630 | subtest_ok = subtest_ok && almost_equal(grad_history[1], 2.0, atol); |
| 631 | subtest_ok = subtest_ok && almost_equal(grad_history[3], 4.0, atol); |
| 632 | subtest_ok = subtest_ok && almost_equal(grad_history[5], 6.0, atol); |
| 633 | } else if (loss_type == GGML_OPT_LOSS_TYPE_MEAN) { |
| 634 | if (nbatch_physical == 1) { |
| 635 | subtest_ok = subtest_ok && almost_equal(grad_history[0], 1.0/ndata, atol); |
| 636 | subtest_ok = subtest_ok && almost_equal(grad_history[2], 3.0/ndata, atol); |
| 637 | subtest_ok = subtest_ok && almost_equal(grad_history[4], 5.0/ndata, atol); |
no test coverage detected