Method apply

src/execution_algorithms/ltfb.cpp:44–103

namespace lbann {

void LTFB::apply(ExecutionContext& context,
                 model& m,
                 data_coordinator& dc,
                 execution_mode /*mode*/)
{
  LBANN_CALIPER_MARK_FUNCTION;
  TimerMap ltfb_timer(build_string("LTFB::",
                                   this->get_name(),
                                   " (trainer:",
                                   get_trainer().get_comm()->get_trainer_rank(),
                                   ")"));

  auto const& ltfb_term = m_termination_criteria;
  auto& ltfb_ctxt = dynamic_cast<ExeContextType&>(context);

  // Sync trainers (Assumption: all trainers in this lbann_comm are
  // participating in this training algorithm)
  m.get_comm()->intertrainer_barrier();

  // LTFB likely has different stopping criteria than SGD (e.g., K
  // tournament rounds; some specified relative or absolute
  // reduction in objective function value; etc.), or its stopping
  // criteria might be defined in terms of the SGD stopping criteria
  // (e.g., N total sgd batches). That complexity lives in the
  // ltfb::TerminationCriteria class.
  while (!ltfb_term(ltfb_ctxt)) {
    {
      ScopeTimer _(ltfb_timer, "local apply");
      m_local_algo->apply(m, dc);
    }
    {
      ScopeTimer _(ltfb_timer, "metalearning strategy");
      if (m.get_comm()->get_grid_type() == GridType::NO_GRID or
          m.get_comm()->get_grid_type() == GridType::PRIMARY_GRID or
          m.get_comm()->get_KFAC_subgrid_create_two_models()) {
        m_meta_learning_strategy->select_next(m, ltfb_ctxt, dc);
      }
    }

    ltfb_ctxt.inc_step();
  }

  // Final sweep of local training. The timer is looped into the inner
  // loop "local apply" timer.
  {
    ScopeTimer _(ltfb_timer, "local apply");
    m_local_algo->apply(m, dc);
  }

  if (!m_suppress_timer && m.get_comm()->am_trainer_master())
    ltfb_timer.print(std::cout);

  // TODO: How do we support aggregate outputs? What does "output"
  // mean here? Do we communicate among all trainers and just write
  // some interesting subset to disk? Top-k best models, e.g.
  //
  // maybe:
  //