Method on_batch_begin

src/callbacks/ltfb.cpp:753–833

}

void ltfb::on_batch_begin(model* m)
{
  auto& local_model = *m;
  auto& context = local_model.get_execution_context();
  auto&& comm = *local_model.get_comm();

  // Check whether to start LTFB round
  const auto mode = context.get_execution_mode();
  const auto step = context.get_step();
  if (mode != execution_mode::training || step == 0) {
    return;
  }

  // Print message
  const auto message_prefix =
    (std::string{} + "LTFB (" + "model \"" + local_model.get_name() + "\", " +
     "step " + std::to_string(step) + "): ");
  if (comm.am_world_master()) {
    std::cout << message_prefix + "starting tournament...\n";
  }

  // Determine partner model for tournament
  const El::Int local_trainer = comm.get_trainer_rank();
  const El::Int partner_trainer = get_partner_trainer(comm, message_prefix);

  // Evaluate local model
  if (comm.am_world_master()) {
    std::cout << message_prefix + "evaluating local model...\n";
  }
  auto local_score = evaluate(local_model, m_metric_name);

  // Get model from partner trainer
  if (comm.am_world_master()) {
    std::cout << message_prefix + "exchanging model data...\n";
  }

  model partner_model(local_model);
  if (comm_algo_)
    comm_algo_->exchange_models(partner_model, partner_trainer, step);
  else
    LBANN_ERROR("No communication algorithm.");

  // Evaluate partner model
  if (comm.am_world_master()) {
    std::cout << message_prefix + "evaluating partner model...\n";
  }
  auto partner_score = evaluate(partner_model, m_metric_name);

  // Choose tournament winner
  // Note: restore local model data if it got a better score.
  El::Int tournament_winner = local_trainer;
  if ((m_low_score_wins && partner_score <= local_score) ||
      (!m_low_score_wins && partner_score >= local_score) ||
      (!std::isfinite(local_score) && std::isfinite(partner_score))) {
    tournament_winner = partner_trainer;

    /// @todo Use move assignment operator once LTFB is moved into a
    /// training algorithm