| 57 | void summary::on_train_begin(model* m) { save_histograms(m); } |
| 58 | |
| 59 | void summary::on_batch_end(model* m) |
| 60 | { |
| 61 | if (!m_summarizer) { |
| 62 | LBANN_ERROR("Summary callback failed: m_summarizer does not exist."); |
| 63 | } |
| 64 | |
| 65 | const auto& c = m->get_execution_context(); |
| 66 | |
| 67 | prof_region_begin("summary-batch", prof_colors[0], false); |
| 68 | m->summarize_stats(*m_summarizer); |
| 69 | if (m_mat_interval > 0 && c.get_step() % m_mat_interval == 0) { |
| 70 | m->summarize_matrices(*m_summarizer); |
| 71 | } |
| 72 | lbann_comm* comm = m->get_comm(); |
| 73 | size_t bytes_sent = comm->get_bytes_sent(); |
| 74 | size_t bytes_received = comm->get_bytes_received(); |
| 75 | size_t trainer_barriers = comm->get_num_trainer_barriers(); |
| 76 | size_t intertrainer_barriers = comm->get_num_intertrainer_barriers(); |
| 77 | size_t global_barriers = comm->get_num_global_barriers(); |
| 78 | comm->reset_stats_counters(); |
| 79 | m_summarizer->sum_reduce_scalar("bytes_sent", bytes_sent, c.get_step()); |
| 80 | m_summarizer->sum_reduce_scalar("bytes_received", |
| 81 | bytes_received, |
| 82 | c.get_step()); |
| 83 | m_summarizer->reduce_scalar("trainer_barriers", |
| 84 | trainer_barriers, |
| 85 | c.get_step()); |
| 86 | m_summarizer->reduce_scalar("intertrainer_barriers", |
| 87 | intertrainer_barriers, |
| 88 | c.get_step()); |
| 89 | m_summarizer->reduce_scalar("global_barriers", global_barriers, c.get_step()); |
| 90 | prof_region_end("summary-batch", false); |
| 91 | } |
| 92 | |
| 93 | void summary::on_epoch_end(model* m) |
| 94 | { |
nothing calls this directly
no test coverage detected