| 128 | } |
| 129 | |
| 130 | void RegularizedEvolution::select_next(model& m, |
| 131 | ltfb::LTFBExecutionContext& ctxt, |
| 132 | data_coordinator& dc) const |
| 133 | { |
| 134 | auto const& comm = *(m.get_comm()); |
| 135 | const unsigned int num_trainers = comm.get_num_trainers(); |
| 136 | const int trainer_id = comm.get_trainer_rank(); |
| 137 | auto const step = ctxt.get_step(); |
| 138 | |
| 139 | std::vector<unsigned> sample_trainers(num_trainers); |
| 140 | if (comm.am_world_master()) { |
| 141 | std::iota(begin(sample_trainers), end(sample_trainers), 0U); |
| 142 | std::shuffle(sample_trainers.begin(), |
| 143 | sample_trainers.end(), |
| 144 | get_ltfb_generator()); |
| 145 | |
| 146 | // Print trainers selected in sample |
| 147 | std::cout << "Trainers in sample at step " << step << " -"; |
| 148 | for (int i = 0; i < m_sample_size; i++) |
| 149 | std::cout << " " << sample_trainers[i]; |
| 150 | std::cout << std::endl; |
| 151 | } |
| 152 | comm.world_broadcast(comm.get_world_master(), |
| 153 | sample_trainers.data(), |
| 154 | num_trainers); |
| 155 | |
| 156 | El::Int score = evaluate_model(m, ctxt, dc); |
| 157 | |
| 158 | // AllGather scores from all trainers |
| 159 | std::vector<EvalType> score_list_all(num_trainers); |
| 160 | comm.trainer_barrier(); |
| 161 | if (comm.am_trainer_master()) { |
| 162 | comm.all_gather<EvalType>(score, |
| 163 | score_list_all, |
| 164 | comm.get_intertrainer_comm()); |
| 165 | } |
| 166 | |
| 167 | // Use scores only for samples selected from sample_trainers above |
| 168 | // and place them in the same order as in sample_trainers |
| 169 | std::vector<EvalType> score_list_samples(m_sample_size); |
| 170 | for (int i = 0; i < m_sample_size; i++) { |
| 171 | score_list_samples[i] = score_list_all[sample_trainers[i]]; |
| 172 | } |
| 173 | |
| 174 | // Communicate sample score list from trainer master to other procs in |
| 175 | // trainer |
| 176 | comm.trainer_broadcast(comm.get_trainer_master(), |
| 177 | score_list_samples.data(), |
| 178 | m_sample_size); |
| 179 | |
| 180 | // Find winning trainer in sample according to metric strategy |
| 181 | El::Int winner_id; |
| 182 | if (m_metric_strategy == |
| 183 | RegularizedEvolution::metric_strategy::HIGHER_IS_BETTER) |
| 184 | winner_id = sample_trainers[std::distance( |
| 185 | score_list_samples.begin(), |
| 186 | std::max_element(score_list_samples.begin(), score_list_samples.end()))]; |
| 187 | else if (m_metric_strategy == |
no test coverage detected