Get mean metric value with validation set. */
| 658 | |
| 659 | /** Get mean metric value with validation set. */ |
| 660 | EvalType evaluate(model& m, const std::string& metric_name) |
| 661 | { |
| 662 | auto& c = m.get_execution_context(); |
| 663 | // Make sure data readers finish asynchronous work |
| 664 | const auto original_mode = c.get_execution_mode(); |
| 665 | data_coordinator& dc = get_trainer().get_data_coordinator(); |
| 666 | dc.collect_background_data_fetch(original_mode); |
| 667 | |
| 668 | if (!dc.is_execution_mode_valid(execution_mode::tournament)) { |
| 669 | LBANN_ERROR("LTFB requires ", |
| 670 | to_string(execution_mode::tournament), |
| 671 | " execution mode"); |
| 672 | } |
| 673 | // Mark the data store as loading - Note that this is a temporary fix |
| 674 | // for the current use of the tournament |
| 675 | dc.mark_data_store_explicitly_loading(execution_mode::tournament); |
| 676 | |
| 677 | // Evaluate model on validation set |
| 678 | get_trainer().evaluate(&m, execution_mode::tournament); |
| 679 | |
| 680 | // Get metric value |
| 681 | bool found_metric = false; |
| 682 | EvalType metric_value = 0; |
| 683 | for (const auto& met : m.get_metrics()) { |
| 684 | if (met->name() == metric_name) { |
| 685 | found_metric = true; |
| 686 | metric_value = met->get_mean_value(execution_mode::tournament); |
| 687 | break; |
| 688 | } |
| 689 | } |
| 690 | if (!found_metric) { |
| 691 | LBANN_ERROR("could not find metric \"", |
| 692 | metric_name, |
| 693 | "\" ", |
| 694 | "in model \"", |
| 695 | m.get_name(), |
| 696 | "\""); |
| 697 | } |
| 698 | |
| 699 | // Mark the data store as loaded - Note that this is a temporary fix |
| 700 | // for the current use of the tournament |
| 701 | dc.make_data_store_preloaded(execution_mode::tournament); |
| 702 | |
| 703 | // Clean up and return metric value |
| 704 | m.reset_mode(c, original_mode); |
| 705 | dc.reset_mode(c); |
| 706 | return metric_value; |
| 707 | } |
| 708 | |
| 709 | } // namespace |
| 710 |
no test coverage detected