(
task: BenchmarkTask,
data_cache_dir: Path,
random_seed: int,
)
| 51 | |
| 52 | |
| 53 | def _collect_metrics( |
| 54 | task: BenchmarkTask, |
| 55 | data_cache_dir: Path, |
| 56 | random_seed: int, |
| 57 | ) -> Optional[Mapping[Metric, float]]: |
| 58 | print("Loading dataset:", task.dataset_name) |
| 59 | dataset_fac = DATASET_FACTORIES.get(task.dataset_name) |
| 60 | dataset = dataset_fac.create_dataset(data_cache_dir) |
| 61 | print(tabulate(dataset.stats, showindex="never")) |
| 62 | train_data = dataset.train |
| 63 | test_data = dataset.test |
| 64 | |
| 65 | rng = np.random.default_rng(random_seed) |
| 66 | model_fac = MODEL_FACTORIES.get(task.model_name) |
| 67 | model = model_fac.create_model(train_data, rng) |
| 68 | |
| 69 | metrics = {} |
| 70 | |
| 71 | model.predict_y(test_data.X) # Warm-up TF. |
| 72 | |
| 73 | print("Model before training:") |
| 74 | gpflow.utilities.print_summary(model) |
| 75 | |
| 76 | if task.do_optimise: |
| 77 | t_before = perf_counter() |
| 78 | loss_fn = gpflow.models.training_loss_closure(model, train_data.XY, compile=task.do_compile) |
| 79 | opt_log = gpflow.optimizers.Scipy().minimize( |
| 80 | loss_fn, |
| 81 | variables=model.trainable_variables, |
| 82 | compile=task.do_compile, |
| 83 | options={"disp": 10, "maxiter": 1_000}, |
| 84 | ) |
| 85 | t_after = perf_counter() |
| 86 | n_iter = opt_log.nit |
| 87 | t_train = t_after - t_before |
| 88 | metrics[mt.n_training_iterations] = n_iter |
| 89 | metrics[mt.training_time] = t_train |
| 90 | metrics[mt.training_iteration_time] = t_train / n_iter |
| 91 | print(f"Training took {t_after - t_before}s for {n_iter} iterations.") |
| 92 | print("Model after training:") |
| 93 | gpflow.utilities.print_summary(model) |
| 94 | |
| 95 | likelihood = model.likelihood |
| 96 | |
| 97 | if task.do_predict: |
| 98 | t_before = perf_counter() |
| 99 | f_m, f_v = model.predict_f(test_data.X) |
| 100 | t_after = perf_counter() |
| 101 | metrics[mt.prediction_time] = t_after - t_before |
| 102 | |
| 103 | metrics[mt.nlpd] = -np.sum( |
| 104 | likelihood.predict_log_density(test_data.X, f_m, f_v, test_data.Y) |
| 105 | ) |
| 106 | |
| 107 | y_m, _y_v = likelihood.predict_mean_and_var(test_data.X, f_m, f_v) |
| 108 | error = test_data.Y - y_m |
| 109 | metrics[mt.mae] = np.average(np.abs(error)) |
| 110 | metrics[mt.rmse] = np.average(error ** 2) ** 0.5 |
no test coverage detected
searching dependent graphs…