Return the average best error of the experiment Average best error is defined as the average of bandit.true_loss, weighted by the probability that the corresponding bandit.loss is best. For domains with loss measurement variance of 0, this function simply returns th
(self, bandit=None)
| 542 | return list(map(bandit.status, self.results, self.specs)) |
| 543 | |
| 544 | def average_best_error(self, bandit=None): |
| 545 | """Return the average best error of the experiment |
| 546 | |
| 547 | Average best error is defined as the average of bandit.true_loss, |
| 548 | weighted by the probability that the corresponding bandit.loss is best. |
| 549 | |
| 550 | For domains with loss measurement variance of 0, this function simply |
| 551 | returns the true_loss corresponding to the result with the lowest loss. |
| 552 | """ |
| 553 | |
| 554 | if bandit is None: |
| 555 | results = self.results |
| 556 | loss = [r["loss"] for r in results if r["status"] == STATUS_OK] |
| 557 | loss_v = [ |
| 558 | r.get("loss_variance", 0) for r in results if r["status"] == STATUS_OK |
| 559 | ] |
| 560 | true_loss = [ |
| 561 | r.get("true_loss", r["loss"]) |
| 562 | for r in results |
| 563 | if r["status"] == STATUS_OK |
| 564 | ] |
| 565 | else: |
| 566 | |
| 567 | def fmap(f): |
| 568 | rval = np.asarray( |
| 569 | [ |
| 570 | f(r, s) |
| 571 | for (r, s) in zip(self.results, self.specs) |
| 572 | if bandit.status(r) == STATUS_OK |
| 573 | ] |
| 574 | ).astype("float") |
| 575 | if not np.all(np.isfinite(rval)): |
| 576 | raise ValueError() |
| 577 | return rval |
| 578 | |
| 579 | loss = fmap(bandit.loss) |
| 580 | loss_v = fmap(bandit.loss_variance) |
| 581 | true_loss = fmap(bandit.true_loss) |
| 582 | loss3 = list(zip(loss, loss_v, true_loss)) |
| 583 | if not loss3: |
| 584 | raise ValueError("Empty loss vector") |
| 585 | loss3.sort() |
| 586 | loss3 = np.asarray(loss3) |
| 587 | if np.all(loss3[:, 1] == 0): |
| 588 | best_idx = np.argmin(loss3[:, 0]) |
| 589 | return loss3[best_idx, 2] |
| 590 | else: |
| 591 | cutoff = 0 |
| 592 | sigma = np.sqrt(loss3[0][1]) |
| 593 | while cutoff < len(loss3) and loss3[cutoff][0] < loss3[0][0] + 3 * sigma: |
| 594 | cutoff += 1 |
| 595 | pmin = pmin_sampled(loss3[:cutoff, 0], loss3[:cutoff, 1]) |
| 596 | avg_true_loss = (pmin * loss3[:cutoff, 2]).sum() |
| 597 | return avg_true_loss |
| 598 | |
| 599 | @property |
| 600 | def best_trial(self): |