(self, hyperparameterSpace, results)
| 1416 | return statistics |
| 1417 | |
| 1418 | def computeAllResultStatistics(self, hyperparameterSpace, results): |
| 1419 | losses = numpy.array( |
| 1420 | sorted([result["loss"] for result in results if result["loss"] is not None]) |
| 1421 | ) |
| 1422 | |
| 1423 | if len(set(losses)) > 1: |
| 1424 | percentile10Loss = numpy.percentile(losses, 10) |
| 1425 | percentile20Loss = numpy.percentile(losses, 20) |
| 1426 | percentile30Loss = numpy.percentile(losses, 30) |
| 1427 | else: |
| 1428 | percentile10Loss = losses[0] |
| 1429 | percentile20Loss = losses[0] |
| 1430 | percentile30Loss = losses[0] |
| 1431 | |
| 1432 | allResults = list(results) |
| 1433 | percentile10Results = [ |
| 1434 | result |
| 1435 | for result in results |
| 1436 | if result["loss"] is not None and result["loss"] <= percentile10Loss |
| 1437 | ] |
| 1438 | percentile20Results = [ |
| 1439 | result |
| 1440 | for result in results |
| 1441 | if result["loss"] is not None and result["loss"] <= percentile20Loss |
| 1442 | ] |
| 1443 | percentile30Results = [ |
| 1444 | result |
| 1445 | for result in results |
| 1446 | if result["loss"] is not None and result["loss"] <= percentile30Loss |
| 1447 | ] |
| 1448 | |
| 1449 | recent10Count = min(len(results), 10) |
| 1450 | recent10Results = results[-recent10Count:] |
| 1451 | |
| 1452 | recent25Count = min(len(results), 25) |
| 1453 | recent25Results = results[-recent25Count:] |
| 1454 | |
| 1455 | recent15PercentCount = max(math.ceil(len(results) * 0.15), 5) |
| 1456 | recent15PercentResults = results[-recent15PercentCount:] |
| 1457 | |
| 1458 | statistics = {} |
| 1459 | allResultStatistics = self.computePartialResultStatistics( |
| 1460 | hyperparameterSpace, allResults |
| 1461 | ) |
| 1462 | for stat, value in allResultStatistics.items(): |
| 1463 | statistics["all_" + stat] = value |
| 1464 | |
| 1465 | percentile10Statistics = self.computePartialResultStatistics( |
| 1466 | hyperparameterSpace, percentile10Results |
| 1467 | ) |
| 1468 | for stat, value in percentile10Statistics.items(): |
| 1469 | statistics["top_10%_" + stat] = value |
| 1470 | |
| 1471 | percentile20Statistics = self.computePartialResultStatistics( |
| 1472 | hyperparameterSpace, percentile20Results |
| 1473 | ) |
| 1474 | for stat, value in percentile20Statistics.items(): |
| 1475 | statistics["top_20%_" + stat] = value |
no test coverage detected