| 35 | |
| 36 | |
| 37 | def create_parser(): |
| 38 | parser = ArgumentParser( |
| 39 | description='versatile benchmark output compare tool') |
| 40 | |
| 41 | parser.add_argument( |
| 42 | '-a', |
| 43 | '--display_aggregates_only', |
| 44 | dest='display_aggregates_only', |
| 45 | action="store_true", |
| 46 | help="If there are repetitions, by default, we display everything - the" |
| 47 | " actual runs, and the aggregates computed. Sometimes, it is " |
| 48 | "desirable to only view the aggregates. E.g. when there are a lot " |
| 49 | "of repetitions. Do note that only the display is affected. " |
| 50 | "Internally, all the actual runs are still used, e.g. for U test.") |
| 51 | |
| 52 | parser.add_argument( |
| 53 | '--no-color', |
| 54 | dest='color', |
| 55 | default=True, |
| 56 | action="store_false", |
| 57 | help="Do not use colors in the terminal output" |
| 58 | ) |
| 59 | |
| 60 | parser.add_argument( |
| 61 | '-d', |
| 62 | '--dump_to_json', |
| 63 | dest='dump_to_json', |
| 64 | help="Additionally, dump benchmark comparison output to this file in JSON format.") |
| 65 | |
| 66 | utest = parser.add_argument_group() |
| 67 | utest.add_argument( |
| 68 | '--no-utest', |
| 69 | dest='utest', |
| 70 | default=True, |
| 71 | action="store_false", |
| 72 | help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS)) |
| 73 | alpha_default = 0.05 |
| 74 | utest.add_argument( |
| 75 | "--alpha", |
| 76 | dest='utest_alpha', |
| 77 | default=alpha_default, |
| 78 | type=float, |
| 79 | help=("significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)") % |
| 80 | alpha_default) |
| 81 | |
| 82 | subparsers = parser.add_subparsers( |
| 83 | help='This tool has multiple modes of operation:', |
| 84 | dest='mode') |
| 85 | |
| 86 | parser_a = subparsers.add_parser( |
| 87 | 'benchmarks', |
| 88 | help='The most simple use-case, compare all the output of these two benchmarks') |
| 89 | baseline = parser_a.add_argument_group( |
| 90 | 'baseline', 'The benchmark baseline') |
| 91 | baseline.add_argument( |
| 92 | 'test_baseline', |
| 93 | metavar='test_baseline', |
| 94 | type=argparse.FileType('r'), |