Parse the benchmark result and extract relevant information, namely: GEMM param, Strategy, GEMM config, Measurements
(
json_results: Dict, measurement_method="avg"
)
| 486 | |
| 487 | |
| 488 | def extract_benchmark_results( |
| 489 | json_results: Dict, measurement_method="avg" |
| 490 | ) -> Generator[BenchmarkResult, None, None]: |
| 491 | """ Parse the benchmark result and extract relevant information, namely: |
| 492 | GEMM param, |
| 493 | Strategy, |
| 494 | GEMM config, |
| 495 | Measurements |
| 496 | """ |
| 497 | for json_res in json_results: |
| 498 | # Get example test and test data. |
| 499 | # There should only be 1 test per run |
| 500 | example_tests = list(json_res["tests"].items()) |
| 501 | assert len(example_tests) == 1 |
| 502 | example_fn, example_test_data = example_tests[0] |
| 503 | |
| 504 | # Process example file name |
| 505 | example_fn = example_fn.split(os.path.sep)[-1] |
| 506 | |
| 507 | # Get strategy |
| 508 | strategy = EXAMPLE_FILE_2_STRATEGY[example_fn] |
| 509 | |
| 510 | # Get gemm params + gemm configs from example args |
| 511 | benchmark_args = parse_benchmark_commandline(json_res["CommandLine"]) |
| 512 | Gemm_Example_Args_T = GEMM_EXAMPLE_ARGS_FACTORY[strategy] |
| 513 | example_args = Gemm_Example_Args_T( |
| 514 | *(benchmark_args["example_args"].split(","))) |
| 515 | # Gemm_Example_Arg consists of GEMMParam first and then GEMMConfig (in that order) |
| 516 | # However data type option is parsed separately from end of options, hence -1 is applied to fields length |
| 517 | gemm_param_fields_len = len(GEMMParam._fields) - 1 |
| 518 | gemm_param = GEMMParam.parse_from_strs( |
| 519 | *example_args[:gemm_param_fields_len], |
| 520 | data_type = benchmark_args["type"]) |
| 521 | GEMMConfig = GEMM_CONFIG_FACTORY[strategy] |
| 522 | gemm_config = GEMMConfig.parse_from_strs( |
| 523 | *example_args[gemm_param_fields_len:]) |
| 524 | |
| 525 | # Get OpenCL_Time_Ms stats |
| 526 | measurements = list(example_test_data["measurements"].items()) |
| 527 | # For reshaped RHS only we have two measurements (one also for the reshape kernel) |
| 528 | # Hence we must parse and sum them |
| 529 | measurement_ms_reshape = 0 |
| 530 | measurement_ms_kernel = 0 |
| 531 | for single_measurement in measurements: |
| 532 | measurement_instrument, data = single_measurement |
| 533 | # Get instrument name and assert that it is the one we expect |
| 534 | measurement_instrument_name = measurement_instrument.split("/")[0] |
| 535 | assert measurement_instrument_name == "OpenCLTimer" |
| 536 | # Take either the minimum or the average of the raw data as the measurement value |
| 537 | if measurement_method == "min": |
| 538 | measurement_val = min(data["raw"]) |
| 539 | elif measurement_method == "avg": |
| 540 | measurement_val = sum(data["raw"]) / len(data["raw"]) |
| 541 | else: |
| 542 | raise ValueError( |
| 543 | "Invalid measurement method: {}".format(measurement_method) |
| 544 | ) |
| 545 |
no test coverage detected