A recorder that records and organises GEMM Benchmark results, and produces various reports on the record.
| 199 | |
| 200 | |
| 201 | class GEMMBenchmarkResultRecorder: |
| 202 | """ A recorder that records and organises GEMM Benchmark results, and produces various reports on the record. |
| 203 | """ |
| 204 | |
| 205 | SummaryLevel = Enum("SummaryLevel", ["Short", "Detailed"]) |
| 206 | |
| 207 | def __init__(self, tol=0.01): |
| 208 | """ Initializer |
| 209 | """ |
| 210 | self._benchmark_result_record: List[BenchmarkResult] = [] |
| 211 | # Strategies recorded |
| 212 | self._strategies = set() |
| 213 | self._tol = tol |
| 214 | |
| 215 | def add(self, benchmark_result: BenchmarkResult): |
| 216 | """ Add a benchmark result to the record. |
| 217 | """ |
| 218 | gemm_param, strategy, gemm_config, measurement = benchmark_result |
| 219 | # Update strategies encoutnered |
| 220 | self._strategies.add(strategy) |
| 221 | |
| 222 | self._benchmark_result_record.append(benchmark_result) |
| 223 | |
| 224 | def get_record(self) -> Generator[BenchmarkResult, None, None]: |
| 225 | """ Return an iterator that iterates over the record. |
| 226 | """ |
| 227 | yield from self._benchmark_result_record |
| 228 | |
| 229 | def get_best_gemm_configs(self): |
| 230 | """ Get the best GEMMConfig set per GEMMParam per Strategy |
| 231 | """ |
| 232 | best_gc_sets: Dict[ |
| 233 | Tuple[GEMMParam, Strategy], List[Tuple[GEMMConfig, Measurement]] |
| 234 | ] = defaultdict(list) |
| 235 | for gemm_param, strategy, gemm_config, measurement in self.get_record(): |
| 236 | best_gc_set = best_gc_sets.setdefault((gemm_param, strategy), []) |
| 237 | best_gc_set.append((gemm_config, measurement)) |
| 238 | # Sort the best config set (list) |
| 239 | best_gc_set = sorted( |
| 240 | best_gc_set, key=lambda gc_and_m: gc_and_m[1].get_total_ms() |
| 241 | ) |
| 242 | # Filter out configs that are beyond tolerance to the best GEMMConfig's measurement |
| 243 | best_gc, best_m = best_gc_set[0] |
| 244 | best_gc_set_new = [ |
| 245 | (gemm_config, measurement) |
| 246 | for gemm_config, measurement in best_gc_set[1:] |
| 247 | if measurement.is_close_to(best_m, self._tol) |
| 248 | ] |
| 249 | # Add back the best config |
| 250 | best_gc_set_new.insert(0, (best_gc, best_m)) |
| 251 | best_gc_sets[(gemm_param, strategy)] = best_gc_set_new |
| 252 | |
| 253 | return best_gc_sets |
| 254 | |
| 255 | def get_best_gemm_configs_as_sequence(self): |
| 256 | """ Get the best GEMMConfig set per GEMMParam per Strategy, and flatten the result into a sequence |
| 257 | of BenchmarkResults |
| 258 | """ |