| 18 | |
| 19 | |
| 20 | class BenchmarkRunner: |
| 21 | def __init__(self, database: Database, workdir: Path, exit_on_error=True): |
| 22 | self.database = database |
| 23 | self.workdir = workdir.resolve() |
| 24 | self.workdir.mkdir(parents=True, exist_ok=True) |
| 25 | self.exit_on_error = exit_on_error |
| 26 | |
| 27 | def materialize_and_skip(self, descriptors: List[BenchmarkDescriptor]) -> List[BenchmarkInstance]: |
| 28 | instances = create_benchmark_instances(descriptors, workdir=self.workdir) |
| 29 | return self._skip_completed(instances) |
| 30 | |
| 31 | def compute_materialized( |
| 32 | self, instances: List[BenchmarkInstance] |
| 33 | ) -> Iterable[Tuple[BenchmarkInstance, BenchmarkResult]]: |
| 34 | for instance in instances: |
| 35 | identifier = instance.identifier |
| 36 | |
| 37 | logging.info(f"Executing benchmark {identifier}") |
| 38 | ctx = BenchmarkContext(workdir=Path(identifier.workdir), timeout_s=identifier.timeout) |
| 39 | executor = self._create_executor(instance.descriptor) |
| 40 | |
| 41 | try: |
| 42 | result = executor.execute(instance.descriptor, ctx=ctx) |
| 43 | except BaseException: |
| 44 | tb = traceback.format_exc() |
| 45 | logging.error(f"Unexpected benchmarking error has occurred: {tb}") |
| 46 | result = Failure(tb) |
| 47 | |
| 48 | self._handle_result(identifier, result) |
| 49 | |
| 50 | yield (instance, result) |
| 51 | |
| 52 | def compute(self, descriptors: List[BenchmarkDescriptor]) -> Iterable[Tuple[BenchmarkInstance, BenchmarkResult]]: |
| 53 | instances = self.materialize_and_skip(descriptors) |
| 54 | yield from self.compute_materialized(instances) |
| 55 | |
| 56 | def save(self): |
| 57 | self.database.save() |
| 58 | summary_txt = self.workdir / "summary-wip.txt" |
| 59 | generate_summary_text(self.database, summary_txt) |
| 60 | |
| 61 | def _skip_completed(self, infos: List[BenchmarkInstance]) -> List[BenchmarkInstance]: |
| 62 | not_completed = [] |
| 63 | visited = set() |
| 64 | skipped = 0 |
| 65 | |
| 66 | for info in infos: |
| 67 | key = info.identifier.key |
| 68 | if key in visited: |
| 69 | raise Exception(f"Duplicated identifier: {info.identifier} in {infos}") |
| 70 | visited.add(key) |
| 71 | |
| 72 | if not self.database.has_record_for(info.identifier): |
| 73 | not_completed.append(info) |
| 74 | else: |
| 75 | skipped += 1 |
| 76 | |
| 77 | total_count = skipped + len(not_completed) |
no outgoing calls
no test coverage detected