Raw results of running the benchmark. Attributes: num_iter: the number of iterations over examples that were done. num_examples: the number of examples that were processed in these iterations. Note that when examples are batched, then one iteration processes multiple examples.
| 37 | |
| 38 | @dataclasses.dataclass(frozen=True) |
| 39 | class RawBenchmarkResult: |
| 40 | """Raw results of running the benchmark. |
| 41 | |
| 42 | Attributes: |
| 43 | num_iter: the number of iterations over examples that were done. |
| 44 | num_examples: the number of examples that were processed in these |
| 45 | iterations. Note that when examples are batched, then one iteration |
| 46 | processes multiple examples. |
| 47 | start_time: the time (in ns) at which when the benchmark started. |
| 48 | first_batch_time: the time (in ns) at which the first iteration was |
| 49 | processed. |
| 50 | end_time: the time (in ns) at which the benchmark ended. |
| 51 | batch_size: the number of examples in each iteration. |
| 52 | durations_ns: the duration in ns of each iteration that was processed. |
| 53 | """ |
| 54 | |
| 55 | num_iter: int |
| 56 | num_examples: int |
| 57 | start_time: int |
| 58 | first_batch_time: int |
| 59 | end_time: int |
| 60 | batch_size: int |
| 61 | durations_ns: Optional[List[int]] = None |
| 62 | |
| 63 | def examples(self, include_first: bool = True) -> int: |
| 64 | if include_first: |
| 65 | return self.num_examples |
| 66 | return self.num_examples - 1 |
| 67 | |
| 68 | def total_time_s(self, include_first: bool = True) -> float: |
| 69 | if include_first: |
| 70 | return _ns_to_s(self.end_time - self.start_time) |
| 71 | return _ns_to_s(self.end_time - self.first_batch_time) |
| 72 | |
| 73 | def examples_per_second(self, include_first: bool = True) -> float: |
| 74 | return self.examples(include_first) / self.total_time_s(include_first) |
| 75 | |
| 76 | def time_until_first(self, include_first: bool = True) -> Optional[float]: |
| 77 | """Time in seconds that it took to load the first example.""" |
| 78 | if include_first: |
| 79 | return _ns_to_s(self.first_batch_time - self.start_time) |
| 80 | if self.durations_ns is not None and len(self.durations_ns) > 1: |
| 81 | return _ns_to_s(self.durations_ns[1]) |
| 82 | return None |
| 83 | |
| 84 | def durations_s(self, include_first: bool = True) -> List[float]: |
| 85 | if not include_first: |
| 86 | return [_ns_to_s(d) for d in self.durations_ns[1:]] |
| 87 | return [_ns_to_s(d) for d in self.durations_ns] |
| 88 | |
| 89 | def summary_statistics( |
| 90 | self, include_first: bool |
| 91 | ) -> Dict[str, Union[float, List[float]]]: |
| 92 | if self.durations_ns is None: |
| 93 | return {} |
| 94 | durations = self.durations_s(include_first) |
| 95 | return { |
| 96 | 'mean': statistics.mean(durations), |