Checks if the mean of the results for a given trace config is within 0.1% of the true value with the specified confidence level. This assumes Gaussian distribution of the noise and based on https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule. Args: graph_config: An
(self, graph_config, confidence_level)
| 251 | f.write(json.dumps(self.ToDict())) |
| 252 | |
| 253 | def HasEnoughRuns(self, graph_config, confidence_level): |
| 254 | """Checks if the mean of the results for a given trace config is within |
| 255 | 0.1% of the true value with the specified confidence level. |
| 256 | |
| 257 | This assumes Gaussian distribution of the noise and based on |
| 258 | https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule. |
| 259 | |
| 260 | Args: |
| 261 | graph_config: An instance of GraphConfig. |
| 262 | confidence_level: Number of standard deviations from the mean that all |
| 263 | values must lie within. Typical values are 1, 2 and 3 and correspond |
| 264 | to 68%, 95% and 99.7% probability that the measured value is within |
| 265 | 0.1% of the true value. |
| 266 | |
| 267 | Returns: |
| 268 | True if specified confidence level have been achieved. |
| 269 | """ |
| 270 | if not isinstance(graph_config, LeafTraceConfig): |
| 271 | return all(self.HasEnoughRuns(child, confidence_level) |
| 272 | for child in graph_config.children) |
| 273 | |
| 274 | trace = self.traces.get(graph_config.name, {}) |
| 275 | results = trace.get('results', []) |
| 276 | logging.debug('HasEnoughRuns for %s', graph_config.name) |
| 277 | |
| 278 | if len(results) < MIN_RUNS_FOR_CONFIDENCE: |
| 279 | logging.debug(' Ran %d times, need at least %d', |
| 280 | len(results), MIN_RUNS_FOR_CONFIDENCE) |
| 281 | return False |
| 282 | |
| 283 | logging.debug(' Results: %d entries', len(results)) |
| 284 | avg = mean(results) |
| 285 | avg_stderr = stdev(results) / sqrt(len(results)) |
| 286 | logging.debug(' Mean: %.2f, mean_stderr: %.2f', avg, avg_stderr) |
| 287 | logging.info('>>> Confidence level is %.2f', |
| 288 | avg / max(1000.0 * avg_stderr, .1)) |
| 289 | return confidence_level * avg_stderr < avg / 1000.0 |
| 290 | |
| 291 | def __str__(self): # pragma: no cover |
| 292 | return json.dumps(self.ToDict(), indent=2, separators=(',', ': ')) |
no test coverage detected