Function run

benchmark/run.py:249–301

Collect data for a benchmark suite.

(
    shard: ShardingStrategy,
    suite: BenchmarkSuite,
    dest: Path,
    no_subdir: bool,
    cache_dir: Path,
    plot: bool,
)

def run(
    shard: ShardingStrategy,
    suite: BenchmarkSuite,
    dest: Path,
    no_subdir: bool,
    cache_dir: Path,
    plot: bool,
) -> None:
    """ Collect data for a benchmark suite. """
    if shard.setup_dest:
        metadata = BenchmarkMetadata.create(suite.name)
        dest = setup_dest(metadata, dest, no_subdir)
    else:
        assert no_subdir, "Must use 'no-subdir=True' with 'shard={shard}'."
        with open(dest / "metadata.json", "rt") as f:
            metadata = BenchmarkMetadata.from_json(json.load(f))
        assert metadata.suite_name == suite.name
        metadata = metadata.for_shard()

    if shard.print_dest:
        print(dest.resolve())

    if shard.write_metadata:
        metadata_path = dest / f"metadata{shard.file_suffix}.json"
        with open(metadata_path, "wt") as f:
            json.dump(metadata.to_json(), f, indent=4)

    metrics_df: Optional[pd.DataFrame] = None

    if shard.run_benchmarks:
        metrics_df = _run_benchmarks(
            metadata,
            shard,
            suite,
            dest,
            cache_dir,
        )

    if shard.collect:
        assert metrics_df is None
        metrics_path = dest / "metrics.csv"
        metrics_df_list = [pd.read_csv(p) for p in shard.find_shards(metrics_path)]
        metrics_df = pd.concat(metrics_df_list, axis="index", ignore_index=True)
        metrics_df.to_csv(metrics_path, index=False)

    if shard.plot and plot and (metrics_df is not None):
        plot_(
            metadata=metadata,
            suite=suite,
            results_df=metrics_df,
            results_metadata=[metadata],
            dest=dest,
        )


def run_from_argv() -> None: