Commit 49116456 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

Implement summary flag

parent 41e7cfc99526
......@@ -39,6 +39,10 @@ Run the benchmarks:
``./bench.py -x upstream -x jane``
Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``.
Look at the summary:
``./bench.py -s``
Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``, they are overwritten after each run that results in the same name. We may want to add ``hg id`` to identify separate revisions.
For debugging purposes, ``stderr`` and ``stdout`` outputs of the subprocesses are kept in ``logs/{run_uid}/[stdout,stderr]``, with ``{run_uid}`` corresponding to the key of the same name in any JSON results file.
......@@ -14,7 +14,7 @@ BASE_DIR = Path(__file__).parent.absolute()
RESULTS_DIR = BASE_DIR / "results"
LOGGING_DIR = BASE_DIR / "logs"
REPOS_DIR = BASE_DIR / "repos"
REPO_COMMANDS = ["st", "st -mard", "diff"]
REPO_COMMANDS = ["st", "st -mard"]
UPSTREAM_INTERMEDIATE_RESULTS_RE = re.compile(r"Duration of `([^`]+)`: (.+)")
......@@ -128,10 +128,84 @@ def bench(exec_kind: str, exec_config: dict):
print(f"[MAIN] Tested {repos_tested} repositories")
def to_seconds(timing: str):
if timing.endswith("ns"):
return float(timing[:-2]) / 1_000_000_000.0
if timing.endswith("µs"):
return float(timing[:-2]) / 1_000_000.0
if timing.endswith("ms"):
return float(timing[:-2]) / 1_000.0
if timing.endswith("s"):
return float(timing[:-1])
else:
raise RuntimeError(f"Cannot parse timing '{timing}'")
def print_result(res):
intermediates = res["intermediate_timings"]
for intermediate, timings in intermediates.items():
timings = [to_seconds(t) for t in timings]
print(
f"min {min(timings):.6f}s "
f"max {max(timings):.6f}s "
f"mean {sum(timings)/len(timings):.6f}s "
f"{intermediate} "
)
datum = res["hyperfine"]["results"][0]
print(
f"min {datum['min']:.6f}s "
f"max {datum['max']:.6f}s "
f"mean {datum['mean']:.6f}s "
f"total"
)
def summary():
results = defaultdict(lambda: defaultdict(list))
for root, dirs, files in os.walk(RESULTS_DIR):
for file in files:
path = Path(root) / file
if path.suffix != ".json":
continue
with path.open(mode="r") as f:
try:
data = json.load(f)
except json.JSONDecodeError:
print(f"Ignoring invalid json file {path}", file=sys.stderr)
else:
results[data["repository"]][data["command"]].append(data)
for name, commands in sorted(results.items()):
print(f"=== Repository '{name.rsplit(os.sep)[-1]}' ===")
for command, data in sorted(commands.items()):
print(f"\n== Command: '{command}' ==")
for res in sorted(data, key=lambda d: d["config"]["path"]):
print(f"\n--- {res['config']['path'].rsplit(os.sep)[-1]} ---")
print_result(res)
print("")
def main(args: argparse.Namespace):
if not args.summary and not args.executables:
parser.print_help()
exit(255)
if args.summary:
return summary()
config_file = BASE_DIR / "config.json"
if not config_file.exists():
print("abort: no 'config.json' found, see README for info")
print(
"abort: no 'config.json' found, see README for info",
file=sys.stderr,
)
exit(1)
for exec_kind in args.executables:
......@@ -152,9 +226,11 @@ if __name__ == "__main__":
"--executable",
dest="executables",
action="append",
required=True,
choices=["upstream", "jane"],
help="Type(s) of executable to run the tests with",
)
parser.add_argument(
"-s", "--summary", dest="summary", default=False, action="store_true",
)
main(parser.parse_args())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment