Commit f837b786 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

Hardcode jane and upstream executables, provide config

parent 90b4c87d124f
......@@ -5,3 +5,4 @@ __pycache__/
repos/
logs/
results/
config.json
......@@ -8,11 +8,36 @@ You need to have a ``repos`` folder at the root of this repository containing (s
> **Note:** No particular care is taken with those repos, but since we're only testing read-only commands, it should be fine for now.
You also need the ``config.json`` file at the root of this repository. Here is an example:
```json
{
"version": 0,
"executables": {
"upstream": {
"path": "/path/to/upstream/hg",
"env": {
"HGMODULEPOLICY": "rust+c",
"RUST_LOG": "trace"
}
},
"jane": {
"path": "/path/to/hg-jane-fastpath",
"env": {
"HGRS_HG": "/path/to/upstream/hg"
},
"args": "--no-mmap --no-unclean-cache --profile"
}
}
}
```
## Usage
Run the benchmarks:
``./bench.py -x first_executable -x /path/to/other_executable -- --subprocess-args -o here``
``./bench.py -x upstream -x jane``
Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``.
......
#! /usr/bin/env python3
#!/usr/bin/env python3
import re
import sys
import argparse
import json
import os
import subprocess
from collections import defaultdict
from pathlib import Path
from urllib.parse import quote
from uuid import uuid4, UUID
......@@ -12,11 +14,11 @@ BASE_DIR = Path(__file__).parent.absolute()
RESULTS_DIR = BASE_DIR / "results"
LOGGING_DIR = BASE_DIR / "logs"
REPOS_DIR = BASE_DIR / "repos"
REPO_COMMANDS = [
"st",
"st -mard",
"diff",
]
REPO_COMMANDS = ["st", "st -mard", "diff"]
UPSTREAM_INTERMEDIATE_RESULTS_RE = re.compile(r"Duration of `([^`]+)`: (.+)")
JANE_INTERMEDIATE_RESULTS_RE = re.compile(r"duration of (.+): (.+)")
def get_repos():
......@@ -27,10 +29,31 @@ def get_repos():
yield from (REPOS_DIR / repo for repo in os.listdir(REPOS_DIR))
def get_intermediate_timings(exec_kind: str, uid: str):
fd = "stdout" if exec_kind == "upstream" else "stderr"
log_file = LOGGING_DIR / uid / fd
with log_file.open(mode="r") as f:
log_data = f.read()
timings = defaultdict(list)
regex = (
UPSTREAM_INTERMEDIATE_RESULTS_RE
if exec_kind == "upstream"
else JANE_INTERMEDIATE_RESULTS_RE
)
for match in re.findall(regex, log_data):
timings[match[0]].append(match[1])
return timings
def post_process_results(
repo_path: Path,
run_uid: UUID,
executable: str,
exec_kind: str,
exec_config,
cmd: str,
result_path: Path,
):
......@@ -38,25 +61,29 @@ def post_process_results(
with tmp_path.open(mode="r") as f:
hyperfine_data = json.load(f)
timings = get_intermediate_timings(exec_kind, run_uid.hex)
data = {
"version": 0,
"run_uid": run_uid.hex,
"repository": str(repo_path.absolute()),
"executable": executable,
"config": exec_config,
"command": cmd,
"hyperfine": hyperfine_data,
"intermediate_timings": timings,
}
print(f"Writing results to {result_path}")
with result_path.open(mode="w") as f:
json.dump(data, f, sort_keys=True, indent=2)
tmp_path.unlink(missing_ok=True)
def run_repo_command(
repo_path: Path, executable: str, cmd: str, subprocess_args
):
exec_name = executable.rsplit(os.pathsep)[0]
def run_repo_command(repo_path: Path, exec_kind: str, exec_config, cmd: str):
executable = exec_config["path"]
subprocess_args = exec_config.get("args", "")
exec_name = executable.rsplit(os.sep)[-1]
filename = f"{exec_name}_{quote(cmd)}"
uid = uuid4()
......@@ -65,21 +92,16 @@ def run_repo_command(
log_dir.mkdir(parents=True, exist_ok=True)
out_path = RESULTS_DIR / repo_path.name / f"{filename}.json"
out_path = out_path.absolute()
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.touch(exist_ok=True)
# Pass the env to the subcommand of hyperfine, otherwise it won't work
env = {
"HGMODULEPOLICY": "rust+c",
"RUST_LOG": "trace",
}
env = exec_config.get("env", {})
env_string = " ".join(f"{k}={v}" for k, v in env.items())
full_cmd = (
f"{env_string} {executable} {cmd} {subprocess_args} "
f"> {log_dir / 'stdout'} 2> {log_dir / 'stderr'}"
f">> {log_dir / 'stdout'} 2>> {log_dir / 'stderr'}"
)
full_cmd = (
f"hyperfine --warmup 3 --show-output "
......@@ -88,10 +110,10 @@ def run_repo_command(
subprocess.run(full_cmd, check=True, shell=True, cwd=str(repo_path))
post_process_results(repo_path, uid, executable, cmd, out_path)
post_process_results(repo_path, uid, exec_kind, exec_config, cmd, out_path)
def bench(executable: str, subprocess_args):
def bench(exec_kind: str, exec_config: dict):
repos_tested = 0
for repo_path in get_repos():
......@@ -101,15 +123,26 @@ def bench(executable: str, subprocess_args):
repos_tested += 1
for command in REPO_COMMANDS:
run_repo_command(repo_path, executable, command, subprocess_args)
run_repo_command(repo_path, exec_kind, exec_config, command)
print(f"[MAIN] Tested {repos_tested} repositories")
def main(args: argparse.Namespace, subprocess_args: str):
for executable in args.executables:
print(f"[MAIN] Using {executable} executable")
bench(executable, subprocess_args)
def main(args: argparse.Namespace):
config_file = BASE_DIR / "config.json"
if not config_file.exists():
print("abort: no 'config.json' found, see README for info")
exit(1)
for exec_kind in args.executables:
print(f"[MAIN] Using {exec_kind} executable")
with config_file.open(mode="r") as f:
config = json.load(f)
try:
bench(exec_kind, config["executables"][exec_kind])
except subprocess.CalledProcessError:
# errors are logged, keep benching
pass
if __name__ == "__main__":
......@@ -120,17 +153,8 @@ if __name__ == "__main__":
dest="executables",
action="append",
required=True,
help="Executable(s) to run the tests with",
choices=["upstream", "jane"],
help="Type(s) of executable to run the tests with",
)
(known, others) = parser.parse_known_args()
if others and others[0] != "--":
print(
"abort: use '--' to pass arguments to the subprocess",
file=sys.stderr,
)
exit(255)
others = " ".join(others[1:]) if others else ""
main(known, others)
main(parser.parse_args())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment