Commit 2d77dd57 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

Store the executable version when benchmarking upstream

parent 027683f65112
......@@ -30,7 +30,6 @@ You also need the ``config.json`` file at the root of this repository. Here is a
}
}
}
```
## Usage
......@@ -43,6 +42,6 @@ Look at the summary:
``./bench.py -s``
Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``, they are overwritten after each run that results in the same name. We may want to add ``hg id`` to identify separate revisions.
Results are JSON files in ``results/{repo name}/[{executable_id}/]{executable name}_{url-encoded command}.json``, they are overwritten after each run that results in the same name.
For debugging purposes, ``stderr`` and ``stdout`` outputs of the subprocesses are kept in ``logs/{run_uid}/[stdout,stderr]``, with ``{run_uid}`` corresponding to the key of the same name in any JSON results file.
......@@ -63,12 +63,17 @@ def post_process_results(
timings = get_intermediate_timings(exec_kind, run_uid.hex)
# TODO this is a hack to pass it down, use an actual argument
exec_config = exec_config.copy()
executable_version = exec_config.pop("executable_version", None)
data = {
"version": 0,
"run_uid": run_uid.hex,
"repository": str(repo_path.absolute()),
"config": exec_config,
"command": cmd,
"executable_version": executable_version,
"hyperfine": hyperfine_data,
"intermediate_timings": timings,
}
......@@ -91,7 +96,12 @@ def run_repo_command(repo_path: Path, exec_kind: str, exec_config, cmd: str):
log_dir = LOGGING_DIR / uid.hex
log_dir.mkdir(parents=True, exist_ok=True)
out_path = RESULTS_DIR / repo_path.name / f"{filename}.json"
out_path = (
RESULTS_DIR
/ repo_path.name
/ exec_config.get("executable_version", "")
/ f"{filename}.json"
)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.touch(exist_ok=True)
......@@ -116,6 +126,24 @@ def run_repo_command(repo_path: Path, exec_kind: str, exec_config, cmd: str):
def bench(exec_kind: str, exec_config: dict):
repos_tested = 0
if exec_kind == "upstream":
try:
env = os.environ.copy()
env.update(exec_config.get("env", {}))
res = subprocess.run(
[exec_config["path"], "version", "-T{ver}"],
capture_output=True,
text=True,
env=env,
)
except subprocess.CalledProcessError:
print(
"warning: unable to get the current executable id",
file=sys.stderr,
)
else:
exec_config["executable_version"] = res.stdout.strip()
for repo_path in get_repos():
if not repo_path.is_dir():
continue
......@@ -186,7 +214,10 @@ def summary():
print(f"\n== Command: '{command}' ==")
for res in sorted(data, key=lambda d: d["config"]["path"]):
print(f"\n--- {res['config']['path'].rsplit(os.sep)[-1]} ---")
executable_name = res["config"]["path"].rsplit(os.sep)[-1]
print(
f"\n--- {executable_name} ({res.get('executable_version')}) ---"
)
print_result(res)
print("")
......@@ -208,10 +239,11 @@ def main(args: argparse.Namespace):
)
exit(1)
with config_file.open(mode="r") as f:
config = json.load(f)
for exec_kind in args.executables:
print(f"[MAIN] Using {exec_kind} executable")
with config_file.open(mode="r") as f:
config = json.load(f)
try:
bench(exec_kind, config["executables"][exec_kind])
except subprocess.CalledProcessError:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment