Hardcode jane and upstream executables, provide config

.hgignore

@@ -5,3 +5,4 @@ __pycache__/
 repos/
 logs/
 results/
+config.json

README.md

@@ -8,11 +8,36 @@ You need to have a ``repos`` folder at the root of this repository containing (s
 
 > **Note:** No particular care is taken with those repos, but since we're only testing read-only commands, it should be fine for now.
 
+You also need the ``config.json`` file at the root of this repository. Here is an example:
+
+```json
+{
+  "version": 0,
+  "executables": {
+    "upstream": {
+      "path": "/path/to/upstream/hg",
+      "env": {
+        "HGMODULEPOLICY": "rust+c",
+        "RUST_LOG": "trace"
+      }
+    },
+    "jane": {
+      "path": "/path/to/hg-jane-fastpath",
+      "env": {
+        "HGRS_HG": "/path/to/upstream/hg"
+      },
+      "args": "--no-mmap --no-unclean-cache --profile"
+    }
+  }
+}
+
+```
+
 ## Usage
 
 Run the benchmarks:
 
-``./bench.py -x first_executable -x /path/to/other_executable -- --subprocess-args -o here``
+``./bench.py -x upstream -x jane``
 
 Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``.
 

bench.py

-#! /usr/bin/env python3
+#!/usr/bin/env python3
+import re
 import sys
 import argparse
 import json
 import os
 import subprocess
+from collections import defaultdict
 from pathlib import Path
 from urllib.parse import quote
 from uuid import uuid4, UUID
@@ -12,11 +14,11 @@ BASE_DIR = Path(__file__).parent.absolute()
 RESULTS_DIR = BASE_DIR / "results"
 LOGGING_DIR = BASE_DIR / "logs"
 REPOS_DIR = BASE_DIR / "repos"
-REPO_COMMANDS = [
-    "st",
-    "st -mard",
-    "diff",
-]
+REPO_COMMANDS = ["st", "st -mard", "diff"]
+
+
+UPSTREAM_INTERMEDIATE_RESULTS_RE = re.compile(r"Duration of `([^`]+)`: (.+)")
+JANE_INTERMEDIATE_RESULTS_RE = re.compile(r"duration of (.+): (.+)")
 
 
 def get_repos():
@@ -27,10 +29,31 @@ def get_repos():
     yield from (REPOS_DIR / repo for repo in os.listdir(REPOS_DIR))
 
 
+def get_intermediate_timings(exec_kind: str, uid: str):
+    fd = "stdout" if exec_kind == "upstream" else "stderr"
+    log_file = LOGGING_DIR / uid / fd
+    with log_file.open(mode="r") as f:
+        log_data = f.read()
+
+    timings = defaultdict(list)
+
+    regex = (
+        UPSTREAM_INTERMEDIATE_RESULTS_RE
+        if exec_kind == "upstream"
+        else JANE_INTERMEDIATE_RESULTS_RE
+    )
+
+    for match in re.findall(regex, log_data):
+        timings[match[0]].append(match[1])
+
+    return timings
+
+
 def post_process_results(
     repo_path: Path,
     run_uid: UUID,
-    executable: str,
+    exec_kind: str,
+    exec_config,
     cmd: str,
     result_path: Path,
 ):
@@ -38,25 +61,29 @@ def post_process_results(
     with tmp_path.open(mode="r") as f:
         hyperfine_data = json.load(f)
 
+    timings = get_intermediate_timings(exec_kind, run_uid.hex)
+
     data = {
         "version": 0,
         "run_uid": run_uid.hex,
         "repository": str(repo_path.absolute()),
-        "executable": executable,
+        "config": exec_config,
         "command": cmd,
         "hyperfine": hyperfine_data,
+        "intermediate_timings": timings,
     }
 
+    print(f"Writing results to {result_path}")
     with result_path.open(mode="w") as f:
         json.dump(data, f, sort_keys=True, indent=2)
 
     tmp_path.unlink(missing_ok=True)
 
 
-def run_repo_command(
-    repo_path: Path, executable: str, cmd: str, subprocess_args
-):
-    exec_name = executable.rsplit(os.pathsep)[0]
+def run_repo_command(repo_path: Path, exec_kind: str, exec_config, cmd: str):
+    executable = exec_config["path"]
+    subprocess_args = exec_config.get("args", "")
+    exec_name = executable.rsplit(os.sep)[-1]
     filename = f"{exec_name}_{quote(cmd)}"
 
     uid = uuid4()
@@ -65,21 +92,16 @@ def run_repo_command(
     log_dir.mkdir(parents=True, exist_ok=True)
 
     out_path = RESULTS_DIR / repo_path.name / f"{filename}.json"
-    out_path = out_path.absolute()
-
     out_path.parent.mkdir(parents=True, exist_ok=True)
     out_path.touch(exist_ok=True)
 
     # Pass the env to the subcommand of hyperfine, otherwise it won't work
-    env = {
-        "HGMODULEPOLICY": "rust+c",
-        "RUST_LOG": "trace",
-    }
+    env = exec_config.get("env", {})
     env_string = " ".join(f"{k}={v}" for k, v in env.items())
 
     full_cmd = (
         f"{env_string} {executable} {cmd} {subprocess_args} "
-        f"> {log_dir / 'stdout'} 2> {log_dir / 'stderr'}"
+        f">> {log_dir / 'stdout'} 2>> {log_dir / 'stderr'}"
     )
     full_cmd = (
         f"hyperfine --warmup 3 --show-output "
@@ -88,10 +110,10 @@ def run_repo_command(
 
     subprocess.run(full_cmd, check=True, shell=True, cwd=str(repo_path))
 
-    post_process_results(repo_path, uid, executable, cmd, out_path)
+    post_process_results(repo_path, uid, exec_kind, exec_config, cmd, out_path)
 
 
-def bench(executable: str, subprocess_args):
+def bench(exec_kind: str, exec_config: dict):
     repos_tested = 0
 
     for repo_path in get_repos():
@@ -101,15 +123,26 @@ def bench(executable: str, subprocess_args):
         repos_tested += 1
 
         for command in REPO_COMMANDS:
-            run_repo_command(repo_path, executable, command, subprocess_args)
+            run_repo_command(repo_path, exec_kind, exec_config, command)
 
     print(f"[MAIN] Tested {repos_tested} repositories")
 
 
-def main(args: argparse.Namespace, subprocess_args: str):
-    for executable in args.executables:
-        print(f"[MAIN] Using {executable} executable")
-        bench(executable, subprocess_args)
+def main(args: argparse.Namespace):
+    config_file = BASE_DIR / "config.json"
+    if not config_file.exists():
+        print("abort: no 'config.json' found, see README for info")
+        exit(1)
+
+    for exec_kind in args.executables:
+        print(f"[MAIN] Using {exec_kind} executable")
+        with config_file.open(mode="r") as f:
+            config = json.load(f)
+        try:
+            bench(exec_kind, config["executables"][exec_kind])
+        except subprocess.CalledProcessError:
+            # errors are logged, keep benching
+            pass
 
 
 if __name__ == "__main__":
@@ -120,17 +153,8 @@ if __name__ == "__main__":
         dest="executables",
         action="append",
         required=True,
-        help="Executable(s) to run the tests with",
+        choices=["upstream", "jane"],
+        help="Type(s) of executable to run the tests with",
     )
-    (known, others) = parser.parse_known_args()
-
-    if others and others[0] != "--":
-        print(
-            "abort: use '--' to pass arguments to the subprocess",
-            file=sys.stderr,
-        )
-        exit(255)
-
-    others = " ".join(others[1:]) if others else ""
 
-    main(known, others)
+    main(parser.parse_args())