Commit b9e4e040 authored by Raphaël Gomès's avatar Raphaël Gomès
Browse files

Initial commit

syntax: rootglob
# repo-bencher
## Setup
This project requires Python version ``3.6`` or higher.
You need to have a ``repos`` folder at the root of this repository containing (symlinks to) the repositories you want to test.
> **Note:** No particular care is taken with those repos, but since we're only testing read-only commands, it should be fine for now.
## Usage
Run the benchmarks:
``./ -x first_executable -x /path/to/other_executable -- --subprocess-args -o here``
Results are JSON files in ``results/{repo name}/{executable name}_{url-encoded command}.json``.
For debugging purposes, ``stderr`` and ``stdout`` outputs of the subprocesses are kept in ``logs/{run_uid}/[stdout,stderr]``, with ``{run_uid}`` corresponding to the key of the same name in any JSON results file.
#! /usr/bin/env python3
import sys
import argparse
import json
import os
import subprocess
from pathlib import Path
from urllib.parse import quote
from uuid import uuid4, UUID
BASE_DIR = Path(__file__).parent.absolute()
RESULTS_DIR = BASE_DIR / "results"
REPOS_DIR = BASE_DIR / "repos"
"st -mard",
def get_repos():
if not REPOS_DIR.exists():
print("abort: no 'repos' directory found", file=sys.stderr)
yield from (REPOS_DIR / repo for repo in os.listdir(REPOS_DIR))
def post_process_results(
repo_path: Path,
run_uid: UUID,
executable: str,
cmd: str,
result_path: Path,
tmp_path = Path(f"{result_path}.tmp")
with"r") as f:
hyperfine_data = json.load(f)
data = {
"version": 0,
"run_uid": run_uid.hex,
"repository": str(repo_path.absolute()),
"executable": executable,
"command": cmd,
"hyperfine": hyperfine_data,
with"w") as f:
json.dump(data, f, sort_keys=True, indent=2)
def run_repo_command(
repo_path: Path, executable: str, cmd: str, subprocess_args
exec_name = executable.rsplit(os.pathsep)[0]
filename = f"{exec_name}_{quote(cmd)}"
uid = uuid4()
log_dir = LOGGING_DIR / uid.hex
log_dir.mkdir(parents=True, exist_ok=True)
out_path = RESULTS_DIR / / f"{filename}.json"
out_path = out_path.absolute()
out_path.parent.mkdir(parents=True, exist_ok=True)
full_cmd = (
f"{executable} {cmd} {subprocess_args}"
f" > {log_dir / 'stdout'} 2> {log_dir / 'stderr'}"
full_cmd = (
f"hyperfine --warmup 3 --show-output "
f'--export-json {out_path}.tmp "{full_cmd}"'
), check=True, shell=True, cwd=str(repo_path))
post_process_results(repo_path, uid, executable, cmd, out_path)
def bench(executable: str, subprocess_args):
repos_tested = 0
for repo_path in get_repos():
if not repo_path.is_dir():
print(f"[MAIN] Switching to repo '{repo_path}'")
repos_tested += 1
for command in REPO_COMMANDS:
run_repo_command(repo_path, executable, command, subprocess_args)
print(f"[MAIN] Tested {repos_tested} repositories")
def main(args: argparse.Namespace, subprocess_args: str):
for executable in args.executables:
print(f"[MAIN] Using {executable} executable")
bench(executable, subprocess_args)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
help="Executable(s) to run the tests with",
(known, others) = parser.parse_known_args()
if others and others[0] != "--":
"abort: use '--' pass arguments to the subprocess", file=sys.stderr
others = " ".join(others[1:]) if others else ""
main(known, others)
line_length = 80
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment