Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Raphaël Gomès
repo-bencher
Commits
41e7cfc99526
Commit
f837b786
authored
Jun 19, 2020
by
Raphaël Gomès
Browse files
Hardcode jane and upstream executables, provide config
parent
90b4c87d124f
Changes
3
Hide whitespace changes
Inline
Side-by-side
.hgignore
View file @
41e7cfc9
...
...
@@ -5,3 +5,4 @@ __pycache__/
repos/
logs/
results/
config.json
README.md
View file @
41e7cfc9
...
...
@@ -8,11 +8,36 @@ You need to have a ``repos`` folder at the root of this repository containing (s
> **Note:** No particular care is taken with those repos, but since we're only testing read-only commands, it should be fine for now.
You also need the
``config.json``
file at the root of this repository. Here is an example:
```
json
{
"version"
:
0
,
"executables"
:
{
"upstream"
:
{
"path"
:
"/path/to/upstream/hg"
,
"env"
:
{
"HGMODULEPOLICY"
:
"rust+c"
,
"RUST_LOG"
:
"trace"
}
},
"jane"
:
{
"path"
:
"/path/to/hg-jane-fastpath"
,
"env"
:
{
"HGRS_HG"
:
"/path/to/upstream/hg"
},
"args"
:
"--no-mmap --no-unclean-cache --profile"
}
}
}
```
## Usage
Run the benchmarks:
``./bench.py -x
first_executable -x /path/to/other_executable -- --subprocess-args -o her
e``
``./bench.py -x
upstream -x jan
e``
Results are JSON files in
``results/{repo name}/{executable name}_{url-encoded command}.json``
.
...
...
bench.py
View file @
41e7cfc9
#! /usr/bin/env python3
#!/usr/bin/env python3
import
re
import
sys
import
argparse
import
json
import
os
import
subprocess
from
collections
import
defaultdict
from
pathlib
import
Path
from
urllib.parse
import
quote
from
uuid
import
uuid4
,
UUID
...
...
@@ -12,11 +14,11 @@ BASE_DIR = Path(__file__).parent.absolute()
RESULTS_DIR
=
BASE_DIR
/
"results"
LOGGING_DIR
=
BASE_DIR
/
"logs"
REPOS_DIR
=
BASE_DIR
/
"repos"
REPO_COMMANDS
=
[
"st"
,
"st -mard"
,
"diff"
,
]
REPO_COMMANDS
=
[
"st"
,
"st -mard"
,
"diff"
]
UPSTREAM_INTERMEDIATE_RESULTS_RE
=
re
.
compile
(
r
"Duration of `([^`]+)`: (.+)"
)
JANE_INTERMEDIATE_RESULTS_RE
=
re
.
compile
(
r
"duration of (.+): (.+)"
)
def
get_repos
():
...
...
@@ -27,10 +29,31 @@ def get_repos():
yield
from
(
REPOS_DIR
/
repo
for
repo
in
os
.
listdir
(
REPOS_DIR
))
def
get_intermediate_timings
(
exec_kind
:
str
,
uid
:
str
):
fd
=
"stdout"
if
exec_kind
==
"upstream"
else
"stderr"
log_file
=
LOGGING_DIR
/
uid
/
fd
with
log_file
.
open
(
mode
=
"r"
)
as
f
:
log_data
=
f
.
read
()
timings
=
defaultdict
(
list
)
regex
=
(
UPSTREAM_INTERMEDIATE_RESULTS_RE
if
exec_kind
==
"upstream"
else
JANE_INTERMEDIATE_RESULTS_RE
)
for
match
in
re
.
findall
(
regex
,
log_data
):
timings
[
match
[
0
]].
append
(
match
[
1
])
return
timings
def
post_process_results
(
repo_path
:
Path
,
run_uid
:
UUID
,
executable
:
str
,
exec_kind
:
str
,
exec_config
,
cmd
:
str
,
result_path
:
Path
,
):
...
...
@@ -38,25 +61,29 @@ def post_process_results(
with
tmp_path
.
open
(
mode
=
"r"
)
as
f
:
hyperfine_data
=
json
.
load
(
f
)
timings
=
get_intermediate_timings
(
exec_kind
,
run_uid
.
hex
)
data
=
{
"version"
:
0
,
"run_uid"
:
run_uid
.
hex
,
"repository"
:
str
(
repo_path
.
absolute
()),
"
executable"
:
executable
,
"
config"
:
exec_config
,
"command"
:
cmd
,
"hyperfine"
:
hyperfine_data
,
"intermediate_timings"
:
timings
,
}
print
(
f
"Writing results to
{
result_path
}
"
)
with
result_path
.
open
(
mode
=
"w"
)
as
f
:
json
.
dump
(
data
,
f
,
sort_keys
=
True
,
indent
=
2
)
tmp_path
.
unlink
(
missing_ok
=
True
)
def
run_repo_command
(
repo_path
:
Path
,
executable
:
str
,
cmd
:
str
,
subprocess_args
)
:
exec_name
=
executable
.
rsplit
(
os
.
path
sep
)[
0
]
def
run_repo_command
(
repo_path
:
Path
,
exec_kind
:
str
,
exec_config
,
cmd
:
str
):
executable
=
exec_config
[
"path"
]
subprocess_args
=
exec_config
.
get
(
"args"
,
""
)
exec_name
=
executable
.
rsplit
(
os
.
sep
)[
-
1
]
filename
=
f
"
{
exec_name
}
_
{
quote
(
cmd
)
}
"
uid
=
uuid4
()
...
...
@@ -65,21 +92,16 @@ def run_repo_command(
log_dir
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
out_path
=
RESULTS_DIR
/
repo_path
.
name
/
f
"
{
filename
}
.json"
out_path
=
out_path
.
absolute
()
out_path
.
parent
.
mkdir
(
parents
=
True
,
exist_ok
=
True
)
out_path
.
touch
(
exist_ok
=
True
)
# Pass the env to the subcommand of hyperfine, otherwise it won't work
env
=
{
"HGMODULEPOLICY"
:
"rust+c"
,
"RUST_LOG"
:
"trace"
,
}
env
=
exec_config
.
get
(
"env"
,
{})
env_string
=
" "
.
join
(
f
"
{
k
}
=
{
v
}
"
for
k
,
v
in
env
.
items
())
full_cmd
=
(
f
"
{
env_string
}
{
executable
}
{
cmd
}
{
subprocess_args
}
"
f
">
{
log_dir
/
'stdout'
}
2>
{
log_dir
/
'stderr'
}
"
f
">
>
{
log_dir
/
'stdout'
}
2>
>
{
log_dir
/
'stderr'
}
"
)
full_cmd
=
(
f
"hyperfine --warmup 3 --show-output "
...
...
@@ -88,10 +110,10 @@ def run_repo_command(
subprocess
.
run
(
full_cmd
,
check
=
True
,
shell
=
True
,
cwd
=
str
(
repo_path
))
post_process_results
(
repo_path
,
uid
,
exec
utable
,
cmd
,
out_path
)
post_process_results
(
repo_path
,
uid
,
exec
_kind
,
exec_config
,
cmd
,
out_path
)
def
bench
(
exec
utable
:
str
,
subprocess_args
):
def
bench
(
exec
_kind
:
str
,
exec_config
:
dict
):
repos_tested
=
0
for
repo_path
in
get_repos
():
...
...
@@ -101,15 +123,26 @@ def bench(executable: str, subprocess_args):
repos_tested
+=
1
for
command
in
REPO_COMMANDS
:
run_repo_command
(
repo_path
,
exec
utable
,
command
,
subprocess_args
)
run_repo_command
(
repo_path
,
exec
_kind
,
exec_config
,
command
)
print
(
f
"[MAIN] Tested
{
repos_tested
}
repositories"
)
def
main
(
args
:
argparse
.
Namespace
,
subprocess_args
:
str
):
for
executable
in
args
.
executables
:
print
(
f
"[MAIN] Using
{
executable
}
executable"
)
bench
(
executable
,
subprocess_args
)
def
main
(
args
:
argparse
.
Namespace
):
config_file
=
BASE_DIR
/
"config.json"
if
not
config_file
.
exists
():
print
(
"abort: no 'config.json' found, see README for info"
)
exit
(
1
)
for
exec_kind
in
args
.
executables
:
print
(
f
"[MAIN] Using
{
exec_kind
}
executable"
)
with
config_file
.
open
(
mode
=
"r"
)
as
f
:
config
=
json
.
load
(
f
)
try
:
bench
(
exec_kind
,
config
[
"executables"
][
exec_kind
])
except
subprocess
.
CalledProcessError
:
# errors are logged, keep benching
pass
if
__name__
==
"__main__"
:
...
...
@@ -120,17 +153,8 @@ if __name__ == "__main__":
dest
=
"executables"
,
action
=
"append"
,
required
=
True
,
help
=
"Executable(s) to run the tests with"
,
choices
=
[
"upstream"
,
"jane"
],
help
=
"Type(s) of executable to run the tests with"
,
)
(
known
,
others
)
=
parser
.
parse_known_args
()
if
others
and
others
[
0
]
!=
"--"
:
print
(
"abort: use '--' to pass arguments to the subprocess"
,
file
=
sys
.
stderr
,
)
exit
(
255
)
others
=
" "
.
join
(
others
[
1
:])
if
others
else
""
main
(
known
,
others
)
main
(
parser
.
parse_args
()
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment