diff --git a/colors/read_plus.py b/colors/read_plus.py index ce5fff5..dedb552 100644 --- a/colors/read_plus.py +++ b/colors/read_plus.py @@ -1,3 +1,4 @@ +"""Colorize read_plus.zsh output.""" import sys import re import termcolor @@ -6,22 +7,38 @@ SIZE = sys.argv[1] FILE = open(sys.argv[2], 'a') if len(sys.argv) >= 3 else None COLORS = {"data": "red", "hole": "green", "d": "cyan", "h": "yellow"} -def write_log(text, color=None, attrs=[], end=""): - if FILE: FILE.write(text + end) + +def write_log(text: str, color: str = None, + attrs: list = [], end: str = "") -> None: + """Print a single formatted string to the screen and possibly a file.""" + if FILE: + FILE.write(text + end) termcolor.cprint(text, color=color, attrs=attrs, end=end) -def print_allocation(prefix, name, filetype, extra): + +def print_allocation(prefix: str, name: str, + filetype: str, extra: str) -> None: + """Format and print file allocation information.""" write_log(prefix, color="white", attrs=["dark"]) write_log(name, color=COLORS[filetype], attrs=["bold"]) write_log(extra, color="white", attrs=["dark"]) -def print_allocation_done(done, time, end): + +def print_allocation_done(done: str, time: str, end: str) -> None: + """Format and print the time to complete file allocation.""" write_log(done, color="white", attrs=["dark"]) write_log(time, color="blue", attrs=["bold"]) write_log(end, color="white", attrs=["dark"]) -def print_reading(prefix, name, precache, cached, postcache): - ccolor = "yellow" if cached == "uncached" else "green" + +def print_reading(prefix: str, name: str, + precache: str, cached: str, postcache: str) -> None: + """Format and print which file we are currently reading.""" + match cached: + case "uncached": ccolor = "yellow" + case "cached": ccolor = "green" + case "unknown": ccolor = "grey" + case _: ccolor = "white" write_log(prefix, color="white", attrs=["dark"]) write_log(name, color=COLORS[filetype], attrs=["bold"]) @@ -30,20 +47,26 @@ def print_reading(prefix, name, precache, cached, postcache): write_log(postcache, color="white", attrs=["dark"]) write_log(" ...", color="white", attrs=["dark"]) -def print_stats_1(walltime, speed): + +def print_stats_1(walltime: str, speed: str) -> None: + """Format and print the transfer time and rate information.""" write_log(" ") write_log(walltime, color="blue", attrs=["bold"]) write_log(", ", color="white", attrs=["dark"]) write_log(speed, color="green", attrs=["bold"]) write_log(", ", color="white", attrs=["dark"]) -def print_stats_2(kerntime, kernunits, cpu): + +def print_stats_2(kerntime: str, kernunits: str, cpu: str) -> None: + """Format and print time spent in the kernel and cpu usage.""" write_log(kerntime, color="cyan", attrs=["bold"], end=" ") write_log(kernunits, color="cyan", attrs=["bold"]) write_log(", ", color="white", attrs=["dark"]) write_log(cpu, color="red", attrs=["bold"]) -def print_iteration(prefix, cur, sep, total): + +def print_iteration(prefix: str, cur: str, sep: str, total: str) -> None: + """Format and print the current iteration header.""" write_log(prefix, color="magenta", attrs=["bold", "underline"]) write_log(cur, color="green", attrs=["bold", "underline"]) write_log(sep, color="magenta", attrs=["bold", "underline"]) @@ -55,22 +78,29 @@ while c := sys.stdin.read(1): text += c line = text text = "" - if match := re.match(f"(Allocating file: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line): + if match := re.match(r"(Allocating file: )" + rf"({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line): filetype = match.group(3) if match.group(4) is None else match.group(4) - print_allocation(match.group(1), match.group(2), filetype, match.group(5)) + print_allocation(match.group(1), match.group(2), + filetype, match.group(5)) elif match := re.match(r"( \[Done: )(\d+\.\d+s)(\]\n)", line): print_allocation_done(match.group(1), match.group(2), match.group(3)) - elif match := re.match(f"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( \()((un)?cached)( on \w*\))\n", line): + elif match := re.match(rf"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))" + r"( \()((un)?(cached|known))( on [\w-]*\))\n", + line): filetype = match.group(3) if match.group(4) is None else match.group(4) - print_reading(match.group(1), match.group(2), match.group(5), match.group(6), match.group(8)) - elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), ((\d+ MB/s)|(\d\.\d GB/s))\n", line): + print_reading(match.group(1), match.group(2), match.group(5), + match.group(6), match.group(9)) + elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), " + r"(\d+\.?\d+ [MG]B/s)\n", line): print_stats_1(match.group(1), match.group(2)) - elif match := re.match("(\d+\.\d+)(s kern), (\d+% cpu)", line): + elif match := re.match(r"(\d+\.\d+)(s kern), (\d+% cpu)", line): print_stats_2(match.group(1), match.group(2), match.group(3)) elif match := re.match(r"\d+\+\d+ records (in|out)\n", line): pass - elif match := re.match("(Iteration )(\d+)( / )(\d+)\n", line): - print_iteration(match.group(1), match.group(2), match.group(3), match.group(4)) + elif match := re.match(r"(Iteration )(\d+)( / )(\d+)\n", line): + print_iteration(match.group(1), match.group(2), + match.group(3), match.group(4)) elif c == "\n": write_log(line) else: diff --git a/completions/_read_plus.zsh b/completions/_read_plus.zsh index 52d0b40..8540f5d 100644 --- a/completions/_read_plus.zsh +++ b/completions/_read_plus.zsh @@ -6,9 +6,12 @@ function _read_plus.zsh() { "hosts\:hosts\: _ssh_hosts" "domains\:domains\:($(virsh list --all --name))"' \ {-d,--direct}'[call dd with iflag=direct]' \ + --dmesg'[print client dmesg log after running tests]' \ {-f,--file}'[write output to file]: : _files' \ {-i,--iterations}'[number of times to repeat the test]: :($(seq 100))' \ {-p,--mountpoint}'[the directory to mount the server]: : _files -/' \ + --no-server-vmtouch'[do not vmtouch the files on the server]' \ + --srvdmesg'[print server dmesg log after running tests]' \ {-s,--server}'[the server to test against]: : _alternative "hosts\:hosts\: _ssh_hosts" "domains\:domains\:($(virsh list --all --name))"' \ diff --git a/read_plus.zsh b/read_plus.zsh index cbca586..07570f5 100755 --- a/read_plus.zsh +++ b/read_plus.zsh @@ -3,9 +3,11 @@ source common.zsh SIZE=(2048) ITERATIONS=(1) -zparseopts -F -K \ +zparseopts -D -F -K \ c:=CLIENT -client:=CLIENT \ d=DIRECT -direct=DIRECT \ + -dmesg=DMESG -srvdmesg=SRVDMESG \ + -no-server-vmtouch=NO_VMTOUCH \ f:=FILE -file:=FILE \ i:=ITERATIONS -iterations:=ITERATIONS \ p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \ @@ -13,14 +15,19 @@ zparseopts -F -K \ x:=EXPORT -export:=EXPORT \ z:=SIZE -size:=SIZE -COLOR="python -u $BIN/colors/read_plus.py ${SIZE[-1]} ${FILE[-1]}" +COLOR="python -u $COLORS/read_plus.py ${SIZE[-1]} ${FILE[-1]}" IFLAG= SRV_DIR=${EXPORT[-1]}/$USER/read_plus TEST_DIR=${MOUNTPOINT[-1]}/read_plus TEST_DEV=${SERVER[-1]}:$SRV_DIR -TEST_FILES=($(echo ${SIZE[-1]}M-{data,hole,mixed-{1,2,4,8,16}{d,h}})) +TEST_FILES=(${SIZE[-1]}M-{data,hole,mixed-{1,2,3,4,5,6,7,8,16}{d,h}}) +CACHE_STATUS=(uncached cached) TIME_CMD="sudo /usr/bin/time -f \"%Ss kern, %P cpu\"" +if [ "$#" -gt 0 ]; then + TEST_FILES=($*) +fi + # # Prepare to test # @@ -28,10 +35,9 @@ function setup_client_func() { client_setup read_plus.zsh ${SERVER[-1]} ${EXPORT[-1]} \ ${MOUNTPOINT[-1]} $USER ${SIZE[-1]} \ - 1,2,4,8,16 | eval ${COLOR} + 1,2,3,4,5,6,7,8,16 | eval ${COLOR} } -prepare_to_test if [ ${#FILE} -gt 1 ]; then mkdir -p $(dirname ${FILE[-1]}) [[ -f ${FILE[-1]} ]] && rm -fv ${FILE[-1]} @@ -40,7 +46,11 @@ fi if [ ${#DIRECT} -gt 0 ]; then IFLAG="iflag=direct" fi +if [ ${#NO_VMTOUCH} -gt 0 ]; then + CACHE_STATUS=unknown +fi +prepare_to_test function dd_file() { echo "Reading: $1 ($2 on ${SERVER[-1]})" @@ -48,15 +58,18 @@ function dd_file() { case $2 in "uncached") args="eq" ;; "cached") args="tq" ;; + "unknown") args= ;; esac - ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1" + if [ ! -z $args ]; then + ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1" + fi ssh ${CLIENT[-1]} "sudo vmtouch -eq $TEST_DIR/$1" ssh ${CLIENT[-1]} "$TIME_CMD dd if=$TEST_DIR/$1 $IFLAG of=/dev/null bs=$3" } for i in $(seq ${ITERATIONS[-1]}); do - echo + [[ $i -gt 1 ]] && echo [[ ${#FILE} -gt 1 ]] && echo >> ${FILE[-1]} ssh ${CLIENT[-1]} "sudo mount -o sec=sys,v4.2 $TEST_DEV $TEST_DIR" @@ -65,7 +78,7 @@ for i in $(seq ${ITERATIONS[-1]}); do echo "Iteration $i / ${ITERATIONS[-1]}" | eval ${COLOR} fi for f in $TEST_FILES; do - for cache in uncached cached; do + for cache in $CACHE_STATUS; do dd_file $f $cache $bsize 2>&1 | eval ${COLOR} done done diff --git a/setup/read_plus-files.py b/setup/read_plus-files.py index cf9d73f..415ee73 100755 --- a/setup/read_plus-files.py +++ b/setup/read_plus-files.py @@ -1,4 +1,5 @@ #!/usr/bin/python -u +"""Create sparse files for testing with various patterns.""" import datetime import pathlib import os @@ -21,7 +22,8 @@ FILE_SIZE_BYTES = FILE_SIZE * 1024 * 1024 FILE_N_PAGES = FILE_SIZE_BYTES // PAGE_SIZE -def allocate_testfile(name, n_pages, hole): +def allocate_testfile(name: str, n_pages: int, hole: bool) -> None: + """Allocate a single test file.""" file = FILE_BASE / f"{FILE_SIZE}M-{name}" if file.exists(): return diff --git a/setup/read_plus.zsh b/setup/read_plus.zsh index d0c9f42..68d21f3 100755 --- a/setup/read_plus.zsh +++ b/setup/read_plus.zsh @@ -18,5 +18,5 @@ TRAPEXIT() { } mkdir -p -m 777 $MOUNTPOINT/$USER/read_plus -bin/setup/read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS +setup.zsh read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS sync