anna
/
scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

read_plus.zsh: Various Updates 

* Clean up colors/read_plus.py to match flake8 style guidelines
* Add --dmesg, --srvdmesg, and --no-server-vmtouch options
* Use $COLORS from common.zsh
* Accept a specific filename to test as a command line argument
* Add tests for 3, 5, 6, and 7 page segments

Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
This commit is contained in:
Anna Schumaker 2023-07-19 14:44:10 -04:00
parent 3e0f52b4e4
commit 8cc4c23e71
5 changed files with 75 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
colors/read_plus.py
View File

@ -1,3 +1,4 @@
"""Colorize read_plus.zsh output."""
import sys
import re
import termcolor
@ -6,22 +7,38 @@ SIZE = sys.argv[1]
FILE = open(sys.argv[2], 'a') if len(sys.argv) >= 3 else None
COLORS = {"data": "red", "hole": "green", "d": "cyan", "h": "yellow"}
def write_log(text, color=None, attrs=[], end=""):
if FILE: FILE.write(text + end)
def write_log(text: str, color: str = None,
attrs: list = [], end: str = "") -> None:
"""Print a single formatted string to the screen and possibly a file."""
if FILE:
FILE.write(text + end)
termcolor.cprint(text, color=color, attrs=attrs, end=end)
def print_allocation(prefix, name, filetype, extra):
def print_allocation(prefix: str, name: str,
filetype: str, extra: str) -> None:
"""Format and print file allocation information."""
write_log(prefix, color="white", attrs=["dark"])
write_log(name, color=COLORS[filetype], attrs=["bold"])
write_log(extra, color="white", attrs=["dark"])
def print_allocation_done(done, time, end):
def print_allocation_done(done: str, time: str, end: str) -> None:
"""Format and print the time to complete file allocation."""
write_log(done, color="white", attrs=["dark"])
write_log(time, color="blue", attrs=["bold"])
write_log(end, color="white", attrs=["dark"])
def print_reading(prefix, name, precache, cached, postcache):
ccolor = "yellow" if cached == "uncached" else "green"
def print_reading(prefix: str, name: str,
precache: str, cached: str, postcache: str) -> None:
"""Format and print which file we are currently reading."""
match cached:
case "uncached": ccolor = "yellow"
case "cached": ccolor = "green"
case "unknown": ccolor = "grey"
case _: ccolor = "white"
write_log(prefix, color="white", attrs=["dark"])
write_log(name, color=COLORS[filetype], attrs=["bold"])
@ -30,20 +47,26 @@ def print_reading(prefix, name, precache, cached, postcache):
write_log(postcache, color="white", attrs=["dark"])
write_log(" ...", color="white", attrs=["dark"])
def print_stats_1(walltime, speed):
def print_stats_1(walltime: str, speed: str) -> None:
"""Format and print the transfer time and rate information."""
write_log(" ")
write_log(walltime, color="blue", attrs=["bold"])
write_log(", ", color="white", attrs=["dark"])
write_log(speed, color="green", attrs=["bold"])
write_log(", ", color="white", attrs=["dark"])
def print_stats_2(kerntime, kernunits, cpu):
def print_stats_2(kerntime: str, kernunits: str, cpu: str) -> None:
"""Format and print time spent in the kernel and cpu usage."""
write_log(kerntime, color="cyan", attrs=["bold"], end=" ")
write_log(kernunits, color="cyan", attrs=["bold"])
write_log(", ", color="white", attrs=["dark"])
write_log(cpu, color="red", attrs=["bold"])
def print_iteration(prefix, cur, sep, total):
def print_iteration(prefix: str, cur: str, sep: str, total: str) -> None:
"""Format and print the current iteration header."""
write_log(prefix, color="magenta", attrs=["bold", "underline"])
write_log(cur, color="green", attrs=["bold", "underline"])
write_log(sep, color="magenta", attrs=["bold", "underline"])
@ -55,22 +78,29 @@ while c := sys.stdin.read(1):
text += c
line = text
text = ""
if match := re.match(f"(Allocating file: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line):
if match := re.match(r"(Allocating file: )"
rf"({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line):
filetype = match.group(3) if match.group(4) is None else match.group(4)
print_allocation(match.group(1), match.group(2), filetype, match.group(5))
print_allocation(match.group(1), match.group(2),
filetype, match.group(5))
elif match := re.match(r"( \[Done: )(\d+\.\d+s)(\]\n)", line):
print_allocation_done(match.group(1), match.group(2), match.group(3))
elif match := re.match(f"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( \()((un)?cached)( on \w*\))\n", line):
elif match := re.match(rf"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))"
r"( \()((un)?(cached|known))( on [\w-]*\))\n",
line):
filetype = match.group(3) if match.group(4) is None else match.group(4)
print_reading(match.group(1), match.group(2), match.group(5), match.group(6), match.group(8))
elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), ((\d+ MB/s)|(\d\.\d GB/s))\n", line):
print_reading(match.group(1), match.group(2), match.group(5),
match.group(6), match.group(9))
elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), "
r"(\d+\.?\d+ [MG]B/s)\n", line):
print_stats_1(match.group(1), match.group(2))
elif match := re.match("(\d+\.\d+)(s kern), (\d+% cpu)", line):
elif match := re.match(r"(\d+\.\d+)(s kern), (\d+% cpu)", line):
print_stats_2(match.group(1), match.group(2), match.group(3))
elif match := re.match(r"\d+\+\d+ records (in|out)\n", line):
pass
elif match := re.match("(Iteration )(\d+)( / )(\d+)\n", line):
print_iteration(match.group(1), match.group(2), match.group(3), match.group(4))
elif match := re.match(r"(Iteration )(\d+)( / )(\d+)\n", line):
print_iteration(match.group(1), match.group(2),
match.group(3), match.group(4))
elif c == "\n":
write_log(line)
else:

3
completions/_read_plus.zsh
View File

@ -6,9 +6,12 @@ function _read_plus.zsh() {
"hosts\:hosts\: _ssh_hosts"
"domains\:domains\:($(virsh list --all --name))"' \
{-d,--direct}'[call dd with iflag=direct]' \
--dmesg'[print client dmesg log after running tests]' \
{-f,--file}'[write output to file]: : _files' \
{-i,--iterations}'[number of times to repeat the test]: :($(seq 100))' \
{-p,--mountpoint}'[the directory to mount the server]: : _files -/' \
--no-server-vmtouch'[do not vmtouch the files on the server]' \
--srvdmesg'[print server dmesg log after running tests]' \
{-s,--server}'[the server to test against]: : _alternative
"hosts\:hosts\: _ssh_hosts"
"domains\:domains\:($(virsh list --all --name))"' \

29
read_plus.zsh
View File

@ -3,9 +3,11 @@ source common.zsh
SIZE=(2048)
ITERATIONS=(1)
zparseopts -F -K \
zparseopts -D -F -K \
c:=CLIENT -client:=CLIENT \
d=DIRECT -direct=DIRECT \
-dmesg=DMESG -srvdmesg=SRVDMESG \
-no-server-vmtouch=NO_VMTOUCH \
f:=FILE -file:=FILE \
i:=ITERATIONS -iterations:=ITERATIONS \
p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \
@ -13,14 +15,19 @@ zparseopts -F -K \
x:=EXPORT -export:=EXPORT \
z:=SIZE -size:=SIZE
COLOR="python -u $BIN/colors/read_plus.py ${SIZE[-1]} ${FILE[-1]}"
COLOR="python -u $COLORS/read_plus.py ${SIZE[-1]} ${FILE[-1]}"
IFLAG=
SRV_DIR=${EXPORT[-1]}/$USER/read_plus
TEST_DIR=${MOUNTPOINT[-1]}/read_plus
TEST_DEV=${SERVER[-1]}:$SRV_DIR
TEST_FILES=($(echo ${SIZE[-1]}M-{data,hole,mixed-{1,2,4,8,16}{d,h}}))
TEST_FILES=(${SIZE[-1]}M-{data,hole,mixed-{1,2,3,4,5,6,7,8,16}{d,h}})
CACHE_STATUS=(uncached cached)
TIME_CMD="sudo /usr/bin/time -f \"%Ss kern, %P cpu\""
if [ "$#" -gt 0 ]; then
TEST_FILES=($*)
fi
#
# Prepare to test
#
@ -28,10 +35,9 @@ function setup_client_func()
{
client_setup read_plus.zsh ${SERVER[-1]} ${EXPORT[-1]} \
${MOUNTPOINT[-1]} $USER ${SIZE[-1]} \
1,2,4,8,16 | eval ${COLOR}
1,2,3,4,5,6,7,8,16 | eval ${COLOR}
}
prepare_to_test
if [ ${#FILE} -gt 1 ]; then
mkdir -p $(dirname ${FILE[-1]})
[[ -f ${FILE[-1]} ]] && rm -fv ${FILE[-1]}
@ -40,7 +46,11 @@ fi
if [ ${#DIRECT} -gt 0 ]; then
IFLAG="iflag=direct"
fi
if [ ${#NO_VMTOUCH} -gt 0 ]; then
CACHE_STATUS=unknown
fi
prepare_to_test
function dd_file() {
echo "Reading: $1 ($2 on ${SERVER[-1]})"
@ -48,15 +58,18 @@ function dd_file() {
case $2 in
"uncached") args="eq" ;;
"cached") args="tq" ;;
"unknown") args= ;;
esac
ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1"
if [ ! -z $args ]; then
ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1"
fi
ssh ${CLIENT[-1]} "sudo vmtouch -eq $TEST_DIR/$1"
ssh ${CLIENT[-1]} "$TIME_CMD dd if=$TEST_DIR/$1 $IFLAG of=/dev/null bs=$3"
}
for i in $(seq ${ITERATIONS[-1]}); do
echo
[[ $i -gt 1 ]] && echo
[[ ${#FILE} -gt 1 ]] && echo >> ${FILE[-1]}
ssh ${CLIENT[-1]} "sudo mount -o sec=sys,v4.2 $TEST_DEV $TEST_DIR"
@ -65,7 +78,7 @@ for i in $(seq ${ITERATIONS[-1]}); do
echo "Iteration $i / ${ITERATIONS[-1]}" | eval ${COLOR}
fi
for f in $TEST_FILES; do
for cache in uncached cached; do
for cache in $CACHE_STATUS; do
dd_file $f $cache $bsize 2>&1 | eval ${COLOR}
done
done

4
setup/read_plus-files.py
View File

@ -1,4 +1,5 @@
#!/usr/bin/python -u
"""Create sparse files for testing with various patterns."""
import datetime
import pathlib
import os
@ -21,7 +22,8 @@ FILE_SIZE_BYTES = FILE_SIZE * 1024 * 1024
FILE_N_PAGES = FILE_SIZE_BYTES // PAGE_SIZE
def allocate_testfile(name, n_pages, hole):
def allocate_testfile(name: str, n_pages: int, hole: bool) -> None:
"""Allocate a single test file."""
file = FILE_BASE / f"{FILE_SIZE}M-{name}"
if file.exists():
return

2
setup/read_plus.zsh
View File

@ -18,5 +18,5 @@ TRAPEXIT() {
}
mkdir -p -m 777 $MOUNTPOINT/$USER/read_plus
bin/setup/read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS
setup.zsh read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS
sync