read_plus: Add script for testing the READ_PLUS operation
Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
This commit is contained in:
parent
a6d46ff0e1
commit
dd97c7079b
|
@ -0,0 +1,77 @@
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import termcolor
|
||||||
|
|
||||||
|
SIZE = sys.argv[1]
|
||||||
|
FILE = open(sys.argv[2], 'a') if len(sys.argv) >= 3 else None
|
||||||
|
COLORS = {"data": "red", "hole": "green", "d": "cyan", "h": "yellow"}
|
||||||
|
|
||||||
|
def write_log(text, color=None, attrs=[], end=""):
|
||||||
|
if FILE: FILE.write(text + end)
|
||||||
|
termcolor.cprint(text, color=color, attrs=attrs, end=end)
|
||||||
|
|
||||||
|
def print_allocation(prefix, name, filetype, extra):
|
||||||
|
write_log(prefix, color="white", attrs=["dark"])
|
||||||
|
write_log(name, color=COLORS[filetype], attrs=["bold"])
|
||||||
|
write_log(extra, color="white", attrs=["dark"])
|
||||||
|
|
||||||
|
def print_allocation_done(done, time, end):
|
||||||
|
write_log(done, color="white", attrs=["dark"])
|
||||||
|
write_log(time, color="blue", attrs=["bold"])
|
||||||
|
write_log(end, color="white", attrs=["dark"])
|
||||||
|
|
||||||
|
def print_reading(prefix, name, precache, cached, postcache):
|
||||||
|
ccolor = "yellow" if cached == "uncached" else "green"
|
||||||
|
|
||||||
|
write_log(prefix, color="white", attrs=["dark"])
|
||||||
|
write_log(name, color=COLORS[filetype], attrs=["bold"])
|
||||||
|
write_log(precache, color="white", attrs=["dark"])
|
||||||
|
write_log(cached, color=ccolor, attrs=["bold"])
|
||||||
|
write_log(postcache, color="white", attrs=["dark"])
|
||||||
|
write_log(" ...", color="white", attrs=["dark"])
|
||||||
|
|
||||||
|
def print_stats_1(walltime, speed):
|
||||||
|
write_log(" ")
|
||||||
|
write_log(walltime, color="blue", attrs=["bold"])
|
||||||
|
write_log(", ", color="white", attrs=["dark"])
|
||||||
|
write_log(speed, color="green", attrs=["bold"])
|
||||||
|
write_log(", ", color="white", attrs=["dark"])
|
||||||
|
|
||||||
|
def print_stats_2(kerntime, kernunits, cpu):
|
||||||
|
write_log(kerntime, color="cyan", attrs=["bold"], end=" ")
|
||||||
|
write_log(kernunits, color="cyan", attrs=["bold"])
|
||||||
|
write_log(", ", color="white", attrs=["dark"])
|
||||||
|
write_log(cpu, color="red", attrs=["bold"])
|
||||||
|
|
||||||
|
def print_iteration(prefix, cur, sep, total):
|
||||||
|
write_log(prefix, color="magenta", attrs=["bold", "underline"])
|
||||||
|
write_log(cur, color="green", attrs=["bold", "underline"])
|
||||||
|
write_log(sep, color="magenta", attrs=["bold", "underline"])
|
||||||
|
write_log(total, color="blue", attrs=["bold", "underline"], end="\n")
|
||||||
|
|
||||||
|
|
||||||
|
text = ""
|
||||||
|
while c := sys.stdin.read(1):
|
||||||
|
text += c
|
||||||
|
line = text
|
||||||
|
text = ""
|
||||||
|
if match := re.match(f"(Allocating file: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line):
|
||||||
|
filetype = match.group(3) if match.group(4) is None else match.group(4)
|
||||||
|
print_allocation(match.group(1), match.group(2), filetype, match.group(5))
|
||||||
|
elif match := re.match(r"( \[Done: )(\d+\.\d+s)(\]\n)", line):
|
||||||
|
print_allocation_done(match.group(1), match.group(2), match.group(3))
|
||||||
|
elif match := re.match(f"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( \()((un)?cached)( on \w*\))\n", line):
|
||||||
|
filetype = match.group(3) if match.group(4) is None else match.group(4)
|
||||||
|
print_reading(match.group(1), match.group(2), match.group(5), match.group(6), match.group(8))
|
||||||
|
elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), ((\d+ MB/s)|(\d\.\d GB/s))\n", line):
|
||||||
|
print_stats_1(match.group(1), match.group(2))
|
||||||
|
elif match := re.match("(\d+\.\d+)(s kern), (\d+% cpu)", line):
|
||||||
|
print_stats_2(match.group(1), match.group(2), match.group(3))
|
||||||
|
elif match := re.match(r"\d+\+\d+ records (in|out)\n", line):
|
||||||
|
pass
|
||||||
|
elif match := re.match("(Iteration )(\d+)( / )(\d+)\n", line):
|
||||||
|
print_iteration(match.group(1), match.group(2), match.group(3), match.group(4))
|
||||||
|
elif c == "\n":
|
||||||
|
write_log(line)
|
||||||
|
else:
|
||||||
|
text = line
|
|
@ -0,0 +1,17 @@
|
||||||
|
#compdef read_plus.zsh
|
||||||
|
|
||||||
|
function _read_plus.zsh() {
|
||||||
|
_arguments \
|
||||||
|
{-c,--client}'[the client to test]: : _alternative
|
||||||
|
"hosts\:hosts\: _ssh_hosts"
|
||||||
|
"domains\:domains\:($(virsh list --all --name))"' \
|
||||||
|
{-d,--direct}'[call dd with iflag=direct]' \
|
||||||
|
{-f,--file}'[write output to file]: : _files' \
|
||||||
|
{-i,--iterations}'[number of times to repeat the test]: :($(seq 100))' \
|
||||||
|
{-p,--mountpoint}'[the directory to mount the server]: : _files -/' \
|
||||||
|
{-s,--server}'[the server to test against]: : _alternative
|
||||||
|
"hosts\:hosts\: _ssh_hosts"
|
||||||
|
"domains\:domains\:($(virsh list --all --name))"' \
|
||||||
|
{-x,--export}'[the exported directory on the server]: : _files -/' \
|
||||||
|
{-z,--size}'[the size of the test files]: :($(seq 5120))'
|
||||||
|
}
|
|
@ -2,7 +2,8 @@
|
||||||
|
|
||||||
BIN=$HOME/bin
|
BIN=$HOME/bin
|
||||||
SCRIPTS=(grub-list.zsh setup-testdirs.zsh \
|
SCRIPTS=(grub-list.zsh setup-testdirs.zsh \
|
||||||
setup-xfstests.zsh run-xfstests.zsh)
|
setup-xfstests.zsh run-xfstests.zsh \
|
||||||
|
setup-read_plus.zsh setup-read_plus-files.py)
|
||||||
|
|
||||||
function install_script() {
|
function install_script() {
|
||||||
ssh $1 mkdir -p bin/
|
ssh $1 mkdir -p bin/
|
||||||
|
|
|
@ -0,0 +1,75 @@
|
||||||
|
#!/bin/zsh -e
|
||||||
|
CLIENT=(client)
|
||||||
|
SERVER=(server)
|
||||||
|
EXPORT=(/srv/test)
|
||||||
|
MOUNTPOINT=(/mnt/test)
|
||||||
|
SIZE=(2048)
|
||||||
|
ITERATIONS=(1)
|
||||||
|
|
||||||
|
zparseopts -F -K \
|
||||||
|
c:=CLIENT -client:=CLIENT \
|
||||||
|
d=DIRECT -direct=DIRECT \
|
||||||
|
f:=FILE -file:=FILE \
|
||||||
|
i:=ITERATIONS -iterations:=ITERATIONS \
|
||||||
|
p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \
|
||||||
|
s:=SERVER -server:=SERVER \
|
||||||
|
x:=EXPORT -export:=EXPORT \
|
||||||
|
z:=SIZE -size:=SIZE
|
||||||
|
|
||||||
|
BIN=$HOME/bin
|
||||||
|
COLOR="python -u $BIN/colors/read_plus.py ${SIZE[-1]} ${FILE[-1]}"
|
||||||
|
IFLAG=
|
||||||
|
RUN_READ_PLUS="sudo run-read_plus.zsh"
|
||||||
|
USER=$(whoami)
|
||||||
|
SRV_DIR=${EXPORT[-1]}/$USER/read_plus
|
||||||
|
TEST_DIR=${MOUNTPOINT[-1]}/read_plus
|
||||||
|
TEST_DEV=${SERVER[-1]}:$SRV_DIR
|
||||||
|
TEST_FILES=($(echo ${SIZE[-1]}M-{data,hole,mixed-{1,2,4,8,16}{d,h}}))
|
||||||
|
TIME_CMD="sudo /usr/bin/time -f \"%Ss kern, %P cpu\""
|
||||||
|
|
||||||
|
#
|
||||||
|
# Prepare to test
|
||||||
|
#
|
||||||
|
$BIN/vm.zsh boot ${CLIENT[-1]} ${SERVER[-1]}
|
||||||
|
$BIN/install-scripts.zsh ${CLIENT[-1]}
|
||||||
|
if [ ${#FILE} -gt 1 ]; then
|
||||||
|
mkdir -p $(dirname ${FILE[-1]})
|
||||||
|
[[ -f ${FILE[-1]} ]] && rm -fv ${FILE[-1]}
|
||||||
|
touch ${FILE[-1]}
|
||||||
|
fi
|
||||||
|
if [ ${#DIRECT} -gt 0 ]; then
|
||||||
|
IFLAG="iflag=direct"
|
||||||
|
fi
|
||||||
|
|
||||||
|
ssh ${CLIENT[-1]} "sudo setup-read_plus.zsh ${SERVER[-1]} ${EXPORT[-1]} \
|
||||||
|
${MOUNTPOINT[-1]} $USER ${SIZE[-1]} 1,2,4,8,16" | eval ${COLOR}
|
||||||
|
|
||||||
|
function dd_file() {
|
||||||
|
echo "Reading: $1 ($2 on ${SERVER[-1]})"
|
||||||
|
|
||||||
|
case $2 in
|
||||||
|
"uncached") args="eq" ;;
|
||||||
|
"cached") args="tq" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1"
|
||||||
|
ssh ${CLIENT[-1]} "sudo vmtouch -eq $TEST_DIR/$1"
|
||||||
|
ssh ${CLIENT[-1]} "$TIME_CMD dd if=$TEST_DIR/$1 $IFLAG of=/dev/null bs=$3"
|
||||||
|
}
|
||||||
|
|
||||||
|
for i in $(seq ${ITERATIONS[-1]}); do
|
||||||
|
echo
|
||||||
|
[[ ${#FILE} -gt 1 ]] && echo >> ${FILE[-1]}
|
||||||
|
|
||||||
|
ssh ${CLIENT[-1]} "sudo mount -o sec=sys,v4.2 $TEST_DEV $TEST_DIR"
|
||||||
|
bsize=$(ssh ${CLIENT[-1]} "mount | grep $TEST_DEV" | awk -F rsize= '{print $2}' | awk -F, '{print $1}')
|
||||||
|
if [[ ${ITERATIONS[-1]} -gt 1 ]]; then
|
||||||
|
echo "Iteration $i / ${ITERATIONS[-1]}" | eval ${COLOR}
|
||||||
|
fi
|
||||||
|
for f in $TEST_FILES; do
|
||||||
|
for cache in uncached cached; do
|
||||||
|
dd_file $f $cache $bsize 2>&1 | eval ${COLOR}
|
||||||
|
done
|
||||||
|
done
|
||||||
|
ssh ${CLIENT[-1]} "sudo umount $TEST_DIR"
|
||||||
|
done
|
|
@ -0,0 +1,48 @@
|
||||||
|
#!/usr/bin/python -u
|
||||||
|
import datetime
|
||||||
|
import pathlib
|
||||||
|
import os
|
||||||
|
import string
|
||||||
|
import sys
|
||||||
|
|
||||||
|
PAGE_SIZE = 4096
|
||||||
|
PAGE_HEADER = "<<<Start Of Page>>>\n"
|
||||||
|
PAGE_FOOTER = "\n<<<End Of Page>>>\n"
|
||||||
|
|
||||||
|
line = string.ascii_letters + string.digits + "\n"
|
||||||
|
data_len = PAGE_SIZE - len(PAGE_HEADER + PAGE_FOOTER)
|
||||||
|
(n_lines, n_chars) = divmod(data_len, len(line))
|
||||||
|
page_data = PAGE_HEADER + (line * n_lines) + line[:n_chars] + PAGE_FOOTER
|
||||||
|
PAGE_DATA = page_data.encode()
|
||||||
|
|
||||||
|
FILE_BASE = pathlib.Path(sys.argv[1])
|
||||||
|
FILE_SIZE = int(sys.argv[2])
|
||||||
|
FILE_SIZE_BYTES = FILE_SIZE * 1024 * 1024
|
||||||
|
FILE_N_PAGES = FILE_SIZE_BYTES // PAGE_SIZE
|
||||||
|
|
||||||
|
|
||||||
|
def allocate_testfile(name, n_pages, hole):
|
||||||
|
file = FILE_BASE / f"{FILE_SIZE}M-{name}"
|
||||||
|
if file.exists():
|
||||||
|
return
|
||||||
|
print(f"Allocating file: {file.stem} ...", end="")
|
||||||
|
start = datetime.datetime.now()
|
||||||
|
with open(file, 'wb') as f:
|
||||||
|
f.truncate(FILE_SIZE_BYTES)
|
||||||
|
for chunk in range(FILE_N_PAGES // n_pages):
|
||||||
|
if hole:
|
||||||
|
f.seek(PAGE_SIZE * n_pages, os.SEEK_CUR)
|
||||||
|
else:
|
||||||
|
f.write(PAGE_DATA * n_pages)
|
||||||
|
hole = not hole
|
||||||
|
os.fsync(f)
|
||||||
|
tdelta = datetime.datetime.now() - start
|
||||||
|
time = float(f"{tdelta.seconds}.{tdelta.microseconds}")
|
||||||
|
print(f" [Done: {round(time, 2):.2f}s]")
|
||||||
|
|
||||||
|
|
||||||
|
allocate_testfile("data", FILE_N_PAGES, False)
|
||||||
|
allocate_testfile("hole", FILE_N_PAGES, True)
|
||||||
|
for chunk in [int(c) for c in sys.argv[3].split(",")]:
|
||||||
|
allocate_testfile(f"mixed-{chunk}d", chunk, False)
|
||||||
|
allocate_testfile(f"mixed-{chunk}h", chunk, True)
|
|
@ -0,0 +1,22 @@
|
||||||
|
#!/bin/zsh
|
||||||
|
SERVER=$1
|
||||||
|
EXPORT=$2
|
||||||
|
MOUNTPOINT=$3/read_plus
|
||||||
|
USER=$4
|
||||||
|
SIZE=$5
|
||||||
|
CHUNKS=$6
|
||||||
|
|
||||||
|
if [ "$#" -ne 6 ]; then
|
||||||
|
echo "Usage: $0 {server} {export} {mountpoint} {user} {size} {chunks}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p $MOUNTPOINT
|
||||||
|
mount -o sec=sys $SERVER:$EXPORT $MOUNTPOINT
|
||||||
|
TRAPEXIT() {
|
||||||
|
sudo umount -f $MOUNTPOINT
|
||||||
|
}
|
||||||
|
|
||||||
|
mkdir -p -m 777 $MOUNTPOINT/$USER/read_plus
|
||||||
|
setup-read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS
|
||||||
|
sync
|
Loading…
Reference in New Issue