From dd97c7079ba1be05a5f81087d21f42773bb6c77c Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Mon, 18 Apr 2022 11:27:52 -0400 Subject: [PATCH] read_plus: Add script for testing the READ_PLUS operation Signed-off-by: Anna Schumaker --- colors/read_plus.py | 77 ++++++++++++++++++++++++++++++++++++++ completions/_read_plus.zsh | 17 +++++++++ install-scripts.zsh | 3 +- read_plus.zsh | 75 +++++++++++++++++++++++++++++++++++++ setup-read_plus-files.py | 48 ++++++++++++++++++++++++ setup-read_plus.zsh | 22 +++++++++++ 6 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 colors/read_plus.py create mode 100644 completions/_read_plus.zsh create mode 100755 read_plus.zsh create mode 100755 setup-read_plus-files.py create mode 100755 setup-read_plus.zsh diff --git a/colors/read_plus.py b/colors/read_plus.py new file mode 100644 index 0000000..ce5fff5 --- /dev/null +++ b/colors/read_plus.py @@ -0,0 +1,77 @@ +import sys +import re +import termcolor + +SIZE = sys.argv[1] +FILE = open(sys.argv[2], 'a') if len(sys.argv) >= 3 else None +COLORS = {"data": "red", "hole": "green", "d": "cyan", "h": "yellow"} + +def write_log(text, color=None, attrs=[], end=""): + if FILE: FILE.write(text + end) + termcolor.cprint(text, color=color, attrs=attrs, end=end) + +def print_allocation(prefix, name, filetype, extra): + write_log(prefix, color="white", attrs=["dark"]) + write_log(name, color=COLORS[filetype], attrs=["bold"]) + write_log(extra, color="white", attrs=["dark"]) + +def print_allocation_done(done, time, end): + write_log(done, color="white", attrs=["dark"]) + write_log(time, color="blue", attrs=["bold"]) + write_log(end, color="white", attrs=["dark"]) + +def print_reading(prefix, name, precache, cached, postcache): + ccolor = "yellow" if cached == "uncached" else "green" + + write_log(prefix, color="white", attrs=["dark"]) + write_log(name, color=COLORS[filetype], attrs=["bold"]) + write_log(precache, color="white", attrs=["dark"]) + write_log(cached, color=ccolor, attrs=["bold"]) + write_log(postcache, color="white", attrs=["dark"]) + write_log(" ...", color="white", attrs=["dark"]) + +def print_stats_1(walltime, speed): + write_log(" ") + write_log(walltime, color="blue", attrs=["bold"]) + write_log(", ", color="white", attrs=["dark"]) + write_log(speed, color="green", attrs=["bold"]) + write_log(", ", color="white", attrs=["dark"]) + +def print_stats_2(kerntime, kernunits, cpu): + write_log(kerntime, color="cyan", attrs=["bold"], end=" ") + write_log(kernunits, color="cyan", attrs=["bold"]) + write_log(", ", color="white", attrs=["dark"]) + write_log(cpu, color="red", attrs=["bold"]) + +def print_iteration(prefix, cur, sep, total): + write_log(prefix, color="magenta", attrs=["bold", "underline"]) + write_log(cur, color="green", attrs=["bold", "underline"]) + write_log(sep, color="magenta", attrs=["bold", "underline"]) + write_log(total, color="blue", attrs=["bold", "underline"], end="\n") + + +text = "" +while c := sys.stdin.read(1): + text += c + line = text + text = "" + if match := re.match(f"(Allocating file: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( ...)", line): + filetype = match.group(3) if match.group(4) is None else match.group(4) + print_allocation(match.group(1), match.group(2), filetype, match.group(5)) + elif match := re.match(r"( \[Done: )(\d+\.\d+s)(\]\n)", line): + print_allocation_done(match.group(1), match.group(2), match.group(3)) + elif match := re.match(f"(Reading: )({SIZE}M-(data|hole|mixed-\d+(d|h)))( \()((un)?cached)( on \w*\))\n", line): + filetype = match.group(3) if match.group(4) is None else match.group(4) + print_reading(match.group(1), match.group(2), match.group(5), match.group(6), match.group(8)) + elif match := re.match(r"\d+ bytes \([\w,\. ]*\) copied, (\d+\.\d+ s), ((\d+ MB/s)|(\d\.\d GB/s))\n", line): + print_stats_1(match.group(1), match.group(2)) + elif match := re.match("(\d+\.\d+)(s kern), (\d+% cpu)", line): + print_stats_2(match.group(1), match.group(2), match.group(3)) + elif match := re.match(r"\d+\+\d+ records (in|out)\n", line): + pass + elif match := re.match("(Iteration )(\d+)( / )(\d+)\n", line): + print_iteration(match.group(1), match.group(2), match.group(3), match.group(4)) + elif c == "\n": + write_log(line) + else: + text = line diff --git a/completions/_read_plus.zsh b/completions/_read_plus.zsh new file mode 100644 index 0000000..52d0b40 --- /dev/null +++ b/completions/_read_plus.zsh @@ -0,0 +1,17 @@ +#compdef read_plus.zsh + +function _read_plus.zsh() { + _arguments \ + {-c,--client}'[the client to test]: : _alternative + "hosts\:hosts\: _ssh_hosts" + "domains\:domains\:($(virsh list --all --name))"' \ + {-d,--direct}'[call dd with iflag=direct]' \ + {-f,--file}'[write output to file]: : _files' \ + {-i,--iterations}'[number of times to repeat the test]: :($(seq 100))' \ + {-p,--mountpoint}'[the directory to mount the server]: : _files -/' \ + {-s,--server}'[the server to test against]: : _alternative + "hosts\:hosts\: _ssh_hosts" + "domains\:domains\:($(virsh list --all --name))"' \ + {-x,--export}'[the exported directory on the server]: : _files -/' \ + {-z,--size}'[the size of the test files]: :($(seq 5120))' +} diff --git a/install-scripts.zsh b/install-scripts.zsh index cd9fd1a..52ae73d 100755 --- a/install-scripts.zsh +++ b/install-scripts.zsh @@ -2,7 +2,8 @@ BIN=$HOME/bin SCRIPTS=(grub-list.zsh setup-testdirs.zsh \ - setup-xfstests.zsh run-xfstests.zsh) + setup-xfstests.zsh run-xfstests.zsh \ + setup-read_plus.zsh setup-read_plus-files.py) function install_script() { ssh $1 mkdir -p bin/ diff --git a/read_plus.zsh b/read_plus.zsh new file mode 100755 index 0000000..38ea01f --- /dev/null +++ b/read_plus.zsh @@ -0,0 +1,75 @@ +#!/bin/zsh -e +CLIENT=(client) +SERVER=(server) +EXPORT=(/srv/test) +MOUNTPOINT=(/mnt/test) +SIZE=(2048) +ITERATIONS=(1) + +zparseopts -F -K \ + c:=CLIENT -client:=CLIENT \ + d=DIRECT -direct=DIRECT \ + f:=FILE -file:=FILE \ + i:=ITERATIONS -iterations:=ITERATIONS \ + p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \ + s:=SERVER -server:=SERVER \ + x:=EXPORT -export:=EXPORT \ + z:=SIZE -size:=SIZE + +BIN=$HOME/bin +COLOR="python -u $BIN/colors/read_plus.py ${SIZE[-1]} ${FILE[-1]}" +IFLAG= +RUN_READ_PLUS="sudo run-read_plus.zsh" +USER=$(whoami) +SRV_DIR=${EXPORT[-1]}/$USER/read_plus +TEST_DIR=${MOUNTPOINT[-1]}/read_plus +TEST_DEV=${SERVER[-1]}:$SRV_DIR +TEST_FILES=($(echo ${SIZE[-1]}M-{data,hole,mixed-{1,2,4,8,16}{d,h}})) +TIME_CMD="sudo /usr/bin/time -f \"%Ss kern, %P cpu\"" + +# +# Prepare to test +# +$BIN/vm.zsh boot ${CLIENT[-1]} ${SERVER[-1]} +$BIN/install-scripts.zsh ${CLIENT[-1]} +if [ ${#FILE} -gt 1 ]; then + mkdir -p $(dirname ${FILE[-1]}) + [[ -f ${FILE[-1]} ]] && rm -fv ${FILE[-1]} + touch ${FILE[-1]} +fi +if [ ${#DIRECT} -gt 0 ]; then + IFLAG="iflag=direct" +fi + +ssh ${CLIENT[-1]} "sudo setup-read_plus.zsh ${SERVER[-1]} ${EXPORT[-1]} \ + ${MOUNTPOINT[-1]} $USER ${SIZE[-1]} 1,2,4,8,16" | eval ${COLOR} + +function dd_file() { + echo "Reading: $1 ($2 on ${SERVER[-1]})" + + case $2 in + "uncached") args="eq" ;; + "cached") args="tq" ;; + esac + + ssh ${SERVER[-1]} "sudo vmtouch -$args $SRV_DIR/$1" + ssh ${CLIENT[-1]} "sudo vmtouch -eq $TEST_DIR/$1" + ssh ${CLIENT[-1]} "$TIME_CMD dd if=$TEST_DIR/$1 $IFLAG of=/dev/null bs=$3" +} + +for i in $(seq ${ITERATIONS[-1]}); do + echo + [[ ${#FILE} -gt 1 ]] && echo >> ${FILE[-1]} + + ssh ${CLIENT[-1]} "sudo mount -o sec=sys,v4.2 $TEST_DEV $TEST_DIR" + bsize=$(ssh ${CLIENT[-1]} "mount | grep $TEST_DEV" | awk -F rsize= '{print $2}' | awk -F, '{print $1}') + if [[ ${ITERATIONS[-1]} -gt 1 ]]; then + echo "Iteration $i / ${ITERATIONS[-1]}" | eval ${COLOR} + fi + for f in $TEST_FILES; do + for cache in uncached cached; do + dd_file $f $cache $bsize 2>&1 | eval ${COLOR} + done + done + ssh ${CLIENT[-1]} "sudo umount $TEST_DIR" +done diff --git a/setup-read_plus-files.py b/setup-read_plus-files.py new file mode 100755 index 0000000..cf9d73f --- /dev/null +++ b/setup-read_plus-files.py @@ -0,0 +1,48 @@ +#!/usr/bin/python -u +import datetime +import pathlib +import os +import string +import sys + +PAGE_SIZE = 4096 +PAGE_HEADER = "<<>>\n" +PAGE_FOOTER = "\n<<>>\n" + +line = string.ascii_letters + string.digits + "\n" +data_len = PAGE_SIZE - len(PAGE_HEADER + PAGE_FOOTER) +(n_lines, n_chars) = divmod(data_len, len(line)) +page_data = PAGE_HEADER + (line * n_lines) + line[:n_chars] + PAGE_FOOTER +PAGE_DATA = page_data.encode() + +FILE_BASE = pathlib.Path(sys.argv[1]) +FILE_SIZE = int(sys.argv[2]) +FILE_SIZE_BYTES = FILE_SIZE * 1024 * 1024 +FILE_N_PAGES = FILE_SIZE_BYTES // PAGE_SIZE + + +def allocate_testfile(name, n_pages, hole): + file = FILE_BASE / f"{FILE_SIZE}M-{name}" + if file.exists(): + return + print(f"Allocating file: {file.stem} ...", end="") + start = datetime.datetime.now() + with open(file, 'wb') as f: + f.truncate(FILE_SIZE_BYTES) + for chunk in range(FILE_N_PAGES // n_pages): + if hole: + f.seek(PAGE_SIZE * n_pages, os.SEEK_CUR) + else: + f.write(PAGE_DATA * n_pages) + hole = not hole + os.fsync(f) + tdelta = datetime.datetime.now() - start + time = float(f"{tdelta.seconds}.{tdelta.microseconds}") + print(f" [Done: {round(time, 2):.2f}s]") + + +allocate_testfile("data", FILE_N_PAGES, False) +allocate_testfile("hole", FILE_N_PAGES, True) +for chunk in [int(c) for c in sys.argv[3].split(",")]: + allocate_testfile(f"mixed-{chunk}d", chunk, False) + allocate_testfile(f"mixed-{chunk}h", chunk, True) diff --git a/setup-read_plus.zsh b/setup-read_plus.zsh new file mode 100755 index 0000000..c0d8d5c --- /dev/null +++ b/setup-read_plus.zsh @@ -0,0 +1,22 @@ +#!/bin/zsh +SERVER=$1 +EXPORT=$2 +MOUNTPOINT=$3/read_plus +USER=$4 +SIZE=$5 +CHUNKS=$6 + +if [ "$#" -ne 6 ]; then + echo "Usage: $0 {server} {export} {mountpoint} {user} {size} {chunks}" + exit 1 +fi + +mkdir -p $MOUNTPOINT +mount -o sec=sys $SERVER:$EXPORT $MOUNTPOINT +TRAPEXIT() { + sudo umount -f $MOUNTPOINT +} + +mkdir -p -m 777 $MOUNTPOINT/$USER/read_plus +setup-read_plus-files.py $MOUNTPOINT/$USER/read_plus $SIZE $CHUNKS +sync