diff --git a/colors/xfstests.py b/colors/xfstests.py index 648724a..5102862 100644 --- a/colors/xfstests.py +++ b/colors/xfstests.py @@ -1,73 +1,159 @@ +"""Colorize xfstests.zsh output.""" import sys import re import termcolor -PROTO_COLORS = { "tcp":"white", "rdma":"yellow" } -PROTO = sys.argv[1] -VERS_COLORS = { "v3":"red", "v4.0":"green", "v4.1":"blue", "v4.2":"yellow" } +PROTO_COLORS = {"tcp": "white", "rdma": "yellow"} +STATUS_COLORS = {"Ran": "cyan", "Not run": "magenta", "Failures": "red"} +VERS_COLORS = {"v3": "red", "v4.0": "green", "v4.1": "blue", "v4.2": "yellow"} + +PROTO = sys.argv[1] VERS = f"v{sys.argv[2]}" COLOR = VERS_COLORS[VERS] +PROTO_COLOR = PROTO_COLORS[PROTO] +CONFIG = [] -prefix = termcolor.colored("[", "white", attrs=["bold"]) -prefix += termcolor.colored(PROTO.rjust(4), PROTO_COLORS[PROTO], attrs=["dark"]) -prefix += termcolor.colored(",", "white", attrs=["bold"]) -prefix += termcolor.colored(VERS.ljust(4), COLOR, attrs=["bold"]) -prefix += termcolor.colored("] ", "white", attrs=["bold"]) +PREFIX = termcolor.colored("[", "white", attrs=["bold"]) +PREFIX += termcolor.colored(PROTO.rjust(4), PROTO_COLOR, attrs=["dark"]) +PREFIX += termcolor.colored(",", "white", attrs=["bold"]) +PREFIX += termcolor.colored(VERS.ljust(4), COLOR, attrs=["bold"]) +PREFIX += termcolor.colored("] ", "white", attrs=["bold"]) -CONFIG = [ ] -def add_config(line): - [ key, val ] = [ s.strip() for s in re.split("--", line) ] + +def add_config(line: str) -> None: + """Format a configuration entry and add it to the CONFIG list.""" + [key, val] = [s.strip() for s in re.split("--", line)] if key == "MKFS_OPTIONS": return elif key == "MOUNT_OPTIONS" and val[:2] != "-o": return - parts = [ prefix ] + parts = [PREFIX] parts.append(termcolor.colored(key.ljust(14), COLOR, attrs=["bold"])) parts.append(termcolor.colored("-- ", "white", attrs=["dark"])) parts.append(termcolor.colored(val, "white", attrs=["bold"])) CONFIG.append("".join(parts)) -def print_config(): + +def print_config() -> None: + """Print out the collected configuration entries and clear the list.""" if len(CONFIG) > 0: print("\n".join(CONFIG), end="\n\n") CONFIG.clear() -def print_test(match): - parts = [ prefix ] + +def print_test(match: re.Match) -> None: + """Format and print a single testcase.""" + parts = [PREFIX] parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"])) - if m := re.match("(\[not run\])(.*?\n)", match.group(2)): + if m := re.match(r"(\[not run\])(.*?\n)", match.group(2)): return parts.append(termcolor.colored(m.group(1), "yellow")) parts.append(termcolor.colored(m.group(2), "white", attrs=["bold"])) - elif m := re.match("(\d+s)( ... +)(\d+s\n)", match.group(2)): + elif m := re.match(r"(\d+s)( ... +)(\d+s\n)", match.group(2)): parts.append(termcolor.colored(m.group(1), "green")) parts.append(termcolor.colored(m.group(2), "white", attrs=["dark"])) parts.append(termcolor.colored(m.group(3), "green")) - elif re.match("\d+s\n", match.group(2)): + elif re.match(r"\d+s\n", match.group(2)): parts.append(termcolor.colored(match.group(2), "green")) - else: # Test failed - line = re.sub("\]-", "\] -", match.group(2)) + else: # Test failed + line = re.sub(r"\]-", r"\] -", match.group(2)) parts.append(termcolor.colored(line, "magenta", attrs=["bold"])) print("".join(parts), end="") -def print_diff(line): - print(prefix, end="") - if re.match(" +@@.*?@@\n", line): + +def print_diff(line: str) -> None: + """Format and print out a single line of a failing test diff.""" + print(PREFIX, end="") + if re.match(r" +@@.*?@@\n", line): termcolor.cprint(line, color="yellow", attrs=["dark"], end="") - elif re.match(" +-", line): + elif re.match(r" +-", line): termcolor.cprint(line, color="magenta", attrs=["dark"], end="") - elif re.match(" +\+", line): + elif re.match(r" +\+", line): termcolor.cprint(line, color="cyan", attrs=["dark"], end="") - elif re.match(" +\(Run", line): + elif re.match(r" +\(Run", line): termcolor.cprint(line, color="magenta", attrs=["bold"], end="") else: termcolor.cprint(line, color="white", attrs=["dark"], end="") +def print_test_list(status: str, line: str) -> None: + """Format and print the list of tests that ran, failed, or were skipped.""" + tests = dict() + _tmp = [t.split("/") for t in line[len(status)+1:].split()] + [tests.setdefault(t[0], []).append(t[1]) for t in _tmp] + + for key in sorted(tests.keys()): + parts = [PREFIX] + parts.append(termcolor.colored(status, STATUS_COLORS[status], + attrs=["bold"])) + parts.append(termcolor.colored(": ", "white", attrs=["bold"])) + parts.append(termcolor.colored(key, color=COLOR, attrs=["bold"])) + parts.append(termcolor.colored(": [", "white", attrs=["bold"])) + for i, test_num in enumerate(sorted(tests[key])): + sep = termcolor.colored(", ", attrs=["bold"] if i > 0 else "") + if i != 0: + parts.append(sep) + attr = ["bold", "dark"] if i % 2 else ["bold"] + parts.append(termcolor.colored(test_num, STATUS_COLORS[status], + attrs=attr)) + parts.append(termcolor.colored("]", "white", attrs=["bold"])) + print("".join(parts)) + + +def print_failed(match: re.Match) -> None: + """Format and print the number of failing tests.""" + parts = [PREFIX] + parts.append(termcolor.colored("Failed ", "red", attrs=["bold"])) + parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"])) + parts.append(termcolor.colored(" of ", "red", attrs=["bold"])) + parts.append(termcolor.colored(match.group(2), COLOR, attrs=["bold"])) + parts.append(termcolor.colored(" tests", "red", attrs=["bold"])) + print("".join(parts)) + + +def print_all_passed(match: re.Match) -> None: + """Format and print a status line if all tests pass.""" + parts = [PREFIX] + parts.append(termcolor.colored("Passed all ", "green", attrs=["bold"])) + parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"])) + parts.append(termcolor.colored(" tests", "green", attrs=["bold"])) + print("".join(parts)) + + +def print_xunit(match: re.Match) -> None: + """Format and print the xunit file location.""" + parts = [PREFIX] + parts.append(termcolor.colored(match.group(1), PROTO_COLOR, + attrs=["dark", "bold"])) + parts.append(termcolor.colored(": ", "white", attrs=["bold"])) + parts.append(termcolor.colored(match.group(2), COLOR, attrs=["bold"])) + print("".join(parts)) + + +def print_xfstestsdb(match: re.Match) -> None: + """Format and print the xfstestsdb xunit-add line.""" + parts = [PREFIX] + parts.append(termcolor.colored(match.group(1).title(), PROTO_COLOR, + attrs=["dark", "bold"])) + parts.append(termcolor.colored(" '", "white", attrs=["bold"])) + parts.append(termcolor.colored(match.group(2), COLOR, + attrs=["bold"])) + parts.append(termcolor.colored("' ", "white", attrs=["bold"])) + parts.append(termcolor.colored(match.group(3), PROTO_COLOR, + attrs=["dark", "bold"])) + parts.append(termcolor.colored(match.group(4), COLOR, attrs=["bold"])) + parts.append(termcolor.colored(match.group(5), PROTO_COLOR, + attrs=["dark", "bold"])) + parts.append(termcolor.colored(" #", "white", attrs=["bold"])) + parts.append(termcolor.colored(match.group(6), COLOR, + attrs=["bold"])) + print("".join(parts)) + + for line in sys.stdin: if line == "\n": print_config() @@ -76,11 +162,20 @@ for line in sys.stdin: add_config(match.group(1)) line = f"COMPILED -- {match.group(3)}" add_config(line) - elif match := re.match("(^[a-z]+/[\d]+[ | ]+)(.*?\n)", line): + elif match := re.match(r"(^[a-z]+/\d+[ | ]+)(.*?\n)", line): print_test(match) elif match := re.match("^ (.*?)\n", line): print_diff(line) + elif match := re.match(r"^(Ran|Not run|Failures):([ a-z]+/\d+)+\n", line): + print_test_list(match.group(1), line) + elif match := re.match(r"^Failed (\d+) of (\d+) tests\n", line): + print_failed(match) + elif match := re.match(r"^Passed all (\d+) tests\n", line): + print_all_passed(match) + elif match := re.match(r"(^Xunit report): (.*?.xml)\n", line): + print_xunit(match) + elif match := re.match(r"(^added) '(.*?)' (with )" + r"(\d+)( testcases to run) #(\d+)\n", line): + print_xfstestsdb(match) else: - #pass - print(prefix, end="") - print(line, end="") + print(PREFIX + line, end="") diff --git a/completions/_xfstests.zsh b/completions/_xfstests.zsh index ef27992..d2ba42e 100644 --- a/completions/_xfstests.zsh +++ b/completions/_xfstests.zsh @@ -8,9 +8,10 @@ function _xfstests.zsh() { {-g,--group}'[the xfstests group to run]: :($(_list_xfstests_groups))' \ {-k,--scratch}'[the exported scratch directory on the server]: : _files -/' \ {-p,--mountpoint}'[the directory to mount the server]: : _files -/' \ - --nconnect'[number of connections]: :($(seq 1 32))' \ + --no-scratch'[do not use a scratch mount during testing]' \ + --no-server-access'[do not attempt to ssh into the server]' \ + \*{-o,--options}'[mount options for all mounts]: :' \ {-r,--rdma}'[test using rdma]: :(rxe siw)' \ - --rwsize'[test with a secific rsize & wsize]: :($(seq 4096 4096 1048576))' \ {-q,--scratchmnt}'[the directory to mount the scratch export]: : _files -/' \ {-s,--server}'[the server to test against]: : _alternative "hosts\:hosts\: _ssh_hosts" diff --git a/run/xfstests.zsh b/run/xfstests.zsh index bb1284f..757c8ba 100755 --- a/run/xfstests.zsh +++ b/run/xfstests.zsh @@ -8,23 +8,31 @@ zparseopts -D -K -server:=SERVER -opts:=OPTS -user:=USER \ xdir=$(echo "nfsv${VERSION[-1]}${PROTO[-1]}" | sed 's/\.//' | sed 's/v40/v4/') cd /home/${USER[-1]}/xfstests-dev -export TEST_DEV=${SERVER[-1]}:${EXPORT[-1]}/${USER[-1]}/$xdir +export TEST_DEV=$(echo ${SERVER[-1]}:${EXPORT[-1]}/${USER[-1]}/$xdir | sed 's|//|/|') export TEST_DIR=${MOUNTPOINT[-1]}/$xdir -export SCRATCH_DEV=${SERVER[-1]}:${SCRATCH[-1]}/${USER[-1]}/$xdir -export SCRATCH_MNT=${SCRATCHMNT[-1]}/$xdir +if [ "${SCRATCH[-1]}" != "NONE" ]; then + export SCRATCH_DEV=${SERVER[-1]}:${SCRATCH[-1]}/${USER[-1]}/$xdir + export SCRATCH_MNT=${SCRATCHMNT[-1]}/$xdir +fi export NFS_MOUNT_OPTIONS="-o proto=${PROTO[-1]},v${VERSION[-1]},${OPTS[-1]}" export RESULT_BASE=/home/${USER[-1]}/xfstests-dev/results/${PROTO[-1]}/${VERSION[-1]} mkdir -p $TEST_DIR -mkdir -p $SCRATCH_MNT +if [ "${SCRATCH[-1]}" != "NONE" ]; then + mkdir -p $SCRATCH_MNT +fi [ -e local.config ] && rm -f local.config [ -e $RESULT_BASE/result.xml ] && rm -f $RESULT_BASE/result.xml echo "TIME -- $(date)" echo "TEST_DEV -- $TEST_DEV" echo "TEST_DIR -- $TEST_DIR" -echo "SCRATCH_DEV -- $SCRATCH_DEV" -echo "SCRATCH_MNT -- $SCRATCH_MNT" +if [ "${SCRATCH[-1]}" != "NONE" ]; then + echo "SCRATCH_DEV -- $SCRATCH_DEV" + echo "SCRATCH_MNT -- $SCRATCH_MNT" +else + echo "SCRATCH_DEV -- NONE" +fi echo "MOUNT_OPTIONS -- $NFS_MOUNT_OPTIONS" if [ ! -z "$*" ]; then @@ -32,4 +40,3 @@ if [ ! -z "$*" ]; then fi ./check -nfs -r -R xunit $* -#./check -nfs -R xunit $* diff --git a/setup/xfstests.zsh b/setup/xfstests.zsh index 5a6179a..b9c0e7d 100755 --- a/setup/xfstests.zsh +++ b/setup/xfstests.zsh @@ -17,4 +17,6 @@ useradd --badnames 123456-fsgqa 2>/dev/null useradd fsgqa2 2>/dev/null setup.zsh testdirs.zsh $SERVER $EXPORT $MOUNTPOINT $USER -setup.zsh testdirs.zsh $SERVER $SCRATCH $SCRATCHMNT $USER +if [ "$SCRATCH" != "NONE" ]; then + setup.zsh testdirs.zsh $SERVER $SCRATCH $SCRATCHMNT $USER +fi diff --git a/xfstests.zsh b/xfstests.zsh index 7c147d7..679aa75 100755 --- a/xfstests.zsh +++ b/xfstests.zsh @@ -8,17 +8,18 @@ zparseopts -D -K \ c:=CLIENT -client:=CLIENT \ g:=GROUP -group:=GROUP \ k:=SCRATCH -scratch:=SCRATCH \ - -nconnect:=NCONNECT \ + -no-scratch=NO_SCRATCH \ + -no-server-access=NO_SERVER_ACCESS \ + o+:=MNTOPTS -options+:=MNTOPTS \ p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \ q:=SCRATCHMNT -scratchmnt:=SCRATCHMNT \ r:=RDMA -rdma:=RDMA \ - -rwsize:=RWSIZE \ s:=SERVER -server:=SERVER \ x:=EXPORT -export:=EXPORT \ v+:=VERSION -version+:=VERSION -COLOR=$BIN/colors/xfstests.py -OPTIONS="sec=sys" +COLOR=$COLORS/xfstests.py +OPTIONS="sec=sys$(echo $MNTOPTS | sed -E 's/ ?-o /,/g')" RESULT_BASE=$HOME/.local/share/xfstests/ DATE=$(date +%Y-%m-%d-%H:%M:%S%z) TODAY=$(date +%Y/%m/%d) @@ -40,12 +41,11 @@ function setup_client_func() ${MOUNTPOINT[-1]} ${SCRATCH[-1]} \ ${SCRATCHMNT[-1]} $USER } + prepare_to_test -if [ ${#NCONNECT} -gt 0 ]; then - OPTIONS="$OPTIONS,nconnect=${NCONNECT[-1]}" -fi -if [ ${#RWSIZE} -gt 0 ]; then - OPTIONS="$OPTIONS,rsize=${RWSIZE[-1]},wsize=${RWSIZE[-1]}" + +if [ ${#NO_SCRATCH} -gt 0 ]; then + SCRATCH+=("NONE") fi testargs=() @@ -55,28 +55,36 @@ for arg in $*; do done run_xfs_tests() { - client_run xfstests.zsh --server ${SERVER[-1]} \ - --export ${EXPORT[-1]} \ - --mountpoint ${MOUNTPOINT[-1]} \ - --scratch ${SCRATCH[-1]} \ - --scratchmnt ${SCRATCHMNT[-1]} \ - --proto $1 \ - --version $2 \ - --opts $OPTIONS \ - --user $USER $GROUP $testargs | python $COLOR $1 $2 - scp -q ${CLIENT[-1]}:$REMOTE_RESULTS/$1/$2/result.xml $TMP_RESULTS/$1-$2.xml + for proto in $PROTO; do + client_run xfstests.zsh --server ${SERVER[-1]} \ + --export ${EXPORT[-1]} \ + --mountpoint ${MOUNTPOINT[-1]} \ + --scratch ${SCRATCH[-1]} \ + --scratchmnt ${SCRATCHMNT[-1]} \ + --proto $proto --version $1 \ + --opts $OPTIONS \ + --user $USER $GROUP $testargs | python $COLOR $proto $1 + + if [[ "$proto" == "rdma" ]]; then + xunit="$TMP_RESULTS/${RDMA[-1]}-$1.xml" + else + xunit="$TMP_RESULTS/$proto-$1.xml" + fi + + scp -q ${CLIENT[-1]}:$REMOTE_RESULTS/$proto/$1/result.xml $xunit + xfstestsdb xunit read $RUNID $xunit | python $COLOR $proto $1 + done } + # # Run tests # mkdir -p $TMP_RESULTS -for proto in $PROTO; do - for vers in $VERSION; do - run_xfs_tests $proto $vers & - done +RUNID=$(xfstestsdb new ${SERVER[-1]}:${EXPORT[-1]} | awk -F" | #" '{print $3}') +for vers in $VERSION; do + run_xfs_tests $vers & done - wait mkdir -p $RESULTS @@ -84,5 +92,12 @@ tar -cJf $RESULTS/$NOW.tar.xz -C $(dirname $TMP_RESULTS) $NOW/ if [ ! -z "$TAG" ]; then mkdir -p $TAGRES ln $RESULTS/$NOW.tar.xz $TAGRES/$DATE.tar.xz + xfstestsdb tag $RUNID $TAG fi + +echo +xfstestsdb xunit list --results --runid $RUNID +echo +xfstestsdb show --failure $RUNID +echo report-xfstests.py $RESULTS/$NOW.tar.xz & disown