xfstests.zsh: Various Updates

* Use $COLORS from common.zsh
* Add a --options option to pass generic NFS mount options
* Add the --no-scratch option to run tests without a scratch mount
* Add the --no-server-access option for testing against somebody else's
  server
* Remove the --nconnect and --rwsize options
* Run tcp and rdma tests sequentially instead of in parallel
* Add xfstestsdb support
* Clean up colors/xfstests.py to fix flake8 style errors
* Colorize xfstestsdb xunit output

Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>

colors/xfstests.py

@@ -1,73 +1,159 @@
+"""Colorize xfstests.zsh output."""
 import sys
 import re
 import termcolor
 
-PROTO_COLORS = { "tcp":"white", "rdma":"yellow" }
-PROTO = sys.argv[1]
 
-VERS_COLORS = { "v3":"red", "v4.0":"green", "v4.1":"blue", "v4.2":"yellow" }
+PROTO_COLORS = {"tcp": "white", "rdma": "yellow"}
+STATUS_COLORS = {"Ran": "cyan", "Not run": "magenta", "Failures": "red"}
+VERS_COLORS = {"v3": "red", "v4.0": "green", "v4.1": "blue", "v4.2": "yellow"}
+
+PROTO = sys.argv[1]
 VERS = f"v{sys.argv[2]}"
 COLOR = VERS_COLORS[VERS]
+PROTO_COLOR = PROTO_COLORS[PROTO]
+CONFIG = []
 
-prefix  = termcolor.colored("[", "white", attrs=["bold"])
+PREFIX = termcolor.colored("[", "white", attrs=["bold"])
-prefix += termcolor.colored(PROTO.rjust(4), PROTO_COLORS[PROTO], attrs=["dark"])
+PREFIX += termcolor.colored(PROTO.rjust(4), PROTO_COLOR, attrs=["dark"])
-prefix += termcolor.colored(",", "white", attrs=["bold"])
+PREFIX += termcolor.colored(",", "white", attrs=["bold"])
-prefix += termcolor.colored(VERS.ljust(4), COLOR, attrs=["bold"])
+PREFIX += termcolor.colored(VERS.ljust(4), COLOR, attrs=["bold"])
-prefix += termcolor.colored("] ", "white", attrs=["bold"])
+PREFIX += termcolor.colored("] ", "white", attrs=["bold"])
 
-CONFIG = [ ]
+
-def add_config(line):
+def add_config(line: str) -> None:
-    [ key, val ] = [ s.strip() for s in re.split("--", line) ]
+    """Format a configuration entry and add it to the CONFIG list."""
+    [key, val] = [s.strip() for s in re.split("--", line)]
     if key == "MKFS_OPTIONS":
         return
     elif key == "MOUNT_OPTIONS" and val[:2] != "-o":
         return
 
-    parts = [ prefix ]
+    parts = [PREFIX]
     parts.append(termcolor.colored(key.ljust(14), COLOR, attrs=["bold"]))
     parts.append(termcolor.colored("-- ", "white", attrs=["dark"]))
     parts.append(termcolor.colored(val, "white", attrs=["bold"]))
     CONFIG.append("".join(parts))
 
-def print_config():
+
+def print_config() -> None:
+    """Print out the collected configuration entries and clear the list."""
     if len(CONFIG) > 0:
         print("\n".join(CONFIG), end="\n\n")
         CONFIG.clear()
 
-def print_test(match):
+
-    parts = [ prefix ]
+def print_test(match: re.Match) -> None:
+    """Format and print a single testcase."""
+    parts = [PREFIX]
     parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"]))
 
-    if m := re.match("(\[not run\])(.*?\n)", match.group(2)):
+    if m := re.match(r"(\[not run\])(.*?\n)", match.group(2)):
         return
         parts.append(termcolor.colored(m.group(1), "yellow"))
         parts.append(termcolor.colored(m.group(2), "white", attrs=["bold"]))
-    elif m := re.match("(\d+s)( ... +)(\d+s\n)", match.group(2)):
+    elif m := re.match(r"(\d+s)( ... +)(\d+s\n)", match.group(2)):
         parts.append(termcolor.colored(m.group(1), "green"))
         parts.append(termcolor.colored(m.group(2), "white", attrs=["dark"]))
         parts.append(termcolor.colored(m.group(3), "green"))
-    elif re.match("\d+s\n", match.group(2)):
+    elif re.match(r"\d+s\n", match.group(2)):
         parts.append(termcolor.colored(match.group(2), "green"))
-    else: # Test failed
+    else:  # Test failed
-        line = re.sub("\]-", "\] -", match.group(2))
+        line = re.sub(r"\]-", r"\] -", match.group(2))
         parts.append(termcolor.colored(line, "magenta", attrs=["bold"]))
 
     print("".join(parts), end="")
 
-def print_diff(line):
+
-    print(prefix, end="")
+def print_diff(line: str) -> None:
-    if re.match(" +@@.*?@@\n", line):
+    """Format and print out a single line of a failing test diff."""
+    print(PREFIX, end="")
+    if re.match(r" +@@.*?@@\n", line):
         termcolor.cprint(line, color="yellow", attrs=["dark"], end="")
-    elif re.match(" +-", line):
+    elif re.match(r" +-", line):
         termcolor.cprint(line, color="magenta", attrs=["dark"], end="")
-    elif re.match(" +\+", line):
+    elif re.match(r" +\+", line):
         termcolor.cprint(line, color="cyan", attrs=["dark"], end="")
-    elif re.match(" +\(Run", line):
+    elif re.match(r" +\(Run", line):
         termcolor.cprint(line, color="magenta", attrs=["bold"], end="")
     else:
         termcolor.cprint(line, color="white", attrs=["dark"], end="")
 
 
+def print_test_list(status: str, line: str) -> None:
+    """Format and print the list of tests that ran, failed, or were skipped."""
+    tests = dict()
+    _tmp = [t.split("/") for t in line[len(status)+1:].split()]
+    [tests.setdefault(t[0], []).append(t[1]) for t in _tmp]
+
+    for key in sorted(tests.keys()):
+        parts = [PREFIX]
+        parts.append(termcolor.colored(status, STATUS_COLORS[status],
+                                       attrs=["bold"]))
+        parts.append(termcolor.colored(": ", "white", attrs=["bold"]))
+        parts.append(termcolor.colored(key, color=COLOR, attrs=["bold"]))
+        parts.append(termcolor.colored(": [", "white", attrs=["bold"]))
+        for i, test_num in enumerate(sorted(tests[key])):
+            sep = termcolor.colored(", ", attrs=["bold"] if i > 0 else "")
+            if i != 0:
+                parts.append(sep)
+            attr = ["bold", "dark"] if i % 2 else ["bold"]
+            parts.append(termcolor.colored(test_num, STATUS_COLORS[status],
+                                           attrs=attr))
+        parts.append(termcolor.colored("]", "white", attrs=["bold"]))
+        print("".join(parts))
+
+
+def print_failed(match: re.Match) -> None:
+    """Format and print the number of failing tests."""
+    parts = [PREFIX]
+    parts.append(termcolor.colored("Failed ", "red", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"]))
+    parts.append(termcolor.colored(" of ", "red", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(2), COLOR, attrs=["bold"]))
+    parts.append(termcolor.colored(" tests", "red", attrs=["bold"]))
+    print("".join(parts))
+
+
+def print_all_passed(match: re.Match) -> None:
+    """Format and print a status line if all tests pass."""
+    parts = [PREFIX]
+    parts.append(termcolor.colored("Passed all ", "green", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(1), COLOR, attrs=["bold"]))
+    parts.append(termcolor.colored(" tests", "green", attrs=["bold"]))
+    print("".join(parts))
+
+
+def print_xunit(match: re.Match) -> None:
+    """Format and print the xunit file location."""
+    parts = [PREFIX]
+    parts.append(termcolor.colored(match.group(1), PROTO_COLOR,
+                                   attrs=["dark", "bold"]))
+    parts.append(termcolor.colored(": ", "white", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(2), COLOR, attrs=["bold"]))
+    print("".join(parts))
+
+
+def print_xfstestsdb(match: re.Match) -> None:
+    """Format and print the xfstestsdb xunit-add line."""
+    parts = [PREFIX]
+    parts.append(termcolor.colored(match.group(1).title(), PROTO_COLOR,
+                                   attrs=["dark", "bold"]))
+    parts.append(termcolor.colored(" '", "white", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(2), COLOR,
+                                   attrs=["bold"]))
+    parts.append(termcolor.colored("' ", "white", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(3), PROTO_COLOR,
+                                   attrs=["dark", "bold"]))
+    parts.append(termcolor.colored(match.group(4), COLOR, attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(5), PROTO_COLOR,
+                                   attrs=["dark", "bold"]))
+    parts.append(termcolor.colored(" #", "white", attrs=["bold"]))
+    parts.append(termcolor.colored(match.group(6), COLOR,
+                                   attrs=["bold"]))
+    print("".join(parts))
+
+
 for line in sys.stdin:
     if line == "\n":
         print_config()
@@ -76,11 +162,20 @@ for line in sys.stdin:
             add_config(match.group(1))
             line = f"COMPILED -- {match.group(3)}"
         add_config(line)
-    elif match := re.match("(^[a-z]+/[\d]+[ |	]+)(.*?\n)", line):
+    elif match := re.match(r"(^[a-z]+/\d+[ |	]+)(.*?\n)", line):
         print_test(match)
     elif match := re.match("^    (.*?)\n", line):
         print_diff(line)
+    elif match := re.match(r"^(Ran|Not run|Failures):([ a-z]+/\d+)+\n", line):
+        print_test_list(match.group(1), line)
+    elif match := re.match(r"^Failed (\d+) of (\d+) tests\n", line):
+        print_failed(match)
+    elif match := re.match(r"^Passed all (\d+) tests\n", line):
+        print_all_passed(match)
+    elif match := re.match(r"(^Xunit report): (.*?.xml)\n", line):
+        print_xunit(match)
+    elif match := re.match(r"(^added) '(.*?)' (with )"
+                           r"(\d+)( testcases to run) #(\d+)\n", line):
+        print_xfstestsdb(match)
     else:
-        #pass
+        print(PREFIX + line, end="")
-        print(prefix, end="")
-        print(line, end="")

completions/_xfstests.zsh

@@ -8,9 +8,10 @@ function _xfstests.zsh() {
 		{-g,--group}'[the xfstests group to run]: :($(_list_xfstests_groups))' \
 		{-k,--scratch}'[the exported scratch directory on the server]: : _files -/' \
 		{-p,--mountpoint}'[the directory to mount the server]: : _files -/' \
-		--nconnect'[number of connections]: :($(seq 1 32))' \
+		--no-scratch'[do not use a scratch mount during testing]' \
+		--no-server-access'[do not attempt to ssh into the server]' \
+		\*{-o,--options}'[mount options for all mounts]: :' \
 		{-r,--rdma}'[test using rdma]: :(rxe siw)' \
-		--rwsize'[test with a secific rsize & wsize]: :($(seq 4096 4096 1048576))' \
 		{-q,--scratchmnt}'[the directory to mount the scratch export]: : _files -/' \
 		{-s,--server}'[the server to test against]: : _alternative
 			"hosts\:hosts\: _ssh_hosts"

run/xfstests.zsh

@@ -8,23 +8,31 @@ zparseopts -D -K -server:=SERVER -opts:=OPTS -user:=USER \
 xdir=$(echo "nfsv${VERSION[-1]}${PROTO[-1]}" | sed 's/\.//' | sed 's/v40/v4/')
 
 cd /home/${USER[-1]}/xfstests-dev
-export TEST_DEV=${SERVER[-1]}:${EXPORT[-1]}/${USER[-1]}/$xdir
+export TEST_DEV=$(echo ${SERVER[-1]}:${EXPORT[-1]}/${USER[-1]}/$xdir | sed 's|//|/|')
 export TEST_DIR=${MOUNTPOINT[-1]}/$xdir
-export SCRATCH_DEV=${SERVER[-1]}:${SCRATCH[-1]}/${USER[-1]}/$xdir
+if [ "${SCRATCH[-1]}" != "NONE" ]; then
-export SCRATCH_MNT=${SCRATCHMNT[-1]}/$xdir
+	export SCRATCH_DEV=${SERVER[-1]}:${SCRATCH[-1]}/${USER[-1]}/$xdir
+	export SCRATCH_MNT=${SCRATCHMNT[-1]}/$xdir
+fi
 export NFS_MOUNT_OPTIONS="-o proto=${PROTO[-1]},v${VERSION[-1]},${OPTS[-1]}"
 export RESULT_BASE=/home/${USER[-1]}/xfstests-dev/results/${PROTO[-1]}/${VERSION[-1]}
 
 mkdir -p $TEST_DIR
-mkdir -p $SCRATCH_MNT
+if [ "${SCRATCH[-1]}" != "NONE" ]; then
+	mkdir -p $SCRATCH_MNT
+fi
 [ -e local.config ] && rm -f local.config
 [ -e $RESULT_BASE/result.xml ] && rm -f $RESULT_BASE/result.xml
 
 echo "TIME          -- $(date)"
 echo "TEST_DEV      -- $TEST_DEV"
 echo "TEST_DIR      -- $TEST_DIR"
-echo "SCRATCH_DEV   -- $SCRATCH_DEV"
+if [ "${SCRATCH[-1]}" != "NONE" ]; then
-echo "SCRATCH_MNT   -- $SCRATCH_MNT"
+	echo "SCRATCH_DEV   -- $SCRATCH_DEV"
+	echo "SCRATCH_MNT   -- $SCRATCH_MNT"
+else
+	echo "SCRATCH_DEV   -- NONE"
+fi
 echo "MOUNT_OPTIONS -- $NFS_MOUNT_OPTIONS"
 
 if [ ! -z "$*" ]; then
@@ -32,4 +40,3 @@ if [ ! -z "$*" ]; then
 fi
 
 ./check -nfs -r -R xunit $*
-#./check -nfs -R xunit $*

setup/xfstests.zsh

@@ -17,4 +17,6 @@ useradd --badnames 123456-fsgqa 2>/dev/null
 useradd fsgqa2 2>/dev/null
 
 setup.zsh testdirs.zsh $SERVER $EXPORT $MOUNTPOINT $USER
-setup.zsh testdirs.zsh $SERVER $SCRATCH $SCRATCHMNT $USER
+if [ "$SCRATCH" != "NONE" ]; then
+	setup.zsh testdirs.zsh $SERVER $SCRATCH $SCRATCHMNT $USER
+fi

xfstests.zsh

@@ -8,17 +8,18 @@ zparseopts -D -K \
 	c:=CLIENT -client:=CLIENT \
 	g:=GROUP -group:=GROUP \
 	k:=SCRATCH -scratch:=SCRATCH \
-	-nconnect:=NCONNECT \
+	-no-scratch=NO_SCRATCH \
+	-no-server-access=NO_SERVER_ACCESS \
+	o+:=MNTOPTS -options+:=MNTOPTS \
 	p:=MOUNTPOINT -mountpoint:=MOUNTPOINT \
 	q:=SCRATCHMNT -scratchmnt:=SCRATCHMNT \
 	r:=RDMA -rdma:=RDMA \
-	-rwsize:=RWSIZE \
 	s:=SERVER -server:=SERVER \
 	x:=EXPORT -export:=EXPORT \
 	v+:=VERSION -version+:=VERSION
 
-COLOR=$BIN/colors/xfstests.py
+COLOR=$COLORS/xfstests.py
-OPTIONS="sec=sys"
+OPTIONS="sec=sys$(echo $MNTOPTS | sed -E 's/ ?-o /,/g')"
 RESULT_BASE=$HOME/.local/share/xfstests/
 DATE=$(date +%Y-%m-%d-%H:%M:%S%z)
 TODAY=$(date +%Y/%m/%d)
@@ -40,12 +41,11 @@ function setup_client_func()
 				  ${MOUNTPOINT[-1]} ${SCRATCH[-1]} \
 				  ${SCRATCHMNT[-1]} $USER
 }
+
 prepare_to_test
-if [ ${#NCONNECT} -gt 0 ]; then
+
-	OPTIONS="$OPTIONS,nconnect=${NCONNECT[-1]}"
+if [ ${#NO_SCRATCH} -gt 0 ]; then
-fi
+	SCRATCH+=("NONE")
-if [ ${#RWSIZE} -gt 0 ]; then
-	OPTIONS="$OPTIONS,rsize=${RWSIZE[-1]},wsize=${RWSIZE[-1]}"
 fi
 
 testargs=()
@@ -55,28 +55,36 @@ for arg in $*; do
 done
 
 run_xfs_tests() {
-	client_run xfstests.zsh --server ${SERVER[-1]} \
+	for proto in $PROTO; do
-				--export ${EXPORT[-1]} \
+		client_run xfstests.zsh --server ${SERVER[-1]} \
-				--mountpoint ${MOUNTPOINT[-1]} \
+					--export ${EXPORT[-1]} \
-				--scratch ${SCRATCH[-1]} \
+					--mountpoint ${MOUNTPOINT[-1]} \
-				--scratchmnt ${SCRATCHMNT[-1]} \
+					--scratch ${SCRATCH[-1]} \
-				--proto $1 \
+					--scratchmnt ${SCRATCHMNT[-1]} \
-				--version $2 \
+					--proto $proto --version $1 \
-				--opts $OPTIONS \
+					--opts $OPTIONS \
-				--user $USER $GROUP $testargs | python $COLOR $1 $2
+					--user $USER $GROUP $testargs | python $COLOR $proto $1
-	scp -q ${CLIENT[-1]}:$REMOTE_RESULTS/$1/$2/result.xml $TMP_RESULTS/$1-$2.xml
+
+		if [[ "$proto" == "rdma" ]]; then
+			xunit="$TMP_RESULTS/${RDMA[-1]}-$1.xml"
+		else
+			xunit="$TMP_RESULTS/$proto-$1.xml"
+		fi
+
+		scp -q ${CLIENT[-1]}:$REMOTE_RESULTS/$proto/$1/result.xml $xunit
+		xfstestsdb xunit read $RUNID $xunit | python $COLOR $proto $1
+	done
 }
 
+
 #
 # Run tests
 #
 mkdir -p $TMP_RESULTS
-for proto in $PROTO; do
+RUNID=$(xfstestsdb new ${SERVER[-1]}:${EXPORT[-1]} | awk -F" | #" '{print $3}')
-	for vers in $VERSION; do
+for vers in $VERSION; do
-		run_xfs_tests $proto $vers &
+	run_xfs_tests $vers &
-	done
 done
-
 wait
 
 mkdir -p $RESULTS
@@ -84,5 +92,12 @@ tar -cJf $RESULTS/$NOW.tar.xz -C $(dirname $TMP_RESULTS) $NOW/
 if [ ! -z "$TAG" ]; then
 	mkdir -p $TAGRES
 	ln $RESULTS/$NOW.tar.xz $TAGRES/$DATE.tar.xz
+	xfstestsdb tag $RUNID $TAG
 fi
+
+echo
+xfstestsdb xunit list --results --runid $RUNID
+echo
+xfstestsdb show --failure $RUNID
+echo
 report-xfstests.py $RESULTS/$NOW.tar.xz & disown