anna/xfstestsdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

xfstestsdb: Add testcases from the xunit file to the database

Browse Source 
I use the testcases table to store testcase information, and store
messages, system out, and system error logs in the messages table.

The testcases_view is used to link testcases with their source xunits,
messages, system out, and system error logs. This view has an INSTEAD
OF INSERT trigger to allow insertions that properly set up message ids
in the testcases table.

Signed-off-by: Anna Schumaker <anna@nowheycreamery.com>
This commit is contained in:
Anna Schumaker 2023-02-08 10:45:17 -05:00
parent 84a7507998
commit 3366c1eb0c
4 changed files with 184 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tests/xunit/test_delete.py
View File

@ -42,6 +42,27 @@ class TestXunitDelete(unittest.TestCase):
cur = self.xfstestsdb.sql("SELECT COUNT(*) FROM xunit_properties")
self.assertEqual(cur.fetchone()["COUNT(*)"], 0)
cur = self.xfstestsdb.sql("SELECT COUNT(*) FROM testcases")
self.assertEqual(cur.fetchone()["COUNT(*)"], 0)
cur = self.xfstestsdb.sql("SELECT COUNT(*) FROM messages")
self.assertEqual(cur.fetchone()["COUNT(*)"], 0)
@unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
def test_delete_testcase(self, mock_stdout: io.StringIO):
"""Test that messages get cleaned up when a testcase is deleted."""
self.xfstestsdb.run(["new", "/dev/test"])
self.xfstestsdb.run(["xunit", "read", "1", str(tests.xunit.XUNIT_1)])
self.xfstestsdb.sql("DELETE FROM testcases WHERE status=?", "skipped")
cur = self.xfstestsdb.sql("SELECT * FROM messages ORDER BY rowid")
self.assertListEqual([row["message"] for row in cur.fetchall()],
["- output mismatch (see somefile)",
"there was a problem with '$SCRATCH_DEV'",
"--- test/09.out\n" +
"+++ results/test/09.out.bad\n" +
"there was a problem with '$SCRATCH_MNT'"])
@unittest.mock.patch("sys.stderr", new_callable=io.StringIO)
def test_delete_error(self, mock_stderr: io.StringIO):
"""Test the `xfstestsdb xunit delete` command with invalid input."""

50
tests/xunit/test_read.py
View File

@ -98,6 +98,56 @@ class TestXunitRead(unittest.TestCase):
"OVL_LOWER": "ovl-lower",
"OVL_WORK": "ovl-work"})
@unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
def test_read_file_testcases(self, mock_stdout: io.StringIO):
"""Test reading a file's testcases into the database."""
self.xfstestsdb.run(["new", "/dev/vda1"])
self.xfstestsdb.run(["xunit", "read", "1", str(tests.xunit.XUNIT_1)])
cur = self.xfstestsdb.sql("SELECT * FROM testcases ORDER BY testcase")
self.assertListEqual([[c for c in row] for row in cur.fetchall()],
[[1, "test/01", "passed", 1, None, None, None],
[1, "test/02", "skipped", 0, 1, None, None],
[1, "test/03", "skipped", 0, 2, None, None],
[1, "test/04", "passed", 4, None, None, None],
[1, "test/05", "passed", 5, None, None, None],
[1, "test/06", "passed", 6, None, None, None],
[1, "test/07", "skipped", 0, 3, None, None],
[1, "test/08", "passed", 8, None, None, None],
[1, "test/09", "failure", 9, 4, 5, 6],
[1, "test/10", "passed", 10, None, None, None]])
@unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
def test_read_file_testcases_view(self, mock_stdout: io.StringIO):
"""Test reading a file's testcases into the database."""
self.xfstestsdb.run(["new", "/dev/vda1"])
self.xfstestsdb.run(["xunit", "read", "1", str(tests.xunit.XUNIT_1)])
rows = self.xfstestsdb.sql("""SELECT * FROM testcases_view
ORDER BY testcase""").fetchall()
for row in rows:
self.assertTupleEqual(row[:4], (1, 1, "/dev/vda1", "test-1"))
self.assertListEqual([row[4:] for row in rows],
[("test/01", "passed", 1, None, None, None),
("test/02", "skipped", 0,
"skipped on $TEST_DEV", None, None),
("test/03", "skipped", 0,
"skipped on $TEST_DIR too", None, None),
("test/04", "passed", 4, None, None, None),
("test/05", "passed", 5, None, None, None),
("test/06", "passed", 6, None, None, None),
("test/07", "skipped", 0,
"fstype \"myfs\" gets skipped", None, None),
("test/08", "passed", 8, None, None, None),
("test/09", "failure", 9,
"- output mismatch (see somefile)",
"there was a problem with '$SCRATCH_DEV'",
"--- test/09.out\n" +
"+++ results/test/09.out.bad\n" +
"there was a problem with '$SCRATCH_MNT'"),
("test/10", "passed", 10, None, None, None)])
@unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
def test_read_file_named(self, mock_stdout: io.StringIO):
"""Test reading a file with the --name= option."""

68
xfstestsdb/xfstestsdb.sql
View File

@ -119,3 +119,71 @@ CREATE TRIGGER cleanup_xunit_properties
(SELECT 1 FROM link_xunit_props
WHERE propid = xunit_properties.rowid);
END;
/***********************************************
* *
* Table for managing test cases *
* *
***********************************************/
CREATE TABLE messages (
messageid INTEGER PRIMARY KEY,
message TEXT NOT NULL,
UNIQUE (message)
);
CREATE TABLE testcases (
xunitid INTEGER NOT NULL,
testcase TEXT NOT NULL,
status TEXT NOT NULL,
time INTEGER NOT NULL,
messageid INTEGER DEFAULT NULL,
stdoutid INTEGER DEFAULT NULL,
stderrid INTEGER DEFAULT NULL,
PRIMARY KEY (xunitid, testcase),
FOREIGN KEY (xunitid) REFERENCES xunits (xunitid)
ON DELETE CASCADE
ON UPDATE CASCADE,
FOREIGN KEY (messageid) REFERENCES messages (messageid),
FOREIGN KEY (stdoutid) REFERENCES messages (messageid),
FOREIGN KEY (stderrid) REFERENCES messages (messageid),
CHECK (status IN ("passed", "failure", "skipped"))
);
CREATE VIEW testcases_view AS
SELECT runid, xunitid, device, xunits.name as xunit, testcase, status,
testcases.time, msg.message as message,
stdout.message as stdout, stderr.message as stderr
FROM testcases
JOIN xunits USING (xunitid)
JOIN xfstests_runs USING (runid)
LEFT JOIN messages msg USING (messageid)
LEFT JOIN messages stdout ON stdout.messageid = testcases.stdoutid
LEFT JOIN messages stderr ON stderr.messageid = testcases.stderrid;
CREATE TRIGGER insert_testcase
INSTEAD OF INSERT ON testcases_view
BEGIN
INSERT OR IGNORE INTO messages (message) VALUES (NEW.message);
INSERT OR IGNORE INTO messages (message) VALUES (NEW.stdout);
INSERT OR IGNORE INTO messages (message) VALUES (NEW.stderr);
INSERT INTO testcases (xunitid, testcase, status, time,
messageid, stdoutid, stderrid)
VALUES ((SELECT xunitid FROM xunits
WHERE runid = NEW.runid AND name = NEW.xunit),
NEW.testcase, NEW.status, NEW.time,
(SELECT messageid FROM messages WHERE message = NEW.message),
(SELECT messageid FROM messages WHERE message = NEW.stdout),
(SELECT messageid FROM messages WHERE message = NEW.stderr));
END;
CREATE TRIGGER cleanup_testcase_messages
AFTER DELETE ON testcases
BEGIN
DELETE FROM messages WHERE NOT EXISTS
(SELECT 1 FROM testcases WHERE
messageid = messages.rowid
OR stdoutid = messages.rowid
OR stderrid = messages.rowid);
END;

45
xfstestsdb/xunit/read.py
View File

@ -4,7 +4,9 @@ import argparse
import datetime
import dateutil.tz
import errno
import html
import pathlib
import re
import sys
import xml.etree.ElementTree
from .. import commands
@ -27,6 +29,41 @@ class Command(commands.Command):
type=argparse.FileType('r'),
help="path to the xunit file to be read")
def sanitize_message(self, text: str, properties: dict) -> str | None:
"""Replace text with xunit property names to reduce duplication."""
if text is not None:
for key in ["TEST_DEV", "TEST_DIR", "SCRATCH_DEV", "SCRATCH_MNT"]:
if (value := properties.get(key)):
text = re.sub(value, f"${key}", text)
return text.strip()
def elm_testcase(self, runid: int, xunit: str,
elm: xml.etree.ElementTree.Element,
properties: dict) -> tuple:
"""Extract testcase data for an xml element."""
status = "passed"
message = None
stdout = None
stderr = None
for e in elm:
match e.tag:
case "failure" | "skipped": status = e.tag
case "system-out": stdout = html.unescape(e.text)
case "system-err":
stderr = re.sub(r"([-+]{3}\s\S+)(\s.*\n)",
lambda m: f"{m.group(1)}\n",
html.unescape(e.text))
stderr = re.sub(r"[+]{3} /?(.*)/(\S+/\d+.*)\n",
lambda m: f"+++ results/{m.group(2)}\n",
stderr)
message = elm[0].attrib["message"]
return (runid, xunit, elm.attrib["name"], status, elm.attrib["time"],
self.sanitize_message(message, properties),
self.sanitize_message(stdout, properties),
self.sanitize_message(stderr, properties))
def do_command(self, args: argparse.Namespace) -> None:
"""Read in an xunit file."""
if self.sql("SELECT rowid FROM xfstests_runs WHERE rowid=?",
@ -40,6 +77,9 @@ class Command(commands.Command):
root = xml.etree.ElementTree.parse(args.file[0]).getroot()
properties = {e.attrib["name"]: e.attrib["value"] for e in root[0]}
testcases = [self.elm_testcase(args.runid[0], xunitname,
elm, properties)
for elm in root[1:]]
timestamp = datetime.datetime.fromisoformat(root.attrib["timestamp"])
timestamp = timestamp.astimezone(dateutil.tz.tzutc())
@ -63,5 +103,10 @@ class Command(commands.Command):
*[(args.runid[0], xunitname, key, value)
for (key, value) in properties.items()])
self.sql.executemany("""INSERT INTO testcases_view
(runid, xunit, testcase, status,
time, message, stdout, stderr)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", *testcases)
print(f"added '{xunitname}' with {root.attrib['tests']} testcases "
f"to run #{args.runid[0]}")