gc: Create the `xfstestsdb gc` command

This command is used to garbage collect the xfstestsdb sqlite database.
It removes xfstests runs that have no added xunits, and runs older than
180 days that have not been tagged.

Implements: #16 (`xfstestsdb gc`)
Signed-off-by: Anna Schumaker <anna@nowheycreamery.com>

tests/test_gc.py

@@ -0,0 +1,123 @@
+# Copyright 2023 (c) Anna Schumaker.
+"""Tests the `xfstestsdb gc` command."""
+import datetime
+import io
+import unittest
+import unittest.mock
+import xfstestsdb.xunit.gc
+import tests.xunit
+
+
+class TestGC(unittest.TestCase):
+    """Tests the `xfstestsdb xunit gc` command."""
+
+    def setUp(self):
+        """Set up common variables."""
+        self.xfstestsdb = xfstestsdb.Command()
+        self.xunit = self.xfstestsdb.commands["xunit"]
+        self.gc = self.xunit.commands["gc"]
+
+    def setup_runs(self, mock_stdout: io.StringIO):
+        """Set up runs in the database and clear stdout."""
+        self.xfstestsdb.run(["new", "/dev/vda1"])
+        self.xfstestsdb.run(["new", "/dev/vda1"])
+        self.xfstestsdb.run(["new", "/dev/vda1"])
+
+        self.xfstestsdb.run(["xunit", "read", "1", str(tests.xunit.XUNIT_1)])
+        self.xfstestsdb.run(["xunit", "read", "3", str(tests.xunit.XUNIT_1)])
+
+        self.xfstestsdb.sql("DELETE FROM testcases WHERE xunitid=2")
+
+        mock_stdout.seek(0)
+        mock_stdout.truncate(0)
+
+    def test_init(self):
+        """Check that the gc command was set up properly."""
+        self.assertIsInstance(self.gc, xfstestsdb.commands.Command)
+        self.assertIsInstance(self.gc, xfstestsdb.xunit.gc.Command)
+        self.assertEqual(self.xunit.subparser.choices["gc"],
+                         self.gc.parser)
+
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def test_gc_empty(self, mock_stdout: io.StringIO):
+        """Test garbage collecting an empty database."""
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_gc_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [])
+
+        self.xfstestsdb.run(["gc"])
+        self.assertEqual(mock_stdout.getvalue(), "")
+
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def test_gc_no_testcases(self, mock_stdout: io.StringIO):
+        """Test garbage collecting runs with no testcases."""
+        self.setup_runs(mock_stdout)
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_gc_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [2, 3])
+
+        self.xfstestsdb.run(["gc"])
+        self.assertRegex(mock_stdout.getvalue(),
+                         "run #2 has been deleted\nrun #3 has been deleted")
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [1])
+
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def test_gc_expired(self, mock_stdout: io.StringIO):
+        """Test garbage collecting old runs."""
+        self.setup_runs(mock_stdout)
+        self.xfstestsdb.run(["xunit", "read", "2", str(tests.xunit.XUNIT_1)])
+
+        utcnow = datetime.datetime.utcnow()
+
+        expired = utcnow - datetime.timedelta(days=180, seconds=1)
+        self.xfstestsdb.sql("""UPDATE xfstests_runs SET timestamp=?
+                            WHERE runid=1""", expired)
+
+        not_expired = utcnow - datetime.timedelta(days=180, seconds=-1)
+        self.xfstestsdb.sql("""UPDATE xfstests_runs SET timestamp=?
+                            WHERE runid=2""", not_expired)
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_gc_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [1, 3])
+
+        self.xfstestsdb.run(["gc"])
+        self.assertRegex(mock_stdout.getvalue(),
+                         "run #1 has been deleted\nrun #3 has been deleted")
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [2])
+
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def test_gc_expired_tagged(self, mock_stdout: io.StringIO):
+        """Test that we don't garbage collect expired runs that are tagged."""
+        self.setup_runs(mock_stdout)
+        self.xfstestsdb.run(["xunit", "read", "2", str(tests.xunit.XUNIT_1)])
+        self.xfstestsdb.run(["tag", "1", "my-tag"])
+        self.xfstestsdb.run(["tag", "3", "my-tag"])
+
+        utcnow = datetime.datetime.utcnow()
+        expired = utcnow - datetime.timedelta(days=181)
+        self.xfstestsdb.sql("UPDATE xfstests_runs SET timestamp=?", expired)
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_gc_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [2, 3])
+
+        self.xfstestsdb.run(["gc"])
+        self.assertRegex(mock_stdout.getvalue(),
+                         "run #2 has been deleted\nrun #3 has been deleted")
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()], [1])
+
+    @unittest.mock.patch("sys.stdout", new_callable=io.StringIO)
+    def test_gc_dry_run(self, mock_stdout: io.StringIO):
+        """Test garbage collecting with the --dry-run option."""
+        self.setup_runs(mock_stdout)
+        self.xfstestsdb.run(["gc", "--dry-run"])
+        self.assertRegex(mock_stdout.getvalue(),
+                         "run #2 would be deleted\nrun #3 would be deleted")
+
+        cur = self.xfstestsdb.sql("SELECT runid FROM xfstests_runs")
+        self.assertListEqual([row['runid'] for row in cur.fetchall()],
+                             [1, 2, 3])

xfstestsdb/__init__.py

@@ -3,6 +3,7 @@
 import argparse
 from . import sqlite
 from . import delete
+from . import gc
 from . import list
 from . import new
 from . import rename
@@ -28,6 +29,7 @@ class Command:
         self.subparser = self.parser.add_subparsers(title="commands")
         self.sql = sqlite.Connection()
         self.commands = {"delete": delete.Command(self.subparser, self.sql),
+                         "gc": gc.Command(self.subparser, self.sql),
                          "list": list.Command(self.subparser, self.sql),
                          "new": new.Command(self.subparser, self.sql),
                          "rename": rename.Command(self.subparser, self.sql),

xfstestsdb/gc.py

@@ -0,0 +1,27 @@
+# Copyright 2023 (c) Anna Schumaker.
+"""The `xfstestsdb gc` command."""
+import argparse
+from . import commands
+from . import sqlite
+
+
+class Command(commands.Command):
+    """The `xfstestsdb gc` command."""
+
+    def __init__(self, subparser: argparse.Action,
+                 sql: sqlite.Connection) -> None:
+        """Set up the gc command."""
+        super().__init__(subparser, sql, "gc",
+                         help="garbage collect xfstestsdb entries")
+        self.parser.add_argument("--dry-run", action="store_true",
+                                 help="do not remove the xunit entries")
+
+    def do_command(self, args: argparse.Namespace) -> None:
+        """Clean up the xunit table."""
+        how = "would be" if args.dry_run else "has been"
+        rows = self.sql("SELECT runid FROM xfstests_gc_runs").fetchall()
+
+        for runid in sorted([row['runid'] for row in rows]):
+            if not args.dry_run:
+                self.sql("DELETE FROM xfstests_runs WHERE runid=?", runid)
+            print(f"run #{runid} {how} deleted")

xfstestsdb/scripts/upgrade-v2.sql

@@ -48,3 +48,17 @@ CREATE TRIGGER cleanup_testcase_messages
 					OR stdoutid = messages.messageid
 					OR stderrid = messages.messageid);
 	END;
+
+
+/*
+ * Create a view on the xfstestsdb_runs to find garbage collection candidates.
+ */
+CREATE VIEW xfstests_gc_runs AS
+	SELECT runid
+	FROM xfstests_runs
+	WHERE NOT EXISTS (SELECT 1 FROM testcases
+			  JOIN xunits USING (xunitid)
+			  WHERE runid = xfstests_runs.runid)
+		OR (timestamp < datetime('now', '-180 days')
+			AND NOT EXISTS (SELECT 1 FROM tags
+					WHERE runid = xfstests_runs.runid));