From 5545cb106d373676fef2c2f33f1fec1aa708b6c0 Mon Sep 17 00:00:00 2001 From: Anna Schumaker Date: Fri, 10 Mar 2023 17:03:19 -0500 Subject: [PATCH] db: Give Libraries an idle Tagger This uses a combination of an Idle Queue, ReaddirThread, and tagger Thread to scan a directory path and tag the audio files found within. I do this by adding a scan() function to each Library object to begin scanning (if not already running). Libraries also have a stop() function to cancel any pending idle tasks and stop any running threads. The Library table makes sure to stop each Library object during shutdown so we don't leave any hanging threads. Signed-off-by: Anna Schumaker --- emmental/db/libraries.py | 56 ++++++++++++++++ tests/db/test_libraries.py | 129 ++++++++++++++++++++++++++++++++++++- 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/emmental/db/libraries.py b/emmental/db/libraries.py index f9bfb7a..e9470dd 100644 --- a/emmental/db/libraries.py +++ b/emmental/db/libraries.py @@ -3,8 +3,10 @@ import pathlib import sqlite3 from gi.repository import GObject +from .. import path from . import idle from . import playlist +from . import tagger class Library(playlist.Playlist): @@ -16,19 +18,55 @@ class Library(playlist.Playlist): deleting = GObject.Property(type=bool, default=False) queue = GObject.Property(type=idle.Queue) + readdir = GObject.Property(type=GObject.TYPE_PYOBJECT) + tagger = GObject.Property(type=GObject.TYPE_PYOBJECT) online = GObject.Property(type=bool, default=False) def __init__(self, **kwargs): """Initialize our Library object.""" super().__init__(queue=idle.Queue(), **kwargs) + self.scan() def __queue_delete(self) -> bool: self.table.delete(self) return True + def __queue_tracks(self) -> bool: + if (files := self.readdir.poll_result()) is None: + self.__stop_thread("readdir") + self.queue.push(self.__stop_thread, "tagger") + return True + + self.queue.push_many(self.__tag_track, [(f,) for f in files]) + return False + + def __tag_track(self, path: pathlib.Path) -> bool: + if self.tagger.ready.is_set(): + (file, tags) = self.tagger.get_result(self.table.sql) + if file is None: + self.tagger.tag_file(path) + else: + return True + return False + + def __scan_library(self) -> bool: + self.readdir = path.readdir_async(self.path) + if self.readdir is not None: + self.online = True + self.queue.push(self.__queue_tracks) + self.tagger = tagger.Thread() + return True + + def __stop_thread(self, thread_name: str) -> bool: + if (thread := self.get_property(thread_name)) is not None: + thread.stop() + self.set_property(thread_name, None) + return True + def do_update(self, column: str) -> bool: """Update a Library playlist.""" match column: + case "readdir" | "tagger": pass case "online": self.table.notify_online(self) case _: return super().do_update(column) return True @@ -36,11 +74,23 @@ class Library(playlist.Playlist): def delete(self) -> bool: """Delete this Library.""" if self.deleting is False: + self.stop() self.deleting = True self.queue.push(self.__queue_delete) return True return False + def scan(self) -> None: + """Scan the Library.""" + if not self.queue.running: + self.queue.push(self.__scan_library) + + def stop(self) -> None: + """Stop this Library's background work.""" + self.__stop_thread("readdir") + self.__stop_thread("tagger") + self.queue.cancel() + @property def primary_key(self) -> int: """Get this library's primary key.""" @@ -88,6 +138,12 @@ class Table(playlist.Table): if not library.online or self.loaded: self.emit("library-online", library) + def stop(self) -> None: + """Stop any background work.""" + for library in self.store: + library.stop() + super().stop() + @GObject.Signal(arg_types=(Library,)) def library_online(self, library: Library) -> None: """Signal that a library online status has changed.""" diff --git a/tests/db/test_libraries.py b/tests/db/test_libraries.py index 0e36b29..6f284ea 100644 --- a/tests/db/test_libraries.py +++ b/tests/db/test_libraries.py @@ -20,10 +20,17 @@ class TestLibraryObject(tests.util.TestCase): path=self.path, name=str(self.path)) + def tearDown(self): + """Clean up.""" + super().tearDown() + self.library.stop() + def test_init(self): """Test that the Library is set up properly.""" self.assertIsInstance(self.library, emmental.db.playlist.Playlist) self.assertIsInstance(self.library.queue, emmental.db.idle.Queue) + self.assertIsNone(self.library.readdir) + self.assertIsNone(self.library.tagger) self.assertEqual(self.library.table, self.table) self.assertEqual(self.library.propertyid, 456) @@ -32,9 +39,12 @@ class TestLibraryObject(tests.util.TestCase): self.assertEqual(self.library.path, self.path) self.assertTrue(self.library.enabled) self.assertFalse(self.library.deleting) - self.assertFalse(self.library.online) self.assertIsNone(self.library.parent) + self.assertFalse(self.library.online) + self.assertEqual(self.library.queue[0], + (self.library._Library__scan_library,)) + def test_delete(self): """Test deleting a Library path.""" with unittest.mock.patch.object(self.table, "delete") as mock_delete: @@ -58,6 +68,114 @@ class TestLibraryObject(tests.util.TestCase): self.library.online = True mock_notify.assert_called_with(self.library) + @unittest.mock.patch.object(pathlib.Path, "iterdir", return_value=[]) + @unittest.mock.patch.object(pathlib.Path, "is_dir") + def test_scan_online(self, mock_is_dir: unittest.mock.Mock, + mock_iterdir: unittest.mock.Mock): + """Test scanning an online Library path.""" + self.assertEqual(self.library.queue[0], + (self.library._Library__scan_library,)) + + with unittest.mock.patch.object(self.library.queue, + "push") as mock_push: + self.library.scan() + mock_push.assert_not_called() + + mock_is_dir.return_value = True + self.assertTrue(self.library.queue.run_task()) + + mock_is_dir.assert_called() + self.assertTrue(self.library.online) + self.assertTrue(self.library.queue.running) + self.assertIsInstance(self.library.readdir, + emmental.path.ReaddirThread) + self.assertIsInstance(self.library.tagger, + emmental.db.tagger.Thread) + self.assertEqual(self.library.queue[0], + (self.library._Library__queue_tracks,)) + + def test_scan_offline(self): + """Test scanning an offline Library path.""" + self.assertEqual(self.library.queue[0], + (self.library._Library__scan_library,)) + + self.assertFalse(self.library.queue.run_task()) + self.assertFalse(self.library.queue.running) + self.assertFalse(self.library.online) + self.assertIsNone(self.library.readdir) + + @unittest.mock.patch.object(pathlib.Path, "iterdir", return_value=[]) + @unittest.mock.patch.object(pathlib.Path, "is_file", return_value=True) + def test_scan_queue_tracks(self, mock_is_file: unittest.mock.Mock, + mock_iterdir: unittest.mock.Mock): + """Test that tracks are queued for scanning.""" + self.library.stop() + self.library.readdir = unittest.mock.Mock() + readdir = self.library.readdir + + readdir.poll_result.return_value = [] + self.assertFalse(self.library._Library__queue_tracks()) + readdir.poll_result.assert_called() + readdir.stop.assert_not_called() + + tag_func = self.library._Library__tag_track + readdir.poll_result.return_value = [pathlib.Path("/a/b/c/1.ogg"), + pathlib.Path("/a/b/c/2.ogg")] + self.assertFalse(self.library._Library__queue_tracks()) + self.assertListEqual(self.library.queue._tasks, + [(tag_func, pathlib.Path("/a/b/c/1.ogg")), + (tag_func, pathlib.Path("/a/b/c/2.ogg"))]) + + readdir.poll_result.return_value = None + self.assertTrue(self.library._Library__queue_tracks()) + self.assertListEqual(self.library.queue._tasks, + [(tag_func, pathlib.Path("/a/b/c/1.ogg")), + (tag_func, pathlib.Path("/a/b/c/2.ogg")), + (self.library._Library__stop_thread, "tagger")]) + readdir.stop.assert_called() + + def test_scan_tag_track(self): + """Test that tracks are tagged during scanning.""" + track = pathlib.Path("/a/b/c/1.ogg") + raw_tags = emmental.audio.tagger._Tags(track, {}) + tags = emmental.db.tagger.Tags(self.sql, raw_tags) + tagger = unittest.mock.Mock() + self.library.tagger = tagger + + tagger.ready.is_set.return_value = False + self.assertFalse(self.library._Library__tag_track(track)) + tagger.get_result.assert_not_called() + tagger.tag_file.assert_not_called() + + tagger.ready.is_set.return_value = True + tagger.get_result.return_value = (None, None) + self.assertFalse(self.library._Library__tag_track(track)) + tagger.get_result.assert_called_with(self.sql) + tagger.tag_file.assert_called_with(track) + + tagger.reset_mock() + tagger.ready.is_set.return_value = True + tagger.get_result.return_value = (track, tags) + self.assertTrue(self.library._Library__tag_track(track)) + tagger.tag_file.assert_not_called() + tagger.get_result.assert_called_with(self.sql) + + def test_stop(self): + """Test stopping a Library's background work.""" + readdir = unittest.mock.Mock() + tagger = unittest.mock.Mock() + self.library.readdir = readdir + self.library.tagger = tagger + self.library.queue.running = True + + self.library.stop() + self.assertIsNone(self.library.readdir) + self.assertIsNone(self.library.tagger) + self.assertFalse(self.library.queue.running) + + readdir.stop.assert_called() + tagger.stop.assert_called() + class TestLibraryTable(tests.util.TestCase): """Tests our library table.""" @@ -154,6 +272,15 @@ class TestLibraryTable(tests.util.TestCase): self.assertEqual(self.table.lookup(pathlib.Path("/a/b/c/")), library) self.assertIsNone(self.table.lookup(pathlib.Path("/no/library/path"))) + def test_stop(self): + """Test stopping the library table.""" + library = self.table.create(pathlib.Path("/a/b/c")) + library.queue.running = True + + self.table.stop() + self.assertFalse(self.table.queue.running) + self.assertFalse(library.queue.running) + def test_update(self): """Test updating genre attributes.""" library = self.table.create("/a/b/c")