curds: Switch over to the new track database system

It is way faster during scanning due to the db key changes, which I
wasn't expecting at the time.

Signed-off-by: Anna Schumaker <Anna@NoWheyCreamery.com>
This commit is contained in:
Anna Schumaker 2019-12-09 14:05:41 -05:00
parent 0db7058836
commit 1fc448996e
25 changed files with 64 additions and 206 deletions

View File

@ -6,7 +6,7 @@ from . import threadqueue
from . import trak
import os
trak.map.load()
trak.load()
playlist.load()
if os.environ.get("EMMENTAL_TESTING"):
playlist.reset()
@ -19,7 +19,7 @@ Track = trak.track.Track
def reset():
notify.clear()
trak.map.clear()
trak.reset()
playlist.reset()
def stop():

View File

@ -18,10 +18,9 @@ class LibraryPlaylist(playlist.Playlist):
return key.strip("/").split("/")
def thread_add(self, path):
self.add(trak.track.lookup(path))
self.add(trak.lookup(path))
def thread_save(self):
trak.map.save()
notify.notify("save-data")
def thread_scan(self):

View File

@ -26,7 +26,7 @@ class Playlist(node.PlaylistNode):
def __getstate__(self):
with self.lock:
state = node.PlaylistNode.__getstate__(self)
state["list"] = [ t.hash() for t in self.list ]
state["list"] = [ t.path for t in self.list ]
return state
def __len__(self):
@ -35,7 +35,7 @@ class Playlist(node.PlaylistNode):
def __setstate__(self, state):
node.PlaylistNode.__setstate__(self, state)
self.list = [ trak.map.get(hash) for hash in state["list"] ]
self.list = [ trak.lookup(path) for path in state["list"] ]
self.sort_order = state["sort_order"]
self.current = state["current"]
self.can_loop = state["can_loop"]

View File

@ -32,14 +32,14 @@ class PlaylistRoot(node.PlaylistNode):
def __getstate__(self):
state = node.PlaylistNode.__getstate__(self)
if self.track != None:
state["track"] = self.track.hash()
state["track"] = self.track.path
return state
def __setstate__(self, state):
node.PlaylistNode.__setstate__(self, state)
self.current = state["current"]
self.track = state["track"]
self.track = trak.map.get(state["track"]) if self.track else None
self.track = trak.lookup(state["track"]) if self.track else None
def current_changed(self, old):
if old != self.current[0]:

View File

@ -12,7 +12,7 @@ test_album = os.path.join("./trier/Test Library/Test Artist 01/Test Album 1")
class TestArtistPlaylist(unittest.TestCase):
def setUp(self):
notify.clear()
trak.map.clear()
trak.reset()
def test_artist_node(self):
anode = artist.ArtistNode()
@ -32,7 +32,7 @@ class TestArtistPlaylist(unittest.TestCase):
anode = artist.ArtistNode()
self.assertEqual(anode.n_children(), 0)
track = trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track = trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
self.assertEqual(anode.n_children(), 1)
plist = anode.nth_child(0)
self.assertEqual(plist.name, "Test Artist 01")
@ -59,7 +59,7 @@ class TestArtistPlaylist(unittest.TestCase):
def test_artist_playlist_add(self):
anode = artist.ArtistNode()
track = trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track = trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
plist = anode.nth_child(0)
self.assertEqual(plist.n_children(), 1)

View File

@ -23,7 +23,7 @@ class TestCollectionPlaylist(unittest.TestCase):
plist = collection.CollectionPlaylist()
self.assertEqual(len(plist), 0)
track1 = trak.track.lookup(os.path.join(test_album, "01 - Test Track.ogg"))
track1 = trak.lookup(os.path.join(test_album, "01 - Test Track.ogg"))
self.assertEqual(len(plist), 1)
self.assertEqual(plist[0], track1)

View File

@ -15,7 +15,7 @@ test_album3 = os.path.join("Test Album 4", "02 - Test Track 02.ogg")
class TestDecadePlaylist(unittest.TestCase):
def setUp(self):
notify.clear()
trak.map.clear()
trak.reset()
def test_decade(self):
self.assertEqual(decade.decade( 1971), "1970s")
@ -41,7 +41,7 @@ class TestDecadePlaylist(unittest.TestCase):
dnode = decade.DecadeNode()
self.assertEqual(dnode.n_children(), 0)
track = trak.track.lookup(os.path.join(test_library, test_album))
track = trak.lookup(os.path.join(test_library, test_album))
self.assertEqual(dnode.n_children(), 1)
plist = dnode.nth_child(0)
self.assertEqual(plist.name, "1970s")
@ -69,7 +69,7 @@ class TestDecadePlaylist(unittest.TestCase):
def test_decade_playlist_add(self):
dnode = decade.DecadeNode()
track = trak.track.lookup(os.path.join(test_library, test_album))
track = trak.lookup(os.path.join(test_library, test_album))
plist = dnode.nth_child(0)
self.assertEqual(plist.n_children(), 1)

View File

@ -12,7 +12,7 @@ test_library = os.path.abspath("./trier/Test Library/Test Artist 01")
class TestGenrePlaylist(unittest.TestCase):
def setUp(self):
notify.clear()
trak.map.clear()
trak.reset()
def test_genre_node(self):
gnode = genre.GenreNode()
@ -22,8 +22,8 @@ class TestGenrePlaylist(unittest.TestCase):
def test_genre_playlist(self):
gnode = genre.GenreNode()
track = trak.track.lookup(os.path.join(test_library, "Test Album 1",
"01 - Test Track 01.ogg"))
track = trak.lookup(os.path.join(test_library, "Test Album 1",
"01 - Test Track 01.ogg"))
self.assertEqual(gnode.n_children(), 1)
plist = gnode.nth_child(0)
@ -42,12 +42,12 @@ class TestGenrePlaylist(unittest.TestCase):
def test_genre_new_tracks(self):
gnode = genre.GenreNode()
track1 = trak.track.lookup(os.path.join(test_library, "Test Album 1",
"01 - Test Track 01.ogg"))
track2 = trak.track.lookup(os.path.join(test_library, "Test Album 2",
"02 - Test Track 02.ogg"))
track3 = trak.track.lookup(os.path.join(test_library, "Test Album 2",
"03 - Test Track 03.ogg"))
track1 = trak.lookup(os.path.join(test_library, "Test Album 1",
"01 - Test Track 01.ogg"))
track2 = trak.lookup(os.path.join(test_library, "Test Album 2",
"02 - Test Track 02.ogg"))
track3 = trak.lookup(os.path.join(test_library, "Test Album 2",
"03 - Test Track 03.ogg"))
self.assertEqual(gnode.n_children(), 2)
plist = gnode.nth_child(0)

View File

@ -15,9 +15,8 @@ test_album = os.path.abspath("./trier/Test Album")
class TestLibraryPlaylist(unittest.TestCase):
def setUp(self):
library.reset()
trak.map.clear()
trak.map.remove()
notify.clear()
trak.reset()
def tearDownClass():
library.stop()
@ -41,7 +40,7 @@ class TestLibraryPlaylist(unittest.TestCase):
def test_library_lookup(self):
lnode = library.LibraryNode()
self.assertFalse(trak.map.exists())
self.assertFalse(trak.exists())
plist = lnode.lookup(test_library)
self.assertEqual(lnode.n_children(), 1)
@ -49,7 +48,7 @@ class TestLibraryPlaylist(unittest.TestCase):
library.join()
self.assertEqual(len(plist), 1250)
self.assertEqual(lnode.lookup(test_library + "/"), plist)
self.assertTrue(trak.map.exists())
self.assertTrue(trak.exists())
plist = lnode.lookup(test_album)
self.assertEqual(lnode.n_children(), 2)

View File

@ -20,13 +20,13 @@ class TestNewPlaylist(unittest.TestCase):
plist = new.NewPlaylist()
self.assertEqual(len(plist), 0)
track = trak.track.lookup(os.path.join(test_album, "01 - Test Track.ogg"))
track = trak.lookup(os.path.join(test_album, "01 - Test Track.ogg"))
self.assertEqual(len(plist), 1)
self.assertEqual(plist[0], track)
def test_new_plist_get_state(self):
plist = new.NewPlaylist()
track = trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track = trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
plist.add(track)
plist.current = 3

View File

@ -12,7 +12,7 @@ test_album = os.path.join(test_library, "Test Artist 01", "Test Album 1")
class TestPlaylist(unittest.TestCase):
def setUp(self):
trak.map.clear()
trak.reset()
notify.register("add-track", self.on_add_remove)
notify.register("remove-track", self.on_add_remove)
notify.register("playlist-changed", self.on_plist_changed)
@ -30,7 +30,7 @@ class TestPlaylist(unittest.TestCase):
tracks = [ ]
for t in tracknos:
path = os.path.join(test_album, f"{t:02} - Test Track {t:02}.ogg")
tracks.append(trak.track.lookup(path))
tracks.append(trak.lookup(path))
self.plist.add(tracks[-1])
return tracks
@ -66,7 +66,7 @@ class TestPlaylist(unittest.TestCase):
state = self.plist.__getstate__()
self.assertEqual(state["name"], "Test Playlist")
self.assertEqual(state["icon"], "")
self.assertEqual(state["list"], [ tracks[0].hash(), tracks[1].hash() ])
self.assertEqual(state["list"], [ tracks[0].path, tracks[1].path ])
def test_playlist__len__(self):
self.assertEqual(len(self.plist), 0)
@ -77,7 +77,7 @@ class TestPlaylist(unittest.TestCase):
tracks = self.add_tracks([1, 2])
state = { "name" : "Test", "icon" : "test", "children" : [ ],
"sibling" : None, "parent" : None,
"list" : [ tracks[0].hash(), tracks[1].hash() ],
"list" : [ tracks[0].path, tracks[1].path ],
"sort_order" : [ 1 ], "current" : 2, "can_loop" : False,
"loop" : True, "can_random" : False, "random" : True,
"visible" : True }
@ -137,7 +137,7 @@ class TestPlaylist(unittest.TestCase):
for track in tracks:
self.assertTrue(self.plist.contains(track))
track = trak.track.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
track = trak.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
self.assertFalse(self.plist.contains(track))
def test_playlist_index(self):
@ -145,7 +145,7 @@ class TestPlaylist(unittest.TestCase):
for i, track in enumerate(tracks):
self.assertEqual(self.plist.index(track), i)
track = trak.track.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
track = trak.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
self.assertIsNone(self.plist.index(track))
self.assertIsNone(self.plist.index(None))
@ -155,7 +155,7 @@ class TestPlaylist(unittest.TestCase):
for i, track in enumerate(tracks):
self.assertEqual(self.plist.index(track), i)
track = trak.track.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
track = trak.lookup(os.path.join(test_album, f"10 - Test Track 10.ogg"))
self.assertIsNone(self.plist.index(track))
self.assertIsNone(self.plist.index(None))

View File

@ -33,7 +33,7 @@ class TestPreviousPlaylist(unittest.TestCase):
def test_previous_getstate(self):
plist = previous.PreviousPlaylist()
track = trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track = trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
plist.add(track)
plist.current = 3
@ -48,9 +48,9 @@ class TestPreviousPlaylist(unittest.TestCase):
plist = previous.PreviousPlaylist()
plist.visible = True
track1 = trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track2 = trak.track.lookup(os.path.join(test_album, "02 - Test Track 02.ogg"))
track3 = trak.track.lookup(os.path.join(test_album, "03 - Test Track 03.ogg"))
track1 = trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
track2 = trak.lookup(os.path.join(test_album, "02 - Test Track 02.ogg"))
track3 = trak.lookup(os.path.join(test_album, "03 - Test Track 03.ogg"))
self.assertEqual(plist.current, -1)
self.assertEqual(plist.next(), None)

View File

@ -22,7 +22,7 @@ test_library = os.path.abspath("./trier/Test Library")
class TestPlaylistRoot(unittest.TestCase):
def setUp(self):
trak.map.clear()
trak.reset()
library.reset()
notify.register("next-track", self.on_next_track)
notify.register("playlist-changed", self.on_playlist_changed)
@ -106,7 +106,7 @@ class TestPlaylistRoot(unittest.TestCase):
track = self.playman.next()
state = { "name" : "Test", "icon" : "test", "children" : self.playman.children,
"sibling" : None, "parent" : None,
"current" : [ 1 , 2, 3 ], "track" : track.hash() }
"current" : [ 1 , 2, 3 ], "track" : track.path }
self.playman.reset()
self.playman.__setstate__(state)

View File

@ -26,7 +26,7 @@ class TestUpNextPlaylist(unittest.TestCase):
tracks = [ ]
for i in range(1, 4,):
path = os.path.join(test_album, f"0{i} - Test Track 0{i}.ogg")
tracks.append(trak.track.lookup(path))
tracks.append(trak.lookup(path))
plist.add(tracks[-1])
return tracks

View File

@ -11,11 +11,8 @@ test_tracks = os.path.abspath("./trier/Test Album")
class TestTags(unittest.TestCase):
def setUp(self):
data.DataFile("tracks.pickle", data.WRITE).remove()
notify.clear()
trak.reset()
trak.map.clear()
trak.map.remove()
self.new_track = None
def on_new_track(self, track):
@ -83,32 +80,32 @@ class TestTags(unittest.TestCase):
self.assertEqual(trak.trak_db, { })
self.assertFalse(trak.exists())
def test_tags_stress(self):
def test_trak_stress(self):
dfile = data.DataFile("tracks.pickle", data.READ)
tq = threadqueue.ThreadQueue()
tracks = 0
self.assertFalse(dfile.exists())
self.assertEqual(len(trak.map.tag_map), 0)
self.assertEqual(len(trak.trak_db), 0)
for dirname, subdirs, files in os.walk(test_library):
save = False
for f in files:
tq.push(trak.track.lookup, os.path.join(dirname, f))
tq.push(trak.lookup, os.path.join(dirname, f))
tracks += 1
save = True
if save == True:
tq.push(trak.map.save)
tq.push(trak.save)
self.assertLess(len(trak.map.tag_map), tracks)
self.assertLess(len(trak.trak_db), tracks)
tq.join()
tq.stop()
self.assertEqual(len(trak.map.tag_map), tracks)
self.assertEqual(len(trak.trak_db), tracks)
self.assertTrue(dfile.exists())
tag_list = list(trak.map.tag_map.values())
tag_list = list(trak.trak_db.values())
trak.map.clear()
trak.map.load()
self.assertEqual(len(trak.map.tag_map), tracks)
trak.clear()
trak.load()
self.assertEqual(len(trak.trak_db), tracks)
for tag in tag_list:
self.assertIn(tag.hash(), trak.map.tag_map)
self.assertIn(tag.path, trak.trak_db)

View File

@ -1,7 +1,6 @@
# Copyright 2019 (c) Anna Schumaker.
from .. import data
from .. import notify
from . import map
from . import track
import threading

View File

@ -1,44 +0,0 @@
# Copyright 2019 (c) Anna Schumaker.
from .. import data
from .. import notify
import os
import threading
tag_file = "tracks.pickle"
tag_lock = threading.Lock()
tag_map = dict()
def clear():
with tag_lock:
tag_map.clear()
def exists():
with tag_lock:
return data.DataFile(tag_file, data.READ).exists()
def get(hash):
with tag_lock:
return tag_map.get(hash, None)
def load():
with tag_lock:
with data.DataFile(tag_file, data.READ) as f:
tag_map.update(f.unpickle())
def lookup(tag):
with tag_lock:
ret = tag_map.setdefault(tag.hash(), tag)
return (ret == tag, ret)
def remove():
with tag_lock:
return data.DataFile(tag_file, data.WRITE).remove()
def save():
with tag_lock:
with data.DataFile(tag_file, data.WRITE) as f:
f.pickle(tag_map)
notify.register("save-data", save)
if os.environ.get("EMMENTAL_TESTING"): remove()

View File

@ -1,62 +0,0 @@
# Copyright 2019 (c) Anna Schumaker.
from . import map
from .. import data
import unittest
class TestTag:
def __init__(self, hash, value):
self.h = hash
self.v = value
def hash(self):
return self.h
class TestMap(unittest.TestCase):
def setUp(self):
data.DataFile("tracks.pickle", data.WRITE).remove()
map.clear()
def test_map(self):
self.assertIsInstance(map.tag_map, dict)
self.assertIsNone(map.get("a"))
map.tag_map["a"] = 1
self.assertEqual(map.get("a"), 1)
map.clear()
self.assertEqual(map.tag_map, { })
self.assertIsNone(map.get("a"))
a = TestTag("a", 1)
b = TestTag("a", 2)
self.assertEqual(map.lookup(a), (True, a))
self.assertEqual(map.lookup(b), (False, a))
def test_map_save_load(self):
dfile = data.DataFile("tracks.pickle", data.READ)
self.assertFalse(dfile.exists())
self.assertFalse(map.exists())
map.load()
self.assertEqual(len(map.tag_map), 0)
a = TestTag("a", 1)
self.assertEqual(map.lookup(a), (True, a))
self.assertFalse(dfile.exists())
self.assertFalse(map.exists())
map.save()
self.assertTrue(dfile.exists())
self.assertTrue(map.exists())
map.clear()
self.assertFalse(a.hash() in map.tag_map)
self.assertIsNone(map.get(a.hash()))
map.load()
self.assertTrue(a.hash() in map.tag_map)
self.assertEqual(map.get(a.hash()).hash(), a.hash())
map.remove()
self.assertFalse(dfile.exists())
self.assertFalse(map.exists())

View File

@ -1,5 +1,4 @@
# Copyright 2019 (c) Anna Schumaker.
from . import map
from . import track
from .. import notify
import hashlib
@ -10,7 +9,6 @@ test_tracks = os.path.abspath("./trier/Test Album")
class TestTrackTag(unittest.TestCase):
def setUp(self):
map.clear()
notify.register("new-track", self.on_new_track)
self.cb_track = None
@ -83,7 +81,7 @@ class TestTrackTag(unittest.TestCase):
self.assertIsNone(t["nokey"])
def test_track_sort_key(self):
t = track.lookup(os.path.join(test_tracks, "01 - Test Track.ogg"))
t = track.Track(os.path.join(test_tracks, "01 - Test Track.ogg"))
self.assertEqual(t.sort_key(["title"]), [[ "test", "track" ]])
self.assertEqual(t.sort_key(["title", "tracknumber"]), [ ["test", "track"], 1 ])
@ -93,20 +91,3 @@ class TestTrackTag(unittest.TestCase):
self.assertEqual(t.sort_key(["title"]), [[ "test", "track" ]])
t.title = "Test Träck 01"
self.assertEqual(t.sort_key(["title"]), [[ "test", "track", "01" ]])
def test_track_lookup(self):
t = track.lookup(os.path.join(test_tracks, "01 - Test Track.ogg"))
self.assertIsNotNone(t)
self.assertIn(t, map.tag_map.values())
self.assertEqual(t, self.cb_track)
self.cb_track = None
for i in range(10):
u = track.lookup(os.path.join(test_tracks, "01 - Test Track.ogg"))
self.assertEqual(t, u)
self.assertIsNone(self.cb_track)
self.assertEqual(len(map.tag_map), 1)
def test_track_lookup_exception(self):
self.assertIsNone(track.lookup(os.path.join(test_tracks, "text.txt")))
self.assertIsNone(track.lookup(os.path.join(test_tracks, "test.txt")))

View File

@ -1,5 +1,4 @@
# Copyright 2019 (c) Anna Schumaker.
from . import map
from . import tagfile
from .. import notify
from .. import sort
@ -50,13 +49,3 @@ class Track:
val = sort.key(val)
ret.append(val)
return ret
def lookup(path):
try:
(new, ret) = map.lookup(Track(path))
if new == True:
notify.notify("new-track", ret)
return ret
except Exception as e:
return None

View File

@ -19,8 +19,8 @@ class TestNodeTreeModel(unittest.TestCase):
self.artist = curds.playlist.lookup("Artists")
self.decade = curds.playlist.lookup("Decades")
self.genre = curds.playlist.lookup("Genres")
curds.trak.track.lookup(os.path.join(test_album1, "01 - Test Track 01.ogg"))
curds.trak.track.lookup(os.path.join(test_album2, "01 - Test Track 01.ogg"))
curds.trak.lookup(os.path.join(test_album1, "01 - Test Track 01.ogg"))
curds.trak.lookup(os.path.join(test_album2, "01 - Test Track 01.ogg"))
def tearDown(self):
curds.notify.cancel("node-inserted", self.model.on_node_inserted)

View File

@ -12,8 +12,8 @@ class TestPlaylistModel(unittest.TestCase):
curds.reset()
self.playlist = curds.Playlist("Test Playlist")
self.track1 = curds.trak.track.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
self.track2 = curds.trak.track.lookup(os.path.join(test_album, "02 - Test Track 02.ogg"))
self.track1 = curds.trak.lookup(os.path.join(test_album, "01 - Test Track 01.ogg"))
self.track2 = curds.trak.lookup(os.path.join(test_album, "02 - Test Track 02.ogg"))
self.playlist.add(self.track1)
self.playlist.add(self.track2)

View File

@ -25,7 +25,7 @@ class TestPlaylistView(unittest.TestCase):
def add_track(self, trackno):
path = os.path.join(test_album, f"{trackno:02} - Test Track {trackno:02}.ogg")
track = curds.trak.track.lookup(path)
track = curds.trak.lookup(path)
self.plist.add(track)
gtk.notify_loop()
@ -155,7 +155,7 @@ class TestPlaylistView(unittest.TestCase):
self.row_deleted = 0
for i in range(1, 11):
track = curds.trak.track.lookup(os.path.join(test_album, f"{i:02} - Test Track {i:02}.ogg"))
track = curds.trak.lookup(os.path.join(test_album, f"{i:02} - Test Track {i:02}.ogg"))
self.plist.add(track)
gtk.notify_loop()
self.assertEqual(self.row_inserted, i)

View File

@ -10,7 +10,7 @@ test_album = os.path.abspath("./trier/Test Library/Test Artist 01/Test Album 1")
class TestAudio(unittest.TestCase):
def test_play_track(self):
path = os.path.join(test_album, "01 - Test Track 01.ogg")
track = curds.trak.track.lookup(path)
track = curds.trak.lookup(path)
self.assertEqual(audio.playbin.get_state(), Gst.State.NULL)
audio.play_track(track)

View File

@ -30,15 +30,15 @@ class TestEmmental(unittest.TestCase):
def test_reset_stop_curds(self):
path = os.path.join("trier", "Test Album", "01 - Test Track.ogg")
track = curds.trak.track.lookup(path)
track = curds.trak.lookup(path)
curds.playlist.library.library_thread.stop()
self.assertIsNotNone(track)
self.assertGreater(len(curds.trak.map.tag_map), 0)
self.assertGreater(len(curds.trak.trak_db), 0)
self.assertFalse(curds.playlist.library.library_thread.is_alive())
curds.reset()
self.assertEqual(curds.trak.map.tag_map, {})
self.assertEqual(curds.trak.trak_db, {})
self.assertTrue(curds.playlist.library.library_thread.is_alive())
self.assertEqual(len(curds.playlist.lookup("Collection")), 0)
self.assertEqual(curds.playlist.lookup("Libraries").n_children(), 0)