diff --git a/include/format.h b/include/format.h
index 5628b136..280d7f9d 100644
--- a/include/format.h
+++ b/include/format.h
@@ -9,6 +9,7 @@ namespace libsaria
 {
 
 	set<string> *format_text(const string &);
+	void format_substrs(const string &, set<string> &);
 	string *lowercase(const string &);
 	string length_string(unsigned int);
 	void print_format_stats();
diff --git a/include/index.h b/include/index.h
deleted file mode 100644
index 24132f1b..00000000
--- a/include/index.h
+++ /dev/null
@@ -1,35 +0,0 @@
-#ifndef LIBSARIA_INDEX_H
-#define LIBSARIA_INDEX_H
-
-#include <map>
-#include <set>
-#include <string>
-using namespace std;
-
-namespace libsaria
-{
-
-	class Track;
-
-	class Index {
-		private:
-			set<Track *> results;
-			set<string> *terms;
-			map<string, set<Track *> > artist;
-			map<string, set<Track *> > album;
-			map<string, set<Track *> > title;
-
-		public:
-			Index();
-			~Index();
-
-			static void print_stats();
-
-			void add_track(Track *);
-			void remove_track(Track *);
-			void do_filter(string &);
-			bool is_visible(Track *);
-	};
-
-}; /* Namespace: libsaria */
-#endif /* LIBSARIA_INDEX_H */
diff --git a/include/playlist.h b/include/playlist.h
index 15ed253a..89649c40 100644
--- a/include/playlist.h
+++ b/include/playlist.h
@@ -3,7 +3,6 @@
 #define LIBSARIA_PLAYLIST_H
 
 #include <notify.h>
-#include <index.h>
 #include <fs.h>
 
 #include <list>
@@ -33,7 +32,6 @@ namespace libsaria
 			DataState data_state;
 			unsigned int cur;
 			vector<Track *> plist;
-			Index index;
 
 			void schedule_save();
 			void rm_file();
@@ -61,8 +59,6 @@ namespace libsaria
 			void save(ofstream &);
 			void prepare_for_removal();
 			void reset_iterator();
-			void set_filter_text(string &);
-			bool is_visible(libsaria::Track *);
 
 			void renumber(int);
 			unsigned int get_number();
diff --git a/include/track.h b/include/track.h
index e4c63eb6..3abf8ba9 100644
--- a/include/track.h
+++ b/include/track.h
@@ -3,6 +3,7 @@
 
 #include <playlist.h>
 
+#include <set>
 #include <list>
 #include <string>
 using namespace std;
@@ -51,6 +52,7 @@ namespace libsaria
 		private:
 			library::Path *path;
 			list<libsaria::Playlist *> playlists;
+			set<string> substrs;
 
 			string filepath;
 			string title;
@@ -76,6 +78,7 @@ namespace libsaria
 			string *album_lc;
 			void read_tags();
 			void mark_played();
+			void find_substrs();
 
 		public:
 			Track();
@@ -89,6 +92,7 @@ namespace libsaria
 			void load(bool);
 			void add_playlist(libsaria::Playlist *);
 			void rm_playlist(libsaria::Playlist *);
+			bool is_visible(set<string> *);
 
 			string get_last_played();
 
diff --git a/libsaria/format.cpp b/libsaria/format.cpp
index 7d0beec3..34f9752b 100644
--- a/libsaria/format.cpp
+++ b/libsaria/format.cpp
@@ -8,9 +8,11 @@
 using namespace std;
 
 static map<string, set<string> > format_cache;
+static map<string, set<string> > substr_cache;
 static map<string, string> lc_cache;
-static unsigned int format_hits;
-static unsigned int lc_hits;
+static unsigned int format_hits = 0;
+static unsigned int substr_hits = 0;
+static unsigned int lc_hits = 0;
 
 static const unsigned int TM_SECOND = 1;
 static const unsigned int TM_MINUTE = 60;
@@ -90,6 +92,33 @@ void find_unique_words(const string &text,
 	do_format(text, words, lc);
 }
 
+static set<string> *gen_substrs(const string &word)
+{
+	string key;
+	set<string> substrs;
+	pair< map<string, set<string> >::iterator, bool > ret;
+
+	for (unsigned int i = 1; i <= word.size(); i++) {
+		key = word.substr(0, i);
+		substrs.insert(key);
+	}
+
+	ret = substr_cache.insert( pair<string, set<string> >(word, substrs));
+	return &(ret.first->second);
+}
+
+static set<string> *find_substrs(const string &word)
+{
+	map<string, set<string> >::iterator it;
+	it = substr_cache.find(word);
+	if (it == substr_cache.end())
+		return gen_substrs(word);
+	else {
+		substr_hits++;
+		return &(it->second);
+	}
+}
+
 void find_lowercase(const string &text,
 		    map<string, string>::iterator &lc)
 {
@@ -129,6 +158,18 @@ namespace libsaria
 		return &(it->second);
 	}
 
+	void format_substrs(const string &text, set<string> &res)
+	{
+		set<string> *words, *substrs;
+		set<string>::iterator it;
+
+		words = format_text(text);
+		for (it = words->begin(); it != words->end(); it++) {
+			substrs = find_substrs(*it);
+			res.insert(substrs->begin(), substrs->end());
+		}
+	}
+
 	string *lowercase(const string &text)
 	{
 		map<string, string>::iterator it;
@@ -161,6 +202,7 @@ namespace libsaria
 	void print_format_stats()
 	{
 		println("Format cache hits: %u size: %u", format_hits, format_cache.size());
+		println("Substring cache hits: %u size: %u", substr_hits, substr_cache.size());
 		println("Lowercase cache hits: %u size: %u", lc_hits, lc_cache.size());
 	}
 
diff --git a/libsaria/index/filter.cpp b/libsaria/index/filter.cpp
deleted file mode 100644
index c69de6d8..00000000
--- a/libsaria/index/filter.cpp
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright (c) 2011 Bryan Schumaker.
-#include <format.h>
-#include <track.h>
-#include <index.h>
-
-/*
- * Thanks to: http://stackoverflow.com/questions/1773526/in-place-c-set-intersection
- * for the set intersection algorithm!
- */
-static void inplace_intersect(set<libsaria::Track *> *results,
-			      set<libsaria::Track *> *tracks)
-{
-	set<libsaria::Track *>::iterator it1 = results->begin();
-	set<libsaria::Track *>::iterator it2 = tracks->begin();
-
-	while ( (it1 != results->end()) && (it2 != tracks->end()) ) {
-		if (*it1 < *it2)
-			results->erase(it1++);
-		else if (*it2 < *it1)
-			it2++;
-		else { /* *it1 == *it2 */
-			it1++;
-			it2++;
-		}
-	}
-
-	/* Remove everything in results that wasn't in inodes */
-	results->erase(it1, results->end());
-}
-
-static void search_index(map<string, set<libsaria::Track *> > *index,
-			 const string &term, set<libsaria::Track *> *res)
-{
-	map<string, set<libsaria::Track *> >::iterator it;
-	it = index->find(term);
-	if (it != index->end())
-		res->insert(it->second.begin(), it->second.end());
-}
-
-namespace libsaria
-{
-
-	void Index::do_filter(string &text)
-	{
-		set<string>::iterator it;
-		set<Track *> found;
-
-		results.clear();
-		terms = format_text(text);
-		if (terms == NULL || terms->size() == 0)
-			return;
-
-		for (it = terms->begin(); it != terms->end(); it++) {
-			found.clear();
-			search_index(&artist, *it, &found);
-			search_index(&album,  *it, &found);
-			search_index(&title,  *it, &found);
-			/*
-			 * Key not found means we don't need to filtered
-			 * anymore just clear the results set and return.
-			 */
-			if (found.size() == 0) {
-				results.clear();
-				return;
-			}
-			/*
-			 * This is the first result, so the result set is
-			 * empty.  Taking an intersection will always give
-			 * us an empty set.
-			 */
-			if (it == terms->begin())
-				results = found;
-			else
-				inplace_intersect(&results, &found);
-		}
-	}
-
-	bool Index::is_visible(Track *track)
-	{
-		if (!terms || terms->size() == 0)
-			return true;
-		return results.find(track) != results.end();
-	}
-
-}; /* Namespace: libsaria */
diff --git a/libsaria/index/index.cpp b/libsaria/index/index.cpp
deleted file mode 100644
index 1da03338..00000000
--- a/libsaria/index/index.cpp
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright (c) 2011 Bryan Schumaker.
-#include <format.h>
-#include <track.h>
-#include <index.h>
-#include <print.h>
-
-#include <map>
-#include <set>
-#include <string>
-using namespace std;
-
-static map<string, set<string> > substr_cache;
-unsigned int hits;
-
-static set<string> *gen_substrs(const string &word)
-{
-	string key;
-	set<string> substrs;
-	pair< map<string, set<string> >::iterator, bool > ret;
-
-	for (unsigned int i = 1; i <= word.size(); i++) {
-		key = word.substr(0, i);
-		substrs.insert(key);
-	}
-
-	ret = substr_cache.insert( pair<string, set<string> >(word, substrs));
-	return &(ret.first->second);
-}
-
-static set<string> *find_substrs(const string &word)
-{
-	map<string, set<string> >::iterator it;
-	it = substr_cache.find(word);
-	if (it == substr_cache.end())
-		return gen_substrs(word);
-	else {
-		hits++;
-		return &(it->second);
-	}
-}
-
-static void index_substr(libsaria::Track *track, const string &key,
-			 map<string, set<libsaria::Track *> > *index)
-{
-	map<string, set<libsaria::Track *> >::iterator it = index->find(key);
-
-	if (it != index->end())
-		it->second.insert(track);
-	else {
-		set<libsaria::Track *> tracks;
-		tracks.insert(track);
-		index->insert(pair<string, set<libsaria::Track *> >(key, tracks));
-	}
-}
-
-static void index_word(libsaria::Track *track, const string &word,
-		       map<string, set<libsaria::Track *> > *index)
-{
-	set<string> *substrs = find_substrs(word);
-	set<string>::iterator it;
-
-	for (it = substrs->begin(); it != substrs->end(); it++)
-		index_substr(track, *it, index);
-}
-
-static void index_tag(libsaria::Track *track, const string &tag,
-		      map<string, set<libsaria::Track *> > *index)
-{
-	set<string> *words;
-	set<string>::iterator it;
-
-	words = libsaria::format_text(tag);
-	for (it = words->begin(); it != words->end(); it++)
-		index_word(track, *it, index);
-}
-
-static void remove_substr(libsaria::Track *track, const string &key,
-			  map<string, set<libsaria::Track *> > *index)
-{
-	map<string, set<libsaria::Track *> >::iterator it = index->find(key);
-	if (it != index->end())
-		it->second.erase(track);
-}
-
-static void remove_word(libsaria::Track *track, const string &word,
-			map<string, set<libsaria::Track *> > *index)
-{
-	set<string> *substrs = find_substrs(word);
-	set<string>::iterator it;
-
-	for (it = substrs->begin(); it != substrs->end(); it++)
-		remove_substr(track, *it, index);
-}
-
-static void remove_tag(libsaria::Track *track, const string &tag,
-		       map<string, set<libsaria::Track *> > *index)
-{
-	set<string> *words;
-	set<string>::iterator it;
-
-	words = libsaria::format_text(tag);
-	for (it = words->begin(); it != words->end(); it++)
-		remove_word(track, *it, index);
-}
-
-namespace libsaria
-{
-
-	Index::Index()
-	{
-		terms = NULL;
-	}
-
-	Index::~Index()
-	{
-	}
-
-	void Index::add_track(Track *track)
-	{
-		index_tag(track, (*track)[TRACK_ARTIST], &artist);
-		index_tag(track, (*track)[TRACK_ALBUM],  &album);
-		index_tag(track, (*track)[TRACK_TITLE],  &title);
-	}
-
-	void Index::remove_track(Track *track)
-	{
-		remove_tag(track, (*track)[TRACK_ARTIST], &artist);
-		remove_tag(track, (*track)[TRACK_ALBUM],  &album);
-		remove_tag(track, (*track)[TRACK_TITLE],  &title);
-	}
-
-	void Index::print_stats()
-	{
-		println("Index cache hits: %u size: %u", hits, substr_cache.size());
-	}
-
-}
diff --git a/libsaria/libsaria.cpp b/libsaria/libsaria.cpp
index a521ad1b..f7782304 100644
--- a/libsaria/libsaria.cpp
+++ b/libsaria/libsaria.cpp
@@ -6,7 +6,6 @@
 #include <prefs.h>
 #include <print.h>
 #include <audio.h>
-#include <index.h>
 #include <track.h>
 #include <deck.h>
 #include <idle.h>
@@ -42,7 +41,6 @@ namespace libsaria
 		app::close_pipe();
 		audio::quit();
 		print_format_stats();
-		Index::print_stats();
 	}
 
 	void play_outside_song(string &filepath)
diff --git a/libsaria/playlist/add.cpp b/libsaria/playlist/add.cpp
index e1c3e43f..bf8b46f7 100644
--- a/libsaria/playlist/add.cpp
+++ b/libsaria/playlist/add.cpp
@@ -17,7 +17,6 @@ namespace libsaria
 
 	void Playlist::add_track(Track *track, unsigned int ins_index)
 	{
-		index.add_track(track);
 		track->add_playlist(this);
 		notify_ui(PLAYLIST_ADD,  track, ins_index);
 		data_state = DIRTY;
@@ -32,7 +31,6 @@ namespace libsaria
 			return;
 
 		Track *track = plist[rm_index];
-		index.remove_track(track);
 		track->rm_playlist(this);
 		plist.erase(plist.begin() + rm_index);
 		if (cur > rm_index)
diff --git a/libsaria/playlist/playlist.cpp b/libsaria/playlist/playlist.cpp
index e48a02fc..10f493a7 100644
--- a/libsaria/playlist/playlist.cpp
+++ b/libsaria/playlist/playlist.cpp
@@ -1,5 +1,4 @@
 // Copyright (c) 2012 Bryan Schumaker.
-#include <index.h>
 #include <track.h>
 #include <library.h>
 #include <playlist.h>
@@ -53,16 +52,6 @@ namespace libsaria
 		rm_file();
 	}
 
-	void Playlist::set_filter_text(string &text)
-	{
-		index.do_filter(text);
-	}
-
-	bool Playlist::is_visible(libsaria::Track *track)
-	{
-		return index.is_visible(track);
-	}
-
 	void Playlist::renumber(int n)
 	{
 		if (number == n)
diff --git a/libsaria/track/track.cpp b/libsaria/track/track.cpp
index 632ccb0f..a93d60a7 100644
--- a/libsaria/track/track.cpp
+++ b/libsaria/track/track.cpp
@@ -104,6 +104,7 @@ namespace libsaria
 		artist_lc = lowercase(artist);
 		album_lc  = lowercase(album);
 		make_lenstr(length, lenstr);
+		find_substrs();
 	}
 
 	Track::Track(ifstream &in, struct library::Path *lib_path,
@@ -144,6 +145,7 @@ namespace libsaria
 
 		artist_lc = lowercase(artist);
 		album_lc  = lowercase(album);
+		find_substrs();
 	}
 
 	Track::~Track()
@@ -232,4 +234,20 @@ namespace libsaria
 		playlists.remove(plist);
 	}
 
+	void Track::find_substrs()
+	{
+		format_substrs(artist, substrs);
+		format_substrs(album, substrs);
+		format_substrs(title, substrs);
+	}
+
+	bool Track::is_visible(set<string> *search) {
+		set<string>::iterator it;
+		for (it = search->begin(); it != search->end(); it++) {
+			if (substrs.find(*it) == substrs.end())
+				return false;
+		}
+		return true;
+	}
+
 } /* Namespace: libsaria */
diff --git a/ocarina/ocarina.h b/ocarina/ocarina.h
index dad0967b..83d071b1 100644
--- a/ocarina/ocarina.h
+++ b/ocarina/ocarina.h
@@ -31,6 +31,7 @@ struct PlaylistWidgets {
 	GtkToggleButton		*disable;
 	GtkToggleButton		*random;
 	GtkToggleButton		*sort;
+	set<string>		*filter_text;
 };
 
 /* library.cpp */
diff --git a/ocarina/playlist.cpp b/ocarina/playlist.cpp
index cbf93dcb..47a22ec8 100644
--- a/ocarina/playlist.cpp
+++ b/ocarina/playlist.cpp
@@ -1,6 +1,7 @@
 // Copyright (c) 2012 Bryan Schumaker
 #include <ban.h>
 #include <track.h>
+#include <format.h>
 #include <library.h>
 #include <stdlib.h>
 #include "ocarina.h"
@@ -118,18 +119,21 @@ void update_playlist(notify_t event, libsaria::PlaylistNotification *data)
 static gboolean is_visible(GtkTreeModel *model, GtkTreeIter *iter, gpointer data)
 {
 	libsaria::Track *track;
-	libsaria::Playlist *plist = (libsaria::Playlist *)data;
+	struct PlaylistWidgets *widgets = (struct PlaylistWidgets *)data;
+
+	if (widgets->filter_text == NULL)
+		return true;
 
 	gtk_tree_model_get(model, iter, 0, &track, -1);
-	return plist->is_visible(track);
+	return track->is_visible(widgets->filter_text);
 }
 
 static void do_filter(GtkWidget *entry, gpointer data)
 {
 	struct PlaylistWidgets *widgets = (struct PlaylistWidgets *)data;
 	string text = gtk_entry_get_text(GTK_ENTRY(entry));
+	widgets->filter_text = libsaria::format_text(text);
 
-	widgets->playlist->set_filter_text(text);
 	gtk_tree_model_filter_refilter(GTK_TREE_MODEL_FILTER(widgets->filter));
 }
 
@@ -419,6 +423,7 @@ void setup_widgets(struct PlaylistWidgets *widgets, libsaria::Playlist *playlist
 	widgets->size_label = GTK_LABEL(gtk_label_new("0"));
 	widgets->page_label = GTK_LABEL(gtk_label_new(""));
 	widgets->entry      = GTK_ENTRY(gtk_entry_new());
+	widgets->filter_text = NULL;
 
 	selection = gtk_tree_view_get_selection(widgets->treeview);
 	gtk_tree_selection_set_mode(selection,GTK_SELECTION_MULTIPLE);
@@ -429,7 +434,7 @@ void setup_widgets(struct PlaylistWidgets *widgets, libsaria::Playlist *playlist
 	gtk_tree_view_set_tooltip_column(widgets->treeview, 9);
 
 	gtk_tree_model_filter_set_visible_func(GTK_TREE_MODEL_FILTER(widgets->filter),
-						is_visible, widgets->playlist, NULL);
+						is_visible, widgets, NULL);
 
 	g_signal_connect(widgets->treeview, "row-activated", G_CALLBACK(track_selected), NULL);
 	g_signal_connect(widgets->treeview, "button-release-event", G_CALLBACK(on_click), NULL);