From e95ddf7fb65e2a1fc82e091473c4c9e6cb09e69d Mon Sep 17 00:00:00 2001
From: Hydrus Network Developer <hydrus.admin@gmail.com>
Date: Wed, 14 Jul 2021 15:42:19 -0500
Subject: [PATCH] Version 447

---
 help/changelog.html                           |   42 +
 help/reducing_lag.html                        |   17 +-
 hydrus/client/ClientData.py                   |    4 +-
 hydrus/client/ClientDefaults.py               |    2 +-
 hydrus/client/ClientDuplicates.py             |   21 +-
 hydrus/client/ClientServices.py               |    5 +
 hydrus/client/db/ClientDB.py                  | 1511 +++++------------
 hydrus/client/db/ClientDBFilesStorage.py      |  665 ++++++++
 hydrus/client/db/ClientDBMappingsStorage.py   |   48 +-
 hydrus/client/db/ClientDBSimilarFiles.py      |   11 +-
 hydrus/client/gui/ClientGUI.py                |   92 +-
 hydrus/client/gui/ClientGUIDragDrop.py        |   59 +-
 hydrus/client/gui/ClientGUIFileSeedCache.py   |   28 +-
 hydrus/client/gui/ClientGUIFunctions.py       |   44 +-
 hydrus/client/gui/ClientGUIMenus.py           |    4 +-
 hydrus/client/gui/ClientGUISubscriptions.py   |   30 +-
 hydrus/client/gui/ClientGUITags.py            |   33 +-
 hydrus/client/gui/QtPorting.py                |    4 +-
 hydrus/client/gui/canvas/ClientGUICanvas.py   |   11 +-
 hydrus/client/gui/lists/ClientGUIListCtrl.py  |   26 +-
 .../client/gui/pages/ClientGUIManagement.py   |   22 +-
 hydrus/client/gui/pages/ClientGUIPages.py     |   28 +-
 .../client/importing/ClientImportFileSeeds.py |   24 +-
 .../client/importing/ClientImportGallery.py   |    4 +-
 .../importing/ClientImportSubscriptions.py    |    4 +-
 .../client/importing/ClientImportWatchers.py  |    4 +-
 hydrus/client/media/ClientMedia.py            |   95 +-
 hydrus/core/HydrusConstants.py                |   25 +-
 hydrus/core/HydrusController.py               |    4 +-
 hydrus/core/HydrusDB.py                       |   12 +-
 hydrus/core/HydrusData.py                     |   30 +-
 hydrus/core/HydrusGlobals.py                  |   21 +-
 hydrus/core/HydrusPubSub.py                   |   10 +-
 hydrus/core/HydrusThreading.py                |    4 +-
 .../core/networking/HydrusServerResources.py  |    8 +-
 35 files changed, 1613 insertions(+), 1339 deletions(-)
 create mode 100644 hydrus/client/db/ClientDBFilesStorage.py

diff --git a/help/changelog.html b/help/changelog.html
index dfe02284..cc6ac91f 100755
--- a/help/changelog.html
+++ b/help/changelog.html
@@ -8,6 +8,48 @@
 		<div class="content">
 			<h3 id="changelog"><a href="#changelog">changelog</a></h3>
 			<ul>
+				<li><h3 id="version_447"><a href="#version_447">version 447</a></h3></li>
+				<ul>
+					<li>misc:</li>
+					<li>fixed drag and dropping multiple newline separated urls onto the client when those urls come from a generic text source</li>
+					<li>pages now cache their 'ordered' file id list. this speeds up several little jobs, but most importantly should reduce session save time for sessions with tens of thousands of files</li>
+					<li>common file resolutions such as 1920x1080 are now replaced in labels with '1080p' strings as already used in the duplicate system. also added 'vertical' variants of 720p, 1080p, and 4k</li>
+					<li>when a page preview viewer gets a call to clear its current media when it is not currently the page in view, it now recognises that properly. this was happening (a 'sticky' preview) on drag and drops that navigated and terminated on other pages</li>
+					<li>the various 'retry ignored' commands on downloaders now give an interstitial dialog where you can choose to retry 'all', '404s', or 'blacklisted' files only</li>
+					<li>manage tag siblings/parents now disables its import button until its data is loaded. imports that were clicked through before loading were being forgotten due to tangled logic, so for now I'll just disable the button!</li>
+					<li>reduced some more spiky database I/O overhead from the UI's perspective (now savepoints are performed after a result is returned, just like I recently did with transaction commit)</li>
+					<li>duplicate potentials search will now update the y in its x/y progress display if many files have been imported since the search was started and x becomes larger than y (due to y secretly growing)</li>
+					<li>fixed the default 'gelbooru md5' file lookup script. if you have a lookup script with this name, it will be updated to my new default automatically. I don't really like fixing this old system, but I am not sure when I will fit in my big rewrite that will merge it with the normal downloader system, so this is a quick fix for the meantime</li>
+					<li>if you are one of the users who had weird unfixable 404 update file problems with the PTR, please try unpausing and doing a metadata resync one more time this week. fingers crossed, this is fixed. please let me know how you get on too, fixed or not, and also if you have had 'malformed' database problems in the past</li>
+					<li>.</li>
+					<li>multi column lists:</li>
+					<li>improved the precision of longer text pixel_width->text and text->pixel_width calculations, which are particularly used in the multi-column list state saving system. another multi-column size calculation bug, where lists could grow by 1 character's width on >~60 character width columns on every dialog reopen, is now fixed</li>
+					<li>multi-column lists should now calculate last column width more precisely and accurately regardless of vertical scrollbar presence or recent show/hide</li>
+					<li>the snapping system that locks last column size to 5-character multiples can now snap up or down, increasing error tolerance</li>
+					<li>I added a hack to stop the bug some people had of multi-column lists suddenly growing wide, up to screen width, in a resize loop. I think it works, but as I cannot reproduce this error, please let me know how you get on. resizing the options->external programs panel seems to initiate it reliably for those users affected</li>
+					<li>.</li>
+					<li>profile mode:</li>
+					<li>all debug profile modes (callto, db, server, menu, pubsub, and ui) are now merged into one mode under help->debug</li>
+					<li>this new mode no longer spams popups, and it only prints 'slow' jobs to the profile log</li>
+					<li>it also makes a new profile log every time it is turned on, using mode start timestamp rather than client boot timestamp, and when profile mode is turned off, there is a popup summary of how many fast and slow jobs passed through during the log time</li>
+					<li>touched up profile code, timing thresholds, summary statements, and the help</li>
+					<li>.</li>
+					<li>special update rule this week:</li>
+					<li>due to the big file storage rework this week, there's some bit rot in older update routines. 447 cannot update databases older than 411, and it _may_ have trouble updating before 436. if this applies to you, the client will error out or warn you before continuing. I'd like to know what happens to you if you are v411-435 so I can refine these messages</li>
+					<li>.</li>
+					<li>boring database refactoring:</li>
+					<li>the primary current, deleted, pending, and petitioned files tables are now split according to service, much as I have done with mapping tables in the past. this saves a little space and accelerates many file calculations on large clients. if you have a client database script or patch that inspects 'current_files' or 'deleted_files', you'll now be looking at client_files_x etc.., where x is the service_id, and they obviously no longer have a service_id column</li>
+					<li>a new file storage database module manages these tables, and also some misc file deletion metadata</li>
+					<li>refactored all raw file storage updates, filters, and searches to the new module</li>
+					<li>the mappings and new file storage database modules are now responsible for various 'num files/mappings' metadata calculations</li>
+					<li>most file operations on smaller domains, typically trash or repository update files, will be significantly faster (since the much larger 'my files' table data isn't fattening the relevant indices, and worst case query planning is so much better)</li>
+					<li>cleaned up a ton of file domain filtering code as a result of all this</li>
+					<li>physical file deletion is now much faster when the client has many pending file uploads to a file repository or IPFS service</li>
+					<li>complicated duplicate file operations of many sorts should be a _little_ faster now, particularly on large clients</li>
+					<li>searching files with 'file import time' sort should be a little faster in many situations</li>
+					<li>tag repositories no longer bother going down to the database level to to see if they have any thumbnails to sync with</li>
+					<li>everyone also gets a local file id cache regen this week, it may take a few seconds on update</li>
+				</ul>
 				<li><h3 id="version_446"><a href="#version_446">version 446</a></h3></li>
 				<ul>
 					<li>misc:</li>
diff --git a/help/reducing_lag.html b/help/reducing_lag.html
index 5b1c3b7e..b6daf984 100644
--- a/help/reducing_lag.html
+++ b/help/reducing_lag.html
@@ -8,6 +8,7 @@
 		<div class="content">
 			<h3 id="intro"><a href="#intro">hydrus is cpu and hdd hungry</a></h3>
 			<p>The hydrus client manages a lot of complicated data and gives you a lot of power over it. To add millions of files and tags to its database, and then to perform difficult searches over that information, it needs to use a lot of CPU time and hard drive time--sometimes in small laggy blips, and occasionally in big 100% CPU chunks. I don't put training wheels or limiters on the software either, so if you search for 300,000 files, the client will try to fetch that many.</p>
+                        <p>Furthermore, I am just one unprofessional guy dealing with a lot of legacy code from when I was even worse at programming. I am always working to reduce lag and other inconveniences, and improve UI feedback when many things are going on, but there is still a lot for me to do.</p>
 			<p>In general, the client works best on snappy computers with low-latency hard drives where it does not have to constantly compete with other CPU- or HDD- heavy programs. Running hydrus on your games computer is no problem at all, but if you leave the client on all the time, then make sure under the options it is set not to do idle work while your CPU is busy, so your games can run freely. Similarly, if you run two clients on the same computer, you should have them set to work at different times, because if they both try to process 500,000 tags at once on the same hard drive, they will each slow to a crawl.</p>
 			<p>If you run on an HDD, keeping it defragged is very important, and good practice for all your programs anyway. Make sure you know what this is and that you do it.</p>
 			<h3 id="maintenance_and_processing"><a href="#maintenance_and_processing">maintenance and processing</a></h3>
@@ -16,22 +17,20 @@
 			<p>If you have an old, slower computer though, or if your hard drive is high latency, make sure these options are set for whatever is best for your situation. Turning off idle time completely is often helpful as some older computers are slow to even recognise--mid task--that you want to use the client again, or take too long to abandon a big task half way through. If you set your client to only do work on shutdown, then you can control exactly when that happens.</p>
 			<h3 id="reducing_lag"><a href="#reducing_lag">reducing search and general gui lag</a></h3>
 			<p>Searching for tags via the autocomplete dropdown and searching for files in general can sometimes take a very long time. It depends on many things. In general, the more predicates (tags and system:something) you have active for a search, and the more specific they are, the faster it will be.</p>
-			<p>You can also look at <i>file->options->speed and memory</i>, again especially if you have a slow computer. Increasing the autocomplete thresholds is very often helpful. You can even force autocompletes to only fetch results when you manually ask for them.</p>
+			<p>You can also look at <i>file->options->speed and memory</i>. Increasing the autocomplete thresholds under <i>tags->manage tag display and search</i> is also often helpful. You can even force autocompletes to only fetch results when you manually ask for them.</p>
 			<p>Having lots of thumbnails open or downloads running can slow many things down. Check the 'pages' menu to see your current session weight. If it is about 50,000, or you have individual pages with more than 10,000 files or download URLs, try cutting down a bit.</p>
 			<h3 id="profiles"><a href="#profiles">finally - profiles</a></h3>
-			<p>Lots of my code remains unoptimised for certain situations. My development environment only has a few thousand images and a few million tags. As I write code, I am usually more concerned with getting it to work at all rather than getting it to work fast for every possible scenario. So, if something is running slow for you, but your computer is otherwise working fine, let me know and I can almost always speed it up.</p>
-			<p>Let me know:</p>
+			<p>Programming is all about re-editing your first, second, third drafts of an idea. You are always going back to old code and adding new features or making it work better. If something is running slow for you, I can almost always speed it up or at least improve the way it schedules that chunk of work.</p>
+                        <p>However figuring out exactly why something is running slow or holding up the UI is tricky and often gives an unexpected result. I can guess what might be running inefficiently from reports, but what I really need to be sure is a <i>profile</i>, which drills down into every function of a job, counting how many times they are called and timing how long they take. A profile for a single call looks like <a href="profile_example.txt">this</a>.</p>
+			<p>So, please let me know:</p>
 			<ul>
 				<li>The general steps to reproduce the problem (e.g. "Running system:numtags>4 is ridiculously slow on its own on 'all known tags'.")</li>
-				<li>Your operating system and its version (e.g. "Windows 8.1")</li>
-				<li>Your computer's general power (e.g. "A couple of years old. It runs most stuff ok.")</li>
+				<li>Your client's approximate overall size (e.g. "500k files, and it syncs to the PTR.")</li>
 				<li>The type of hard drive you are running hydrus from. (e.g. "A 2TB 7200rpm drive that is 20% full. I regularly defrag it.")</li>
 				<li>Any <i>profiles</i> you have collected.</li>
 			</ul>
-			<p>A <i>profile</i> is a large block of debug text that lets me know which parts of my code are running slow for you. A profile for a single call looks like <a href="profile_example.txt">this</a>.</p>
-			<p>It is very helpful to me to have a profile. You can generate one by going <i>help->debug->xxx profile mode</i>, which tells the client to generate profile information for every subsequent xxx request. This can be spammy, so don't leave it on for a very long time (you can turn it off by hitting the help menu entry again).</p>
-			<p>For most problems, you probably want <i>db profile mode</i>.</p>
-			<p>Turn on a profile mode, do the thing that runs slow for you (importing a file, fetching some tags, whatever), and then check your database folder (most likely <i>install_dir/db</i>) for a new 'client profile - DATE.log' file. This file will be filled with several sets of tables with timing information. Please send that whole file to me, or if it is too large, cut what seems important. It should not contain any personal information, but feel free to look through it.</p>
+			<p>You can generate a profile by hitting <i>help->debug->profile mode</i>, which tells the client to generate profile information for almost all of its behind the scenes jobs. This can be spammy, so don't leave it on for a very long time (you can turn it off by hitting the help menu entry again).</p>
+			<p>Turn on profile mode, do the thing that runs slow for you (importing a file, fetching some tags, whatever), and then check your database folder (most likely <i>install_dir/db</i>) for a new 'client profile - DATE.log' file. This file will be filled with several sets of tables with timing information. Please send that whole file to me, or if it is too large, cut what seems important. It should not contain any personal information, but feel free to look through it.</p>
 			<p>There are several ways to <a href="contact.html">contact me</a>.</p>
 		</div>
 	</body>
diff --git a/hydrus/client/ClientData.py b/hydrus/client/ClientData.py
index 8617558d..8528f33b 100644
--- a/hydrus/client/ClientData.py
+++ b/hydrus/client/ClientData.py
@@ -375,7 +375,7 @@ def ShowTextClient( text ):
     
     HG.client_controller.pub( 'message', job_key )
     
-def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'just now', just_now_threshold = 3, show_seconds = True, no_prefix = False ):
+def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'just now', just_now_threshold = 3, history_suffix = ' ago', show_seconds = True, no_prefix = False ):
     
     if HG.client_controller.new_options.GetBoolean( 'always_show_iso_time' ):
         
@@ -383,7 +383,7 @@ def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'just now', just_no
         
     else:
         
-        return HydrusData.TimestampToPrettyTimeDelta( timestamp, just_now_string = just_now_string, just_now_threshold = just_now_threshold, show_seconds = show_seconds, no_prefix = no_prefix )
+        return HydrusData.TimestampToPrettyTimeDelta( timestamp, just_now_string = just_now_string, just_now_threshold = just_now_threshold, history_suffix = history_suffix, show_seconds = show_seconds, no_prefix = no_prefix )
         
     
 class Booru( HydrusData.HydrusYAMLBase ):
diff --git a/hydrus/client/ClientDefaults.py b/hydrus/client/ClientDefaults.py
index b097ef7f..a48e47ab 100644
--- a/hydrus/client/ClientDefaults.py
+++ b/hydrus/client/ClientDefaults.py
@@ -275,7 +275,7 @@ def GetDefaultScriptRows():
     
     script_info.append( ( 32, 'iqdb danbooru', 2, HydrusData.GetNow(), '''["https://danbooru.iqdb.org/", 1, 0, [55, 1, [[], "some hash bytes"]], "file", {}, [[29, 1, ["link to danbooru", [27, 6, [[26, 1, [[62, 2, [0, "td", {"class": "image"}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 0, "href", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], [[30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "tag-list"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {"class": "tag-type-1"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {"class": "search-tag"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "creator"]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "tag-list"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {"class": "tag-type-3"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {"class": "search-tag"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "series"]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "tag-list"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {"class": "tag-type-4"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {"class": "search-tag"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "character"]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "tag-list"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {"class": "tag-type-0"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {"class": "search-tag"}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, ""]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "post-information"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [2, "Rating:*", null, null, "Rating: Safe"]], [55, 1, [[[0, 8]], "Rating: Safe"]]]], 0, false, "rating"]], [30, 4, ["", 7, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "post-information"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {}, null, null, true, [51, 1, [2, "Source:*", null, null, "Source:"]]]], [62, 2, [0, "a", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 0, "href", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, [8, 0]]]]]], [30, 4, ["no iqdb match found", 8, [27, 6, [[26, 1, [[62, 2, [0, "th", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, [false, [51, 1, [2, "Best match", null, null, "Best match"]]]]]]]''' ) )
     script_info.append( ( 32, 'danbooru md5', 2, HydrusData.GetNow(), '''["https://danbooru.donmai.us/", 0, 1, [55, 1, [[[4, "hex"]], "some hash bytes"]], "md5", {"page": "post", "s": "list"}, [[30, 4, ["we got sent back to main gallery page -- title test", 8, [27, 6, [[26, 1, [[62, 2, [0, "head", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "title", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, [true, [51, 1, [2, "Image List", null, null, "Image List"]]]]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-0"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, ""]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-3"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "series"]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-1"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "creator"]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-4"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, "character"]], [30, 4, ["we got sent back to main gallery page -- page links exist", 8, [27, 6, [[26, 1, [[62, 2, [0, "div", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 0, "class", [51, 1, [3, "", null, null, "example string"]], [55, 1, [[], "parsed information"]]]], 0, false, [true, [51, 1, [2, "pagination", null, null, "pagination"]]]]], [30, 4, ["", 0, [27, 6, [[26, 1, [[62, 2, [0, "section", {"id": "post-information"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "li", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "href", [51, 1, [2, "Rating:*", null, null, "Rating: Safe"]], [55, 1, [[[0, 8]], "Rating: Safe"]]]], 0, false, "rating"]]]]''' ) )
-    script_info.append( ( 32, 'gelbooru md5', 1, HydrusData.GetNow(), '''["https://gelbooru.com/index.php", 0, 1, 1, "md5", {"s": "list", "page": "post"}, [[30, 1, ["we got sent back to main gallery page -- title test", 8, [27, 1, [[["head", {}, 0], ["title", {}, 0]], null]], [true, true, "Image List"]]], [30, 1, ["", 0, [27, 1, [[["li", {"class": "tag-type-general"}, null], ["a", {}, 1]], null]], ""]], [30, 1, ["", 0, [27, 1, [[["li", {"class": "tag-type-copyright"}, null], ["a", {}, 1]], null]], "series"]], [30, 1, ["", 0, [27, 1, [[["li", {"class": "tag-type-artist"}, null], ["a", {}, 1]], null]], "creator"]], [30, 1, ["", 0, [27, 1, [[["li", {"class": "tag-type-character"}, null], ["a", {}, 1]], null]], "character"]], [30, 1, ["we got sent back to main gallery page -- page links exist", 8, [27, 1, [[["div", {}, null]], "class"]], [true, true, "pagination"]]]]]''' ) )
+    script_info.append( ( 32, 'gelbooru md5', 2, HydrusData.GetNow(), '''["http://gelbooru.com/index.php", 0, 1, [55, 1, [[[4, "hex"]], "some hash bytes"]], "md5", {"s": "list", "page": "post"}, [[30, 6, ["we got sent back to main gallery page -- title test", 8, [27, 7, [[26, 1, [[62, 2, [0, "head", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "title", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], [true, [51, 1, [2, "Image List", null, null, "Image List"]]]]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-general"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], ""]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-copyright"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "series"]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-artist"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "creator"]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-character"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "character"]], [30, 6, ["we got sent back to main gallery page -- page links exist", 8, [27, 7, [[26, 1, [[62, 2, [0, "div", {"id": "paginator"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 2, "class", [84, 1, [26, 1, []]]]], [true, [51, 1, [3, "", null, null, "pagination"]]]]]]]''' ) )
     
     return script_info
     
diff --git a/hydrus/client/ClientDuplicates.py b/hydrus/client/ClientDuplicates.py
index 1e29743a..731178e9 100644
--- a/hydrus/client/ClientDuplicates.py
+++ b/hydrus/client/ClientDuplicates.py
@@ -174,6 +174,25 @@ class DuplicatesManager( object ):
                 
                 num_searched_estimate += num_done
                 
+                if num_searched_estimate > total_num_files:
+                    
+                    similar_files_maintenance_status = HG.client_controller.Read( 'similar_files_maintenance_status' )
+                    
+                    if similar_files_maintenance_status is None:
+                        
+                        break
+                        
+                    
+                    with self._lock:
+                        
+                        self._similar_files_maintenance_status = similar_files_maintenance_status
+                        
+                        searched_distances_to_count = self._similar_files_maintenance_status
+                        
+                        total_num_files = max( num_searched_estimate, sum( searched_distances_to_count.values() ) )
+                        
+                    
+                
                 text = 'searching: {}'.format( HydrusData.ConvertValueRangeToPrettyString( num_searched_estimate, total_num_files ) )
                 job_key.SetVariable( 'popup_text_1', text )
                 job_key.SetVariable( 'popup_gauge_1', ( num_searched_estimate, total_num_files ) )
@@ -183,7 +202,7 @@ class DuplicatesManager( object ):
                     break
                     
                 
-                time.sleep( time_it_took ) # ideally 0.5s, but potentially longer
+                time.sleep( min( 5, time_it_took ) ) # ideally 0.5s, but potentially longer
                 
             
             job_key.Delete()
diff --git a/hydrus/client/ClientServices.py b/hydrus/client/ClientServices.py
index 0b1636c2..59279401 100644
--- a/hydrus/client/ClientServices.py
+++ b/hydrus/client/ClientServices.py
@@ -2422,6 +2422,11 @@ class ServiceRepository( ServiceRestricted ):
         
         with self._lock:
             
+            if self._service_type != HC.FILE_REPOSITORY:
+                
+                return
+                
+            
             if not self._CanSyncDownload():
                 
                 return
diff --git a/hydrus/client/db/ClientDB.py b/hydrus/client/db/ClientDB.py
index 1acdc0c4..d2511843 100644
--- a/hydrus/client/db/ClientDB.py
+++ b/hydrus/client/db/ClientDB.py
@@ -34,6 +34,7 @@ from hydrus.client import ClientServices
 from hydrus.client import ClientThreading
 from hydrus.client.db import ClientDBDefinitionsCache
 from hydrus.client.db import ClientDBFilesMetadataBasic
+from hydrus.client.db import ClientDBFilesStorage
 from hydrus.client.db import ClientDBMappingsStorage
 from hydrus.client.db import ClientDBMaster
 from hydrus.client.db import ClientDBSerialisable
@@ -528,10 +529,7 @@ class DB( HydrusDB.HydrusDB ):
         
         hash_ids = { row[0] for row in rows }
         
-        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
-            
-            existing_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN current_files USING ( hash_id ) WHERE service_id = ?;'.format( temp_hash_ids_table_name ), ( service_id, ) ) )
-            
+        existing_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, hash_ids )
         
         new_hash_ids = hash_ids.difference( existing_hash_ids )
         
@@ -554,9 +552,12 @@ class DB( HydrusDB.HydrusDB ):
             
             # insert the files
             
-            self._c.executemany( 'INSERT OR IGNORE INTO current_files VALUES ( ?, ?, ? );', ( ( service_id, hash_id, timestamp ) for ( hash_id, timestamp ) in valid_rows ) )
+            pending_changed = self.modules_files_storage.AddFiles( service_id, valid_rows )
             
-            self._c.executemany( 'DELETE FROM file_transfers WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in new_hash_ids ) )
+            if pending_changed:
+                
+                self.pub_after_job( 'notify_new_pending' )
+                
             
             delta_size = self.modules_files_metadata_basic.GetTotalSize( new_hash_ids )
             num_viewable_files = self.modules_files_metadata_basic.GetNumViewable( new_hash_ids )
@@ -574,9 +575,7 @@ class DB( HydrusDB.HydrusDB ):
             
             if service_id != self.modules_services.trash_service_id:
                 
-                self._c.executemany( 'DELETE FROM deleted_files WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in new_hash_ids ) )
-                
-                num_deleted = HydrusDB.GetRowCount( self._c )
+                num_deleted = self.modules_files_storage.ClearDeleteRecord( service_id, new_hash_ids )
                 
                 service_info_updates.append( ( -num_deleted, service_id, HC.SERVICE_INFO_NUM_DELETED_FILES ) )
                 
@@ -613,6 +612,11 @@ class DB( HydrusDB.HydrusDB ):
         
         service_id = self.modules_services.AddService( service_key, service_type, name, dictionary )
         
+        if service_type in HC.SPECIFIC_FILE_SERVICES:
+            
+            self.modules_files_storage.GenerateFilesTables( service_id )
+            
+        
         if service_type in HC.REPOSITORIES:
             
             repository_updates_table_name = GenerateRepositoryUpdatesTableName( service_id )
@@ -807,13 +811,11 @@ class DB( HydrusDB.HydrusDB ):
         
         if len( hash_ids_archived ) > 0:
             
-            with HydrusDB.TemporaryIntegerTable( self._c, hash_ids_archived, 'hash_id' ) as temp_table_name:
-                
-                # temp hashes to files
-                updates = self._c.execute( 'SELECT service_id, COUNT( * ) FROM {} CROSS JOIN current_files USING ( hash_id ) GROUP BY service_id;'.format( temp_table_name ) ).fetchall()
-                
-                self._c.executemany( 'UPDATE service_info SET info = info - ? WHERE service_id = ? AND info_type = ?;', [ ( count, service_id, HC.SERVICE_INFO_NUM_INBOX ) for ( service_id, count ) in updates ] )
-                
+            service_ids_to_counts = self.modules_files_storage.GetServiceIdCounts( hash_ids_archived )
+            
+            update_rows = list( service_ids_to_counts.items() )
+            
+            self._c.executemany( 'UPDATE service_info SET info = info - ? WHERE service_id = ? AND info_type = ?;', [ ( count, service_id, HC.SERVICE_INFO_NUM_INBOX ) for ( service_id, count ) in update_rows ] )
             
         
     
@@ -1309,7 +1311,7 @@ class DB( HydrusDB.HydrusDB ):
         
         self._controller.frame_splash_status.SetSubtext( 'reading local file data' )
         
-        local_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( self.modules_services.combined_local_file_service_id, ) ) )
+        local_hash_ids = self.modules_files_storage.GetCurrentHashIdsList( self.modules_services.combined_local_file_service_id )
         
         BLOCK_SIZE = 10000
         num_to_do = len( local_hash_ids )
@@ -2494,11 +2496,13 @@ class DB( HydrusDB.HydrusDB ):
         
         #
         
-        select_statement = 'SELECT hash_id FROM current_files WHERE service_id = {};'.format( file_service_id )
+        hash_ids = self.modules_files_storage.GetCurrentHashIdsList( file_service_id )
         
-        for ( group_of_hash_ids, num_done, num_to_do ) in HydrusDB.ReadLargeIdQueryInSeparateChunks( self._c, select_statement, 10000 ):
+        BLOCK_SIZE = 10000
+        
+        for ( i, block_of_hash_ids ) in enumerate( HydrusData.SplitListIntoChunks( hash_ids, BLOCK_SIZE ) ):
             
-            self._CacheSpecificMappingsAddFiles( file_service_id, tag_service_id, group_of_hash_ids )
+            self._CacheSpecificMappingsAddFiles( file_service_id, tag_service_id, block_of_hash_ids )
             
         
         self._AnalyzeTable( cache_files_table_name )
@@ -5004,11 +5008,16 @@ class DB( HydrusDB.HydrusDB ):
             
             local_file_service_ids = self.modules_services.GetServiceIds( ( HC.LOCAL_FILE_DOMAIN, HC.LOCAL_FILE_TRASH_DOMAIN ) )
             
-            local_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id IN ' + HydrusData.SplayListForDB( local_file_service_ids ) + ';' ) )
+            local_hash_ids = set()
             
-            combined_local_file_service_id = self.modules_services.GetServiceId( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
+            for local_file_service_id in local_file_service_ids:
+                
+                some_hash_ids = self.modules_files_storage.GetCurrentHashIdsList( local_file_service_id )
+                
+                local_hash_ids.update( some_hash_ids )
+                
             
-            combined_local_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( combined_local_file_service_id, ) ) )
+            combined_local_hash_ids = set( self.modules_files_storage.GetCurrentHashIdsList( self.modules_services.combined_local_file_service_id ) )
             
             in_local_not_in_combined = local_hash_ids.difference( combined_local_hash_ids )
             in_combined_not_in_local = combined_local_hash_ids.difference( local_hash_ids )
@@ -5131,15 +5140,6 @@ class DB( HydrusDB.HydrusDB ):
         self._c.execute( 'CREATE TABLE IF NOT EXISTS ideal_client_files_locations ( location TEXT, weight INTEGER );' )
         self._c.execute( 'CREATE TABLE IF NOT EXISTS ideal_thumbnail_override_location ( location TEXT );' )
         
-        self._c.execute( 'CREATE TABLE current_files ( service_id INTEGER, hash_id INTEGER, timestamp INTEGER, PRIMARY KEY ( service_id, hash_id ) );' )
-        self._CreateIndex( 'current_files', [ 'timestamp' ] )
-        self._CreateIndex( 'current_files', [ 'hash_id' ] )
-        
-        self._c.execute( 'CREATE TABLE deleted_files ( service_id INTEGER, hash_id INTEGER, timestamp INTEGER, original_timestamp INTEGER, PRIMARY KEY ( service_id, hash_id ) );' )
-        self._CreateIndex( 'deleted_files', [ 'timestamp' ] )
-        self._CreateIndex( 'deleted_files', [ 'original_timestamp' ] )
-        self._CreateIndex( 'deleted_files', [ 'hash_id' ] )
-        
         self._c.execute( 'CREATE TABLE IF NOT EXISTS duplicate_files ( media_id INTEGER PRIMARY KEY, king_hash_id INTEGER UNIQUE );' )
         
         self._c.execute( 'CREATE TABLE IF NOT EXISTS duplicate_file_members ( media_id INTEGER, hash_id INTEGER UNIQUE, PRIMARY KEY ( media_id, hash_id ) );' )
@@ -5153,7 +5153,8 @@ class DB( HydrusDB.HydrusDB ):
         self._c.execute( 'CREATE TABLE IF NOT EXISTS confirmed_alternate_pairs ( smaller_media_id INTEGER, larger_media_id INTEGER, PRIMARY KEY ( smaller_media_id, larger_media_id ) );' )
         self._CreateIndex( 'confirmed_alternate_pairs', [ 'larger_media_id', 'smaller_media_id' ], unique = True )
         
-        self._c.execute( 'CREATE TABLE local_file_deletion_reasons ( hash_id INTEGER PRIMARY KEY, reason_id INTEGER );' )
+        self.modules_files_storage.CreateInitialTables()
+        self.modules_files_storage.CreateInitialIndices()
         
         self.modules_files_metadata_basic.CreateInitialTables()
         self.modules_files_metadata_basic.CreateInitialIndices()
@@ -5162,12 +5163,6 @@ class DB( HydrusDB.HydrusDB ):
         self._CreateIndex( 'file_notes', [ 'note_id' ] )
         self._CreateIndex( 'file_notes', [ 'name_id' ] )
         
-        self._c.execute( 'CREATE TABLE file_transfers ( service_id INTEGER, hash_id INTEGER, PRIMARY KEY ( service_id, hash_id ) );' )
-        self._CreateIndex( 'file_transfers', [ 'hash_id' ] )
-        
-        self._c.execute( 'CREATE TABLE file_petitions ( service_id INTEGER, hash_id INTEGER, reason_id INTEGER, PRIMARY KEY ( service_id, hash_id, reason_id ) );' )
-        self._CreateIndex( 'file_petitions', [ 'hash_id' ] )
-        
         self.modules_serialisable.CreateInitialTables()
         self.modules_serialisable.CreateInitialIndices()
         
@@ -5467,16 +5462,13 @@ class DB( HydrusDB.HydrusDB ):
         
         service_type = service.GetServiceType()
         
-        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
-            
-            existing_hash_ids_to_timestamps = dict( self._c.execute( 'SELECT hash_id, timestamp FROM {} CROSS JOIN current_files USING ( hash_id ) WHERE service_id = ?;'.format( temp_hash_ids_table_name ), ( service_id, ) ) )
-            
+        existing_hash_ids_to_timestamps = self.modules_files_storage.GetCurrentHashIdsToTimestamps( service_id, hash_ids )
         
         existing_hash_ids = set( existing_hash_ids_to_timestamps.keys() )
         
         service_info_updates = []
         
-        # do delete outside, we can sometimes delete without ever having added to current_files
+        # do delete outside, file repos and perhaps some other bananas situation can delete without ever having added
         
         now = HydrusData.GetNow()
         
@@ -5493,11 +5485,9 @@ class DB( HydrusDB.HydrusDB ):
             
             if len( deletee_hash_ids ) > 0:
                 
-                inserts = ( ( service_id, hash_id, now, existing_hash_ids_to_timestamps[ hash_id ] if hash_id in existing_hash_ids_to_timestamps else None ) for hash_id in hash_ids )
+                insert_rows = [ ( hash_id, existing_hash_ids_to_timestamps[ hash_id ] if hash_id in existing_hash_ids_to_timestamps else None ) for hash_id in deletee_hash_ids ]
                 
-                self._c.executemany( 'INSERT OR IGNORE INTO deleted_files ( service_id, hash_id, timestamp, original_timestamp ) VALUES ( ?, ?, ?, ? );', inserts )
-                
-                num_new_deleted_files = HydrusDB.GetRowCount( self._c )
+                num_new_deleted_files = self.modules_files_storage.RecordDeleteFiles( service_id, insert_rows )
                 
                 service_info_updates.append( ( num_new_deleted_files, service_id, HC.SERVICE_INFO_NUM_DELETED_FILES ) )
                 
@@ -5507,11 +5497,9 @@ class DB( HydrusDB.HydrusDB ):
             
             # remove them from the service
             
-            self._c.executemany( 'DELETE FROM current_files WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in existing_hash_ids ) )
+            pending_changed = self.modules_files_storage.RemoveFiles( service_id, existing_hash_ids )
             
-            self._c.executemany( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in existing_hash_ids ) )
-            
-            if HydrusDB.GetRowCount( self._c ) > 0:
+            if pending_changed:
                 
                 self.pub_after_job( 'notify_new_pending' )
                 
@@ -5544,15 +5532,14 @@ class DB( HydrusDB.HydrusDB ):
             
             if service_id in local_file_service_ids:
                 
-                with HydrusDB.TemporaryIntegerTable( self._c, existing_hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
-                    
-                    with HydrusDB.TemporaryIntegerTable( self._c, local_file_service_ids, 'service_id' ) as temp_service_ids_table_name:
-                        
-                        non_trashed_hash_ids = self._STS( self._c.execute( 'SELECT DISTINCT hash_id FROM {} CROSS JOIN current_files USING ( hash_id ) CROSS JOIN {} USING ( service_id );'.format( temp_hash_ids_table_name, temp_service_ids_table_name ) ) )
-                        
-                    
+                hash_ids_still_in_another_service = set()
                 
-                trashed_hash_ids = existing_hash_ids.difference( non_trashed_hash_ids )
+                other_local_file_service_ids = set( local_file_service_ids )
+                other_local_file_service_ids.discard( service_id )
+                
+                hash_ids_still_in_another_service = self.modules_files_storage.FilterAllCurrentHashIds( existing_hash_ids, just_these_service_ids = other_local_file_service_ids )
+                
+                trashed_hash_ids = existing_hash_ids.difference( hash_ids_still_in_another_service )
                 
                 if len( trashed_hash_ids ) > 0:
                     
@@ -5600,8 +5587,7 @@ class DB( HydrusDB.HydrusDB ):
             
         elif service.GetServiceType() in ( HC.FILE_REPOSITORY, HC.IPFS ):
             
-            self._c.execute( 'DELETE FROM file_transfers WHERE service_id = ?;', ( service_id, ) )
-            self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ?;', ( service_id, ) )
+            self.modules_files_storage.DeletePending( service_id )
             
         
         self.pub_after_job( 'notify_new_pending' )
@@ -5624,9 +5610,9 @@ class DB( HydrusDB.HydrusDB ):
         
         self._c.executemany( 'DELETE FROM file_maintenance_jobs WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in hash_ids ) )
         
-        potentially_pending_upload_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM file_transfers;', ) )
+        pending_upload_hash_ids = self.modules_files_storage.FilterAllPendingHashIds( hash_ids )
         
-        deletable_file_hash_ids = hash_ids.difference( potentially_pending_upload_hash_ids )
+        deletable_file_hash_ids = hash_ids.difference( pending_upload_hash_ids )
         
         client_files_manager = self._controller.client_files_manager
         
@@ -5637,7 +5623,7 @@ class DB( HydrusDB.HydrusDB ):
             self._controller.CallToThreadLongRunning( client_files_manager.DelayedDeleteFiles, file_hashes )
             
         
-        still_useful_thumbnail_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE hash_id IN ' + HydrusData.SplayListForDB( hash_ids ) + ';' ) )
+        still_useful_thumbnail_hash_ids = self.modules_files_storage.FilterAllCurrentHashIds( hash_ids )
         
         deletable_thumbnail_hash_ids = hash_ids.difference( still_useful_thumbnail_hash_ids )
         
@@ -5660,10 +5646,6 @@ class DB( HydrusDB.HydrusDB ):
         # however, this seemed to cause some immense temp drive space bloat when dropping the mapping tables, as there seems to be a trigger/foreign reference check for every row to be deleted
         # so now we just blat all tables and trust in the Lord that we don't forget to add any new ones in future
         
-        self._c.execute( 'DELETE FROM current_files WHERE service_id = ?;', ( service_id, ) )
-        self._c.execute( 'DELETE FROM deleted_files WHERE service_id = ?;', ( service_id, ) )
-        self._c.execute( 'DELETE FROM file_transfers WHERE service_id = ?;', ( service_id, ) )
-        self._c.execute( 'DELETE FROM file_petitions WHERE service_id = ?;', ( service_id, ) )
         self._c.execute( 'DELETE FROM local_ratings WHERE service_id = ?;', ( service_id, ) )
         self._c.execute( 'DELETE FROM recent_tags WHERE service_id = ?;', ( service_id, ) )
         self._c.execute( 'DELETE FROM remote_thumbnails WHERE service_id = ?;', ( service_id, ) )
@@ -5676,6 +5658,11 @@ class DB( HydrusDB.HydrusDB ):
         self._c.execute( 'DELETE FROM tag_siblings WHERE service_id = ?;', ( service_id, ) )
         self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ?;', ( service_id, ) )
         
+        if service_type in HC.SPECIFIC_FILE_SERVICES:
+            
+            self.modules_files_storage.DropFilesTables( service_id )
+            
+        
         if service_type in HC.REPOSITORIES:
             
             repository_updates_table_name = GenerateRepositoryUpdatesTableName( service_id )
@@ -6252,7 +6239,9 @@ class DB( HydrusDB.HydrusDB ):
             
         else:
             
-            hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM duplicate_file_members NATURAL JOIN current_files WHERE media_id = ? AND service_id = ?;', ( media_id, file_service_id ) ) )
+            table_join = self.modules_files_storage.GetCurrentTableJoinPhrase( file_service_id, 'duplicate_file_members' )
+            
+            hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} WHERE media_id = ?;'.format( table_join ), ( media_id, ) ) )
             
         
         return hash_ids
@@ -6287,9 +6276,9 @@ class DB( HydrusDB.HydrusDB ):
         
         if media_id is not None:
             
-            ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+            table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
             
-            ( num_potentials, ) = self._c.execute( 'SELECT COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ' AND ( smaller_media_id = ? OR larger_media_id = ? ) );', ( media_id, media_id, ) ).fetchone()
+            ( num_potentials, ) = self._c.execute( 'SELECT COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM {} WHERE smaller_media_id = ? OR larger_media_id = ? );'.format( table_join ), ( media_id, media_id, ) ).fetchone()
             
             if num_potentials > 0:
                 
@@ -6468,9 +6457,9 @@ class DB( HydrusDB.HydrusDB ):
             
             if media_id is not None:
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
                 
-                for ( smaller_media_id, larger_media_id ) in self._c.execute( 'SELECT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ' AND ( smaller_media_id = ? OR larger_media_id = ? );', ( media_id, media_id ) ).fetchall():
+                for ( smaller_media_id, larger_media_id ) in self._c.execute( 'SELECT smaller_media_id, larger_media_id FROM {} WHERE smaller_media_id = ? OR larger_media_id = ?;'.format( table_join ), ( media_id, media_id ) ).fetchall():
                     
                     if smaller_media_id != media_id:
                         
@@ -6644,15 +6633,12 @@ class DB( HydrusDB.HydrusDB ):
                 
                 table_join = 'duplicate_file_members'
                 
-                predicate_string = '1=1'
-                
             else:
                 
-                table_join = 'duplicate_file_members NATURAL JOIN current_files'
-                predicate_string = 'service_id = {}'.format( file_service_id )
+                table_join = self.modules_files_storage.GetCurrentTableJoinPhrase( file_service_id, 'duplicate_file_members' )
                 
             
-            query = 'SELECT media_id, COUNT( * ) FROM {} WHERE {} GROUP BY media_id;'.format( table_join, predicate_string )
+            query = 'SELECT media_id, COUNT( * ) FROM {} GROUP BY media_id;'.format( table_join )
             
             media_ids = []
             
@@ -6672,10 +6658,10 @@ class DB( HydrusDB.HydrusDB ):
             
         elif dupe_type == HC.DUPLICATE_POTENTIAL:
             
-            ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+            table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
             
-            smaller_query = 'SELECT smaller_media_id, COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ' ) GROUP BY smaller_media_id;'
-            larger_query = 'SELECT larger_media_id, COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ' ) GROUP BY larger_media_id;'
+            smaller_query = 'SELECT smaller_media_id, COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM {} ) GROUP BY smaller_media_id;'.format( table_join )
+            larger_query = 'SELECT larger_media_id, COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM {} ) GROUP BY larger_media_id;'.format( table_join )
             
             file_service_id = self.modules_services.GetServiceId( file_service_key )
             
@@ -6736,40 +6722,45 @@ class DB( HydrusDB.HydrusDB ):
         return media_id
         
     
-    def _DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( self, file_service_key ):
+    def _DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( self, file_service_key ):
         
         if file_service_key == CC.COMBINED_FILE_SERVICE_KEY:
             
             table_join = 'potential_duplicate_pairs'
-            predicate_string = '1=1'
             
         else:
             
             service_id = self.modules_services.GetServiceId( file_service_key )
             
-            table_join = 'potential_duplicate_pairs, duplicate_file_members AS duplicate_file_members_smaller, current_files AS current_files_smaller, duplicate_file_members AS duplicate_file_members_larger, current_files AS current_files_larger ON ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = current_files_smaller.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = current_files_larger.hash_id )'
-            predicate_string = 'current_files_smaller.service_id = {} AND current_files_larger.service_id = {}'.format( service_id, service_id )
+            current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+            
+            table_join = 'potential_duplicate_pairs, duplicate_file_members AS duplicate_file_members_smaller, {} AS current_files_smaller, duplicate_file_members AS duplicate_file_members_larger, {} AS current_files_larger ON ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = current_files_smaller.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = current_files_larger.hash_id )'.format( current_files_table_name, current_files_table_name )
             
         
-        return ( table_join, predicate_string )
+        return table_join
         
     
-    def _DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnSearchResults( self, file_service_key, results_table_name, both_files_match ):
+    def _DuplicatesGetPotentialDuplicatePairsTableJoinOnSearchResults( self, file_service_key, results_table_name, both_files_match ):
+        
+        if file_service_key == CC.COMBINED_FILE_SERVICE_KEY:
+            
+            file_service_key = CC.COMBINED_LOCAL_FILE_SERVICE_KEY
+            
         
         if both_files_match:
             
             table_join = 'potential_duplicate_pairs, duplicate_file_members AS duplicate_file_members_smaller, {} AS results_smaller, duplicate_file_members AS duplicate_file_members_larger, {} AS results_larger ON ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = results_smaller.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = results_larger.hash_id )'.format( results_table_name, results_table_name )
-            predicate_string = '1=1'
             
         else:
             
             service_id = self.modules_services.GetServiceId( file_service_key )
             
-            table_join = 'potential_duplicate_pairs, duplicate_file_members AS duplicate_file_members_smaller, duplicate_file_members AS duplicate_file_members_larger, {}, current_files ON ( ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = {}.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = current_files.hash_id ) OR ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = current_files.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = {}.hash_id ) )'.format( results_table_name, results_table_name, results_table_name )
-            predicate_string = 'current_files.service_id = {}'.format( service_id )
+            current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+            
+            table_join = 'potential_duplicate_pairs, duplicate_file_members AS duplicate_file_members_smaller, duplicate_file_members AS duplicate_file_members_larger, {} AS results_table_for_this_query, {} AS current_files_for_this_query ON ( ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = results_table_for_this_query.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = current_files_for_this_query.hash_id ) OR ( smaller_media_id = duplicate_file_members_smaller.media_id AND duplicate_file_members_smaller.hash_id = current_files_for_this_query.hash_id AND larger_media_id = duplicate_file_members_larger.media_id AND duplicate_file_members_larger.hash_id = results_table_for_this_query.hash_id ) )'.format( results_table_name, current_files_table_name )
             
         
-        return ( table_join, predicate_string )
+        return table_join
         
     
     def _DuplicatesGetRandomPotentialDuplicateHashes( self, file_search_context, both_files_match ):
@@ -6789,7 +6780,7 @@ class DB( HydrusDB.HydrusDB ):
             
             if file_search_context.IsJustSystemEverything() or file_search_context.HasNoPredicates():
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
                 
             else:
                 
@@ -6810,13 +6801,13 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._AnalyzeTempTable( temp_table_name )
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnSearchResults( file_service_key, temp_table_name, both_files_match )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnSearchResults( file_service_key, temp_table_name, both_files_match )
                 
             
             potential_media_ids = set()
             
             # distinct important here for the search results table join
-            for ( smaller_media_id, larger_media_id ) in self._c.execute( 'SELECT DISTINCT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ';' ):
+            for ( smaller_media_id, larger_media_id ) in self._c.execute( 'SELECT DISTINCT smaller_media_id, larger_media_id FROM {};'.format( table_join ) ):
                 
                 potential_media_ids.add( smaller_media_id )
                 potential_media_ids.add( larger_media_id )
@@ -6885,7 +6876,7 @@ class DB( HydrusDB.HydrusDB ):
             
             if file_search_context.IsJustSystemEverything() or file_search_context.HasNoPredicates():
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
                 
             else:
                 
@@ -6904,11 +6895,11 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._AnalyzeTempTable( temp_table_name )
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnSearchResults( file_service_key, temp_table_name, both_files_match )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnSearchResults( file_service_key, temp_table_name, both_files_match )
                 
             
             # distinct important here for the search results table join
-            result = self._c.execute( 'SELECT DISTINCT smaller_media_id, larger_media_id, distance FROM ' + table_join + ' WHERE ' + predicate_string + ' LIMIT 2500;' ).fetchall()
+            result = self._c.execute( 'SELECT DISTINCT smaller_media_id, larger_media_id, distance FROM {} LIMIT 2500;'.format( table_join ) ).fetchall()
             
         
         MAX_BATCH_SIZE = HG.client_controller.new_options.GetInteger( 'duplicate_filter_max_batch_size' )
@@ -7025,7 +7016,7 @@ class DB( HydrusDB.HydrusDB ):
             
             if file_search_context.IsJustSystemEverything() or file_search_context.HasNoPredicates():
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( file_service_key )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( file_service_key )
                 
             else:
                 
@@ -7035,11 +7026,11 @@ class DB( HydrusDB.HydrusDB ):
                 
                 self._AnalyzeTempTable( temp_table_name )
                 
-                ( table_join, predicate_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnSearchResults( file_service_key, temp_table_name, both_files_match )
+                table_join = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnSearchResults( file_service_key, temp_table_name, both_files_match )
                 
             
             # distinct important here for the search results table join
-            ( potential_duplicates_count, ) = self._c.execute( 'SELECT COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM ' + table_join + ' WHERE ' + predicate_string + ' );', ).fetchone()
+            ( potential_duplicates_count, ) = self._c.execute( 'SELECT COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM {} );'.format( table_join ) ).fetchone()
             
         
         return potential_duplicates_count
@@ -7846,25 +7837,11 @@ class DB( HydrusDB.HydrusDB ):
         
         hashes_to_hash_ids = { hash : self.modules_hashes_local_cache.GetHashId( hash ) for hash in hashes if self._HashExists( hash ) }
         
-        valid_hash_ids = self._FilterHashIdsByFileServiceId( service_id, set( hashes_to_hash_ids.values() ) )
+        valid_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, set( hashes_to_hash_ids.values() ) )
         
         return [ hash for hash in hashes if hash in hashes_to_hash_ids and hashes_to_hash_ids[ hash ] in valid_hash_ids ]
         
     
-    def _FilterHashIdsByFileServiceId( self, service_id: int, hash_ids: typing.Collection[ int ] ) -> typing.Set[ int ]:
-        
-        if service_id == self.modules_services.combined_file_service_id:
-            
-            return set( hash_ids )
-            
-        
-        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_table_name:
-            
-            # temp hashes to files
-            return self._STS( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN current_files USING ( hash_id ) WHERE service_id = ?;'.format( temp_table_name ), ( service_id, ) ) )
-            
-        
-    
     def _FixLogicallyInconsistentMappings( self, tag_service_key = None ):
         
         job_key = ClientThreading.JobKey( cancellable = True )
@@ -8687,8 +8664,10 @@ class DB( HydrusDB.HydrusDB ):
         
         boned_stats = {}
         
-        ( num_total, size_total ) = self._c.execute( 'SELECT COUNT( hash_id ), SUM( size ) FROM files_info NATURAL JOIN current_files WHERE service_id = ?;', ( self.modules_services.combined_local_file_service_id, ) ).fetchone()
-        ( num_inbox, size_inbox ) = self._c.execute( 'SELECT COUNT( hash_id ), SUM( size ) FROM files_info NATURAL JOIN current_files NATURAL JOIN file_inbox WHERE service_id = ?;', ( self.modules_services.combined_local_file_service_id, ) ).fetchone()
+        current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( self.modules_services.combined_local_file_service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        ( num_total, size_total ) = self._c.execute( 'SELECT COUNT( hash_id ), SUM( size ) FROM {} CROSS JOIN files_info USING ( hash_id );'.format( current_files_table_name ) ).fetchone()
+        ( num_inbox, size_inbox ) = self._c.execute( 'SELECT COUNT( hash_id ), SUM( size ) FROM files_info NATURAL JOIN {} NATURAL JOIN file_inbox;'.format( current_files_table_name ) ).fetchone()
         
         if size_total is None:
             
@@ -8729,7 +8708,7 @@ class DB( HydrusDB.HydrusDB ):
         total_alternate_files = sum( ( count for ( alternates_group_id, count ) in self._c.execute( 'SELECT alternates_group_id, COUNT( * ) FROM alternate_file_group_members GROUP BY alternates_group_id;' ) if count > 1 ) )
         total_duplicate_files = sum( ( count for ( media_id, count ) in self._c.execute( 'SELECT media_id, COUNT( * ) FROM duplicate_file_members GROUP BY media_id;' ) if count > 1 ) )
         
-        ( table_join, predicates_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinInfoOnFileService( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
+        ( table_join, predicates_string ) = self._DuplicatesGetPotentialDuplicatePairsTableJoinOnFileService( CC.COMBINED_LOCAL_FILE_SERVICE_KEY )
         
         ( total_potential_pairs, ) = self._c.execute( 'SELECT COUNT( * ) FROM ( SELECT DISTINCT smaller_media_id, larger_media_id FROM {} WHERE {} );'.format( table_join, predicates_string ) ).fetchone()
         
@@ -8827,7 +8806,7 @@ class DB( HydrusDB.HydrusDB ):
             
             if service_type == HC.FILE_REPOSITORY:
                 
-                ( num_local, ) = self._c.execute( 'SELECT COUNT( * ) FROM current_files AS remote_current_files, current_files ON ( remote_current_files.hash_id = current_files.hash_id ) WHERE remote_current_files.service_id = ? AND current_files.service_id = ?;', ( service_id, self.modules_services.combined_local_file_service_id ) ).fetchone()
+                num_local = self.modules_files_storage.GetNumLocal( service_id )
                 
                 num_not_local = num_everything - num_local
                 
@@ -8909,8 +8888,7 @@ class DB( HydrusDB.HydrusDB ):
         
         if hash_ids_to_current_file_service_ids is None:
             
-            # temp hashes to files
-            hash_ids_to_current_file_service_ids = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT hash_id, service_id FROM {} CROSS JOIN current_files USING ( hash_id );'.format( hash_ids_table_name ) ) )
+            hash_ids_to_current_file_service_ids = self.modules_files_storage.GetHashIdsToCurrentServiceIds( hash_ids_table_name )
             
         
         common_file_service_ids_to_hash_ids = self._GroupHashIdsByTagCachedFileServiceId( hash_ids, hash_ids_table_name, hash_ids_to_current_file_service_ids = hash_ids_to_current_file_service_ids )
@@ -9510,20 +9488,13 @@ class DB( HydrusDB.HydrusDB ):
         
         def do_or_preds( or_predicates, query_hash_ids ):
             
-            # updating all this regular search code to do OR and AND naturally will be a big job getting right, so let's get a functional inefficient solution and then optimise later as needed
-            # -tag stuff and various other exclude situations remain a pain to do quickly assuming OR
-            # the future extension of this will be creating an OR_search_context with all the OR_pred's subpreds and have that naturally query_hash_ids.update throughout this func based on file_search_context search_type
-            # this func is called at one of several potential points, kicking in if query_hash_ids are needed but preferring tags or system preds to step in
-            
-            or_predicates = list( or_predicates )
-            
             # better typically to sort by fewest num of preds first, establishing query_hash_ids for longer chains
             def or_sort_key( p ):
                 
                 return len( p.GetValue() )
                 
             
-            or_predicates.sort( key = or_sort_key )
+            or_predicates = sorted( or_predicates, key = or_sort_key )
             
             for or_predicate in or_predicates:
                 
@@ -9540,7 +9511,7 @@ class DB( HydrusDB.HydrusDB ):
                     or_search_context.SetPredicates( [ or_subpredicate ] )
                     
                     # I pass current query_hash_ids here to make these inefficient sub-searches (like -tag) potentially much faster
-                    or_query_hash_ids.update( self._GetHashIdsFromQuery( or_search_context, job_key, query_hash_ids = query_hash_ids ) )
+                    or_query_hash_ids.update( self._GetHashIdsFromQuery( or_search_context, job_key, query_hash_ids = query_hash_ids, apply_implicit_limit = False, sort_by = None, limit_sort_by = None ) )
                     
                     if job_key.IsCancelled():
                         
@@ -9556,17 +9527,14 @@ class DB( HydrusDB.HydrusDB ):
         
         #
         
-        done_or_predicates = False
+        done_or_predicates = len( or_predicates ) == 0
         
         # OR round one--if nothing else will be fast, let's prep query_hash_ids now
-        if not ( there_are_tags_to_search or there_are_simple_files_info_preds_to_search_for ):
+        if not done_or_predicates and not ( there_are_tags_to_search or there_are_simple_files_info_preds_to_search_for ):
             
-            if len( or_predicates ) > 0:
-                
-                query_hash_ids = do_or_preds( or_predicates, query_hash_ids )
-                
-                have_cross_referenced_file_service = True
-                
+            query_hash_ids = do_or_preds( or_predicates, query_hash_ids )
+            
+            have_cross_referenced_file_service = True
             
             done_or_predicates = True
             
@@ -9606,7 +9574,9 @@ class DB( HydrusDB.HydrusDB ):
                 
                 pred_string = ' AND '.join( import_timestamp_predicates )
                 
-                import_timestamp_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ? AND {};'.format( pred_string ), ( file_service_id, ) ) )
+                current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( file_service_id, HC.CONTENT_STATUS_CURRENT )
+                
+                import_timestamp_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} WHERE {};'.format( current_files_table_name, pred_string ) ) )
                 
                 query_hash_ids = intersection_update_qhi( query_hash_ids, import_timestamp_hash_ids )
                 
@@ -9860,14 +9830,11 @@ class DB( HydrusDB.HydrusDB ):
         #
         
         # OR round two--if file preds will not be fast, let's step in to reduce the file domain search space
-        if not ( there_are_simple_files_info_preds_to_search_for or done_or_predicates ):
+        if not done_or_predicates and not there_are_simple_files_info_preds_to_search_for:
             
-            if len( or_predicates ) > 0:
-                
-                query_hash_ids = do_or_preds( or_predicates, query_hash_ids )
-                
-                have_cross_referenced_file_service = True
-                
+            query_hash_ids = do_or_preds( or_predicates, query_hash_ids )
+            
+            have_cross_referenced_file_service = True
             
             done_or_predicates = True
             
@@ -9887,17 +9854,22 @@ class DB( HydrusDB.HydrusDB ):
                 
             else:
                 
-                files_info_predicates.insert( 0, 'service_id = ' + str( file_service_id ) )
+                current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( file_service_id, HC.CONTENT_STATUS_CURRENT )
+                
+                if len( files_info_predicates ) == 0:
+                    
+                    files_info_predicates.insert( 0, '1=1' )
+                    
                 
                 if query_hash_ids is None:
                     
-                    query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM current_files NATURAL JOIN files_info WHERE {};'.format( ' AND '.join( files_info_predicates ) ) ) ) )
+                    query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN files_info WHERE {};'.format( current_files_table_name, ' AND '.join( files_info_predicates ) ) ) ) )
                     
                 else:
                     
                     if is_inbox and len( query_hash_ids ) == len( self.modules_files_metadata_basic.inbox_hash_ids ):
                         
-                        query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN current_files NATURAL JOIN files_info WHERE {};'.format( 'file_inbox', ' AND '.join( files_info_predicates ) ) ) ) )
+                        query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN {} NATURAL JOIN files_info WHERE {};'.format( 'file_inbox', current_files_table_name, ' AND '.join( files_info_predicates ) ) ) ) )
                         
                     else:
                         
@@ -9905,7 +9877,7 @@ class DB( HydrusDB.HydrusDB ):
                             
                             self._AnalyzeTempTable( temp_table_name )
                             
-                            query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN current_files NATURAL JOIN files_info WHERE {};'.format( temp_table_name, ' AND '.join( files_info_predicates ) ) ) ) )
+                            query_hash_ids = intersection_update_qhi( query_hash_ids, self._STS( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN {} NATURAL JOIN files_info WHERE {};'.format( temp_table_name, current_files_table_name, ' AND '.join( files_info_predicates ) ) ) ) )
                             
                         
                     
@@ -9966,7 +9938,7 @@ class DB( HydrusDB.HydrusDB ):
         
         if location_search_context.IsAllLocalFiles():
             
-            repo_update_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( self.modules_services.local_update_service_id, ) ) )
+            repo_update_hash_ids = set( self.modules_files_storage.GetCurrentHashIdsList( self.modules_services.local_update_service_id ) )
             
             query_hash_ids.difference_update( repo_update_hash_ids )
             
@@ -10037,28 +10009,36 @@ class DB( HydrusDB.HydrusDB ):
             
             service_id = self.modules_services.GetServiceId( service_key )
             
-            query_hash_ids = intersection_update_qhi( query_hash_ids, self._STI( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( service_id, ) ) ) )
+            current_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, query_hash_ids )
+            
+            query_hash_ids = intersection_update_qhi( query_hash_ids, current_hash_ids )
             
         
         for service_key in file_services_to_include_pending:
             
             service_id = self.modules_services.GetServiceId( service_key )
             
-            query_hash_ids = intersection_update_qhi( query_hash_ids, self._STI( self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ) ) )
+            pending_hash_ids = self.modules_files_storage.FilterPendingHashIds( service_id, query_hash_ids )
+            
+            query_hash_ids = intersection_update_qhi( query_hash_ids, pending_hash_ids )
             
         
         for service_key in file_services_to_exclude_current:
             
             service_id = self.modules_services.GetServiceId( service_key )
             
-            query_hash_ids.difference_update( self._STI( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( service_id, ) ) ) )
+            current_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, query_hash_ids )
+            
+            query_hash_ids.difference_update( current_hash_ids )
             
         
         for service_key in file_services_to_exclude_pending:
             
             service_id = self.modules_services.GetServiceId( service_key )
             
-            query_hash_ids.difference_update( self._STI( self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ) ) )
+            pending_hash_ids = self.modules_files_storage.FilterPendingHashIds( service_id, query_hash_ids )
+            
+            query_hash_ids.difference_update( pending_hash_ids )
             
         
         #
@@ -10215,7 +10195,7 @@ class DB( HydrusDB.HydrusDB ):
             
         elif must_be_local or must_not_be_local:
             
-            local_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( self.modules_services.combined_local_file_service_id, ) ) )
+            local_hash_ids = self.modules_files_storage.GetCurrentHashIdsList( self.modules_services.combined_local_file_service_id )
             
             if must_be_local:
                 
@@ -10807,26 +10787,11 @@ class DB( HydrusDB.HydrusDB ):
             prefix += ': '
             
         
-        result = self._c.execute( 'SELECT reason_id FROM local_file_deletion_reasons WHERE hash_id = ?;', ( hash_id, ) ).fetchone()
-        
-        if result is None:
-            
-            file_deletion_reason = 'Unknown deletion reason.'
-            
-        else:
-            
-            ( reason_id, ) = result
-            
-            file_deletion_reason = self.modules_texts.GetText( reason_id )
-            
-        
         hash = self.modules_hashes_local_cache.GetHash( hash_id )
         
-        result = self._c.execute( 'SELECT timestamp FROM deleted_files WHERE service_id = ? AND hash_id = ?;', ( self.modules_services.combined_local_file_service_id, hash_id ) ).fetchone()
+        ( is_deleted, timestamp, file_deletion_reason ) = self.modules_files_storage.GetDeletionStatus( self.modules_services.combined_local_file_service_id, hash_id )
         
-        if result is not None:
-            
-            ( timestamp, ) = result
+        if is_deleted:
             
             if timestamp is None:
                 
@@ -10840,22 +10805,22 @@ class DB( HydrusDB.HydrusDB ):
             return ClientImportFiles.FileImportStatus( CC.STATUS_DELETED, hash, note = prefix + note )
             
         
-        result = self._c.execute( 'SELECT timestamp FROM current_files WHERE service_id = ? AND hash_id = ?;', ( self.modules_services.trash_service_id, hash_id ) ).fetchone()
+        result = self.modules_files_storage.GetCurrentTimestamp( self.modules_services.trash_service_id, hash_id )
         
         if result is not None:
             
-            ( timestamp, ) = result
+            timestamp = result
             
             note = 'Currently in trash ({}). Sent there at {}, which was {} before this check.'.format( file_deletion_reason, HydrusData.ConvertTimestampToPrettyTime( timestamp ), ClientData.TimestampToPrettyTimeDelta( timestamp, just_now_threshold = 0 ) )
             
             return ClientImportFiles.FileImportStatus( CC.STATUS_DELETED, hash, note = prefix + note )
             
         
-        result = self._c.execute( 'SELECT timestamp FROM current_files WHERE service_id = ? AND hash_id = ?;', ( self.modules_services.combined_local_file_service_id, hash_id ) ).fetchone()
+        result = self.modules_files_storage.GetCurrentTimestamp( self.modules_services.combined_local_file_service_id, hash_id )
         
         if result is not None:
             
-            ( timestamp, ) = result
+            timestamp = result
             
             mime = self.modules_files_metadata_basic.GetMime( hash_id )
             
@@ -11203,13 +11168,11 @@ class DB( HydrusDB.HydrusDB ):
                 
                 hash_ids_to_info = { hash_id : ClientMediaManagers.FileInfoManager( hash_id, missing_hash_ids_to_hashes[ hash_id ], size, mime, width, height, duration, num_frames, has_audio, num_words ) for ( hash_id, size, mime, width, height, duration, num_frames, has_audio, num_words ) in self._c.execute( 'SELECT * FROM {} CROSS JOIN files_info USING ( hash_id );'.format( temp_table_name ) ) }
                 
-                hash_ids_to_current_file_service_ids_and_timestamps = HydrusData.BuildKeyToListDict( ( ( hash_id, ( service_id, timestamp ) ) for ( hash_id, service_id, timestamp ) in self._c.execute( 'SELECT hash_id, service_id, timestamp FROM {} CROSS JOIN current_files USING ( hash_id );'.format( temp_table_name ) ) ) )
-                
-                hash_ids_to_deleted_file_service_ids_and_timestamps = HydrusData.BuildKeyToListDict( ( ( hash_id, ( service_id, timestamp, original_timestamp ) ) for ( hash_id, service_id, timestamp, original_timestamp ) in self._c.execute( 'SELECT hash_id, service_id, timestamp, original_timestamp FROM {} CROSS JOIN deleted_files USING ( hash_id );'.format( temp_table_name ) ) ) )
-                
-                hash_ids_to_pending_file_service_ids = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT hash_id, service_id FROM {} CROSS JOIN file_transfers USING ( hash_id );'.format( temp_table_name ) ) )
-                
-                hash_ids_to_petitioned_file_service_ids = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT hash_id, service_id FROM {} CROSS JOIN file_petitions USING ( hash_id );'.format( temp_table_name ) ) )
+                ( hash_ids_to_current_file_service_ids_and_timestamps,
+                  hash_ids_to_deleted_file_service_ids_and_timestamps,
+                  hash_ids_to_pending_file_service_ids,
+                  hash_ids_to_petitioned_file_service_ids
+                ) = self.modules_files_storage.GetHashIdsToServiceInfoDicts( temp_table_name )
                 
                 hash_ids_to_urls = HydrusData.BuildKeyToSetDict( self._c.execute( 'SELECT hash_id, url FROM {} CROSS JOIN url_map USING ( hash_id ) CROSS JOIN urls USING ( url_id );'.format( temp_table_name ) ) )
                 
@@ -11595,11 +11558,11 @@ class DB( HydrusDB.HydrusDB ):
                     
                     if account.HasPermission( HC.CONTENT_TYPE_FILES, HC.PERMISSION_ACTION_CREATE ):
                         
-                        result = self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ).fetchone()
+                        result = self.modules_files_storage.GetAPendingHashId( service_id )
                         
                         if result is not None:
                             
-                            ( hash_id, ) = result
+                            hash_id = result
                             
                             media_result = self._GetMediaResults( ( hash_id, ) )[ 0 ]
                             
@@ -11609,9 +11572,9 @@ class DB( HydrusDB.HydrusDB ):
                     
                     if account.HasPermission( HC.CONTENT_TYPE_FILES, HC.PERMISSION_ACTION_PETITION ):
                         
-                        petitioned = list( HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT reason_id, hash_id FROM file_petitions WHERE service_id = ? ORDER BY reason_id LIMIT 100;', ( service_id, ) ) ).items() )
+                        petitioned_rows = self.modules_files_storage.GetSomePetitionedRows( service_id )
                         
-                        for ( reason_id, hash_ids ) in petitioned:
+                        for ( reason_id, hash_ids ) in petitioned_rows:
                             
                             hashes = self.modules_hashes_local_cache.GetHashes( hash_ids )
                             
@@ -11632,11 +11595,11 @@ class DB( HydrusDB.HydrusDB ):
             
         elif service_type == HC.IPFS:
             
-            result = self._c.execute( 'SELECT hash_id FROM file_transfers WHERE service_id = ?;', ( service_id, ) ).fetchone()
+            result = self.modules_files_storage.GetAPendingHashId( service_id )
             
             if result is not None:
                 
-                ( hash_id, ) = result
+                hash_id = result
                 
                 media_result = self._GetMediaResults( ( hash_id, ) )[ 0 ]
                 
@@ -11645,7 +11608,7 @@ class DB( HydrusDB.HydrusDB ):
             
             while True:
                 
-                result = self._c.execute( 'SELECT hash_id FROM file_petitions WHERE service_id = ?;', ( service_id, ) ).fetchone()
+                result = self.modules_files_storage.GetAPetitionedHashId( service_id )
                 
                 if result is None:
                     
@@ -11653,7 +11616,7 @@ class DB( HydrusDB.HydrusDB ):
                     
                 else:
                     
-                    ( hash_id, ) = result
+                    hash_id = result
                     
                     hash = self.modules_hashes_local_cache.GetHash( hash_id )
                     
@@ -11834,7 +11797,9 @@ class DB( HydrusDB.HydrusDB ):
         
         ( num_processed_updates, ) = self._c.execute( 'SELECT COUNT( * ) FROM ' + repository_updates_table_name + ' WHERE processed = ?;', ( True, ) ).fetchone()
         
-        ( num_local_updates, ) = self._c.execute( 'SELECT COUNT( * ) FROM current_files NATURAL JOIN ' + repository_updates_table_name + ' WHERE service_id = ?;', ( self.modules_services.local_update_service_id, ) ).fetchone()
+        table_join = self.modules_files_storage.GetCurrentTableJoinPhrase( self.modules_services.local_update_service_id, repository_updates_table_name )
+        
+        ( num_local_updates, ) = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( table_join ) ).fetchone()
         
         return ( num_local_updates, num_processed_updates, num_updates )
         
@@ -11843,7 +11808,9 @@ class DB( HydrusDB.HydrusDB ):
         
         service_id = self.modules_services.GetServiceId( service_key )
         
-        needed_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM current_files NATURAL JOIN files_info WHERE mime IN ' + HydrusData.SplayListForDB( HC.MIMES_WITH_THUMBNAILS ) + ' and service_id = ? EXCEPT SELECT hash_id FROM remote_thumbnails WHERE service_id = ?;', ( service_id, service_id ) ) )
+        current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        needed_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM {} NATURAL JOIN files_info WHERE mime IN {} EXCEPT SELECT hash_id FROM remote_thumbnails WHERE service_id = ?;'.format( current_files_table_name, HydrusData.SplayListForDB( HC.MIMES_WITH_THUMBNAILS ) ), ( service_id, ) ) )
         
         needed_hashes = []
         
@@ -11897,10 +11864,7 @@ class DB( HydrusDB.HydrusDB ):
         
         if len( unprocessed_hash_ids ) > 0:
             
-            with HydrusDB.TemporaryIntegerTable( self._c, unprocessed_hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
-                
-                local_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN current_files USING ( hash_id ) WHERE service_id = ?;'.format( temp_hash_ids_table_name ), ( self.modules_services.local_update_service_id, ) ) )
-                
+            local_hash_ids = self.modules_files_storage.FilterCurrentHashIds( self.modules_services.local_update_service_id, unprocessed_hash_ids )
             
             hash_ids_i_can_process = []
             
@@ -11968,7 +11932,9 @@ class DB( HydrusDB.HydrusDB ):
         
         desired_hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM {} ORDER BY update_index ASC;'.format( repository_updates_table_name ) ) )
         
-        existing_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?;', ( self.modules_services.local_update_service_id, ) ) )
+        table_join = self.modules_files_storage.GetCurrentTableJoinPhrase( self.modules_services.local_update_service_id, repository_updates_table_name )
+        
+        existing_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {};'.format( table_join ) ) )
         
         needed_hash_ids = [ hash_id for hash_id in desired_hash_ids if hash_id not in existing_hash_ids ]
         
@@ -12097,6 +12063,9 @@ class DB( HydrusDB.HydrusDB ):
             
             for info_type in info_types_missed:
                 
+                info = None
+                result = None
+                
                 save_it = True
                 
                 if service_type in HC.FILE_SERVICES:
@@ -12106,77 +12075,117 @@ class DB( HydrusDB.HydrusDB ):
                         save_it = False
                         
                     
-                    if info_type == HC.SERVICE_INFO_NUM_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM current_files WHERE service_id = ?;', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_VIEWABLE_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM current_files NATURAL JOIN files_info WHERE service_id = ? AND mime IN ' + HydrusData.SplayListForDB( HC.SEARCHABLE_MIMES ) + ';', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_TOTAL_SIZE: result = self._c.execute( 'SELECT SUM( size ) FROM current_files NATURAL JOIN files_info WHERE service_id = ?;', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_DELETED_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM deleted_files WHERE service_id = ?;', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM file_transfers WHERE service_id = ?;', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM file_petitions where service_id = ?;', ( service_id, ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_INBOX: result = self._c.execute( 'SELECT COUNT( * ) FROM file_inbox NATURAL JOIN current_files WHERE service_id = ?;', ( service_id, ) ).fetchone()
+                    if info_type == HC.SERVICE_INFO_NUM_FILES:
+                        
+                        info = self.modules_files_storage.GetCurrentFilesCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_VIEWABLE_FILES:
+                        
+                        info = self.modules_files_storage.GetCurrentFilesCount( service_id, only_viewable = True )
+                        
+                    elif info_type == HC.SERVICE_INFO_TOTAL_SIZE:
+                        
+                        info = self.modules_files_storage.GetCurrentFilesTotalSize( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_DELETED_FILES:
+                        
+                        info = self.modules_files_storage.GetDeletedFilesCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_FILES:
+                        
+                        info = self.modules_files_storage.GetPendingFilesCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_FILES:
+                        
+                        info = self.modules_files_storage.GetPetitionedFilesCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_INBOX:
+                        
+                        info = self.modules_files_storage.GetCurrentFilesInboxCount( service_id )
+                        
                     
                 elif service_type in HC.REAL_TAG_SERVICES:
                     
-                    ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = ClientDBMappingsStorage.GenerateMappingsTableNames( service_id )
-                    
                     if info_type in ( HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS, HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS, HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS, HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS ):
                         
                         save_it = False
                         
                     
-                    if info_type == HC.SERVICE_INFO_NUM_FILES: result = self._c.execute( 'SELECT COUNT( DISTINCT hash_id ) FROM ' + current_mappings_table_name + ';' ).fetchone()
+                    if info_type == HC.SERVICE_INFO_NUM_FILES:
+                        
+                        info = self.modules_mappings_storage.GetCurrentFilesCount( service_id )
+                        
                     elif info_type == HC.SERVICE_INFO_NUM_TAGS:
                         
                         tags_table_name = self._CacheTagsGetTagsTableName( self.modules_services.combined_file_service_id, service_id )
                         
-                        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( tags_table_name ) ).fetchone()
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( tags_table_name ) ).fetchone()
                         
-                    elif info_type in ( HC.SERVICE_INFO_NUM_MAPPINGS, HC.SERVICE_INFO_NUM_PENDING_MAPPINGS ):
+                    elif info_type == HC.SERVICE_INFO_NUM_MAPPINGS:
                         
-                        if info_type == HC.SERVICE_INFO_NUM_MAPPINGS:
+                        ac_cache_table_name = self._CacheMappingsGetACCacheTableName( ClientTags.TAG_DISPLAY_STORAGE, self.modules_services.combined_file_service_id, service_id )
+                        
+                        column_name = 'current_count'
+                        
+                        result = self._c.execute( 'SELECT SUM( {} ) FROM {};'.format( column_name, ac_cache_table_name ) ).fetchone()
+                        
+                        if result is None or result[0] is None:
                             
-                            ac_cache_table_name = self._CacheMappingsGetACCacheTableName( ClientTags.TAG_DISPLAY_STORAGE, self.modules_services.combined_file_service_id, service_id )
+                            info = 0
                             
-                            column_name = 'current_count'
+                        else:
                             
-                            result = self._c.execute( 'SELECT SUM( {} ) FROM {};'.format( column_name, ac_cache_table_name ) ).fetchone()
-                            
-                            if result is None or result[0] is None:
-                                
-                                result = ( 0, )
-                                
-                            
-                        elif info_type == HC.SERVICE_INFO_NUM_PENDING_MAPPINGS:
-                            
-                            # since pending is nearly always far smaller rowcount than current, if I pull this from a/c table, it is a HUGE waste of time and not faster than counting the raw table rows!
-                            
-                            info = self.modules_mappings_storage.GetPendingCount( service_id )
-                            
-                            result = ( info, )
+                            ( info, ) = result
                             
                         
-                    elif info_type == HC.SERVICE_INFO_NUM_DELETED_MAPPINGS: result = self._c.execute( 'SELECT COUNT( * ) FROM ' + deleted_mappings_table_name + ';' ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_MAPPINGS: result = self._c.execute( 'SELECT COUNT( * ) FROM ' + petitioned_mappings_table_name + ';' ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS: result = self._c.execute( 'SELECT COUNT( * ) FROM tag_sibling_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS: result = self._c.execute( 'SELECT COUNT( * ) FROM tag_sibling_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS: result = self._c.execute( 'SELECT COUNT( * ) FROM tag_parent_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchone()
-                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS: result = self._c.execute( 'SELECT COUNT( * ) FROM tag_parent_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchone()
+                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_MAPPINGS:
+                        
+                        # since pending is nearly always far smaller rowcount than current, if I pull this from a/c table, it is a HUGE waste of time and not faster than counting the raw table rows!
+                        
+                        info = self.modules_mappings_storage.GetPendingMappingsCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_DELETED_MAPPINGS:
+                        
+                        # since pending is nearly always far smaller rowcount than current, if I pull this from a/c table, it is a HUGE waste of time and not faster than counting the raw table rows!
+                        
+                        info = self.modules_mappings_storage.GetDeletedMappingsCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_MAPPINGS:
+                        
+                        # since pending is nearly always far smaller rowcount than current, if I pull this from a/c table, it is a HUGE waste of time and not faster than counting the raw table rows!
+                        
+                        info = self.modules_mappings_storage.GetPetitionedMappingsCount( service_id )
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_TAG_SIBLINGS:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM tag_sibling_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchone()
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_TAG_SIBLINGS:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM tag_sibling_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchone()
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PENDING_TAG_PARENTS:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM tag_parent_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PENDING ) ).fetchone()
+                        
+                    elif info_type == HC.SERVICE_INFO_NUM_PETITIONED_TAG_PARENTS:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM tag_parent_petitions WHERE service_id = ? AND status = ?;', ( service_id, HC.CONTENT_STATUS_PETITIONED ) ).fetchone()
+                        
                     
                 elif service_type in ( HC.LOCAL_RATING_LIKE, HC.LOCAL_RATING_NUMERICAL ):
                     
-                    if info_type == HC.SERVICE_INFO_NUM_FILES: result = self._c.execute( 'SELECT COUNT( * ) FROM local_ratings WHERE service_id = ?;', ( service_id, ) ).fetchone()
+                    if info_type == HC.SERVICE_INFO_NUM_FILES:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM local_ratings WHERE service_id = ?;', ( service_id, ) ).fetchone()
+                        
                     
                 elif service_type == HC.LOCAL_BOORU:
                     
-                    if info_type == HC.SERVICE_INFO_NUM_SHARES: result = self._c.execute( 'SELECT COUNT( * ) FROM yaml_dumps WHERE dump_type = ?;', ( ClientDBSerialisable.YAML_DUMP_ID_LOCAL_BOORU, ) ).fetchone()
-                    
-                
-                if result is None:
-                    
-                    info = 0
-                    
-                else:
-                    
-                    ( info, ) = result
+                    if info_type == HC.SERVICE_INFO_NUM_SHARES:
+                        
+                        ( info, ) = self._c.execute( 'SELECT COUNT( * ) FROM yaml_dumps WHERE dump_type = ?;', ( ClientDBSerialisable.YAML_DUMP_ID_LOCAL_BOORU, ) ).fetchone()
+                        
                     
                 
                 if info is None:
@@ -12845,10 +12854,12 @@ class DB( HydrusDB.HydrusDB ):
             
             timestamp_cutoff = HydrusData.GetNow() - minimum_age
             
-            age_phrase = ' AND timestamp < ' + str( timestamp_cutoff )
+            age_phrase = ' WHERE timestamp < ' + str( timestamp_cutoff )
             
         
-        hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM current_files WHERE service_id = ?' + age_phrase + limit_phrase + ';', ( self.modules_services.trash_service_id, ) ) )
+        current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( self.modules_services.trash_service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {}{}{};'.format( current_files_table_name, age_phrase, limit_phrase ) ) )
         
         hash_ids = self._FilterForFileDeleteLock( self.modules_services.trash_service_id, hash_ids )
         
@@ -13181,8 +13192,7 @@ class DB( HydrusDB.HydrusDB ):
         
         if hash_ids_to_current_file_service_ids is None:
             
-            # temp hashes to files
-            hash_ids_to_current_file_service_ids = HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT hash_id, service_id FROM {} CROSS JOIN current_files USING ( hash_id );'.format( hash_ids_table_name ) ) )
+            hash_ids_to_current_file_service_ids = self.modules_files_storage.GetHashIdsToCurrentServiceIds( hash_ids_table_name )
             
         
         cached_file_service_ids = set( self.modules_services.GetServiceIds( HC.AUTOCOMPLETE_CACHE_SPECIFIC_FILE_SERVICES ) )
@@ -13436,12 +13446,11 @@ class DB( HydrusDB.HydrusDB ):
         
         if len( inboxed_hash_ids ) > 0:
             
-            with HydrusDB.TemporaryIntegerTable( self._c, inboxed_hash_ids, 'hash_id' ) as temp_table_name:
+            service_ids_to_counts = self.modules_files_storage.GetServiceIdCounts( inboxed_hash_ids )
+            
+            if len( service_ids_to_counts ) > 0:
                 
-                # temp hashes to files
-                updates = self._c.execute( 'SELECT service_id, COUNT( * ) FROM {} CROSS JOIN current_files USING ( hash_id ) GROUP BY service_id;'.format( temp_table_name ) ).fetchall()
-                
-                self._c.executemany( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', [ ( count, service_id, HC.SERVICE_INFO_NUM_INBOX ) for ( service_id, count ) in updates ] )
+                self._c.executemany( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', [ ( count, service_id, HC.SERVICE_INFO_NUM_INBOX ) for ( service_id, count ) in service_ids_to_counts.items() ] )
                 
             
         
@@ -13481,9 +13490,9 @@ class DB( HydrusDB.HydrusDB ):
                 
                 hash_id = self.modules_hashes_local_cache.GetHashId( hash )
                 
-                result = self._c.execute( 'SELECT 1 FROM current_files WHERE service_id = ? AND hash_id = ?;', ( self.modules_services.combined_local_file_service_id, hash_id ) ).fetchone()
+                hash_ids = self.modules_files_storage.FilterCurrentHashIds( self.modules_services.combined_local_file_service_id, ( hash_id, ) )
                 
-                if result is None:
+                if len( hash_ids ) == 0:
                     
                     return True
                     
@@ -13496,9 +13505,9 @@ class DB( HydrusDB.HydrusDB ):
                 
                 hash_id = self.modules_hashes_local_cache.GetHashId( hash )
                 
-                result = self._c.execute( 'SELECT 1 FROM current_files WHERE hash_id = ?;', ( hash_id, ) ).fetchone()
+                hash_ids = self.modules_files_storage.FilterAllCurrentHashIds( ( hash_id, ) )
                 
-                if result is None:
+                if len( hash_ids ) == 0:
                     
                     return True
                     
@@ -13548,12 +13557,6 @@ class DB( HydrusDB.HydrusDB ):
         
         #
         
-        self.modules_similar_files = ClientDBSimilarFiles.ClientDBSimilarFiles( self._c )
-        
-        self._modules.append( self.modules_similar_files )
-        
-        #
-        
         self.modules_tags_local_cache = ClientDBDefinitionsCache.ClientDBCacheLocalTags( self._c, self.modules_tags )
         
         self._modules.append( self.modules_tags_local_cache )
@@ -13564,10 +13567,20 @@ class DB( HydrusDB.HydrusDB ):
         
         #
         
+        self.modules_files_storage = ClientDBFilesStorage.ClientDBFilesStorage( self._c, self.modules_services, self.modules_texts )
+        
+        self._modules.append( self.modules_files_storage )
+        
         self.modules_mappings_storage = ClientDBMappingsStorage.ClientDBMappingsStorage( self._c, self.modules_services )
         
         self._modules.append( self.modules_mappings_storage )
         
+        #
+        
+        self.modules_similar_files = ClientDBSimilarFiles.ClientDBSimilarFiles( self._c, self.modules_services, self.modules_files_storage )
+        
+        self._modules.append( self.modules_similar_files )
+        
     
     def _ManageDBError( self, job, e ):
         
@@ -14001,34 +14014,19 @@ class DB( HydrusDB.HydrusDB ):
                                 
                                 hashes = sub_row
                                 
-                                # we delete from everywhere, but not for files currently in the trash
-                                
-                                clear_service_ids = self.modules_services.GetServiceIds( ( HC.COMBINED_LOCAL_FILE, HC.LOCAL_FILE_DOMAIN ) )
-                                
                                 if hashes is None:
                                     
-                                    self._c.executemany( 'DELETE FROM deleted_files WHERE service_id = ? AND hash_id NOT IN ( SELECT hash_id FROM current_files WHERE service_id = ? );', ( ( clear_service_id, self.modules_services.trash_service_id ) for clear_service_id in clear_service_ids ) )
-                                    
-                                    self._c.execute( 'DELETE FROM local_file_deletion_reasons WHERE hash_id NOT IN ( SELECT hash_id FROM current_files WHERE service_id = ? );', ( self.modules_services.trash_service_id, ) )
-                                    
-                                    self._c.executemany( 'DELETE FROM service_info WHERE service_id = ? AND info_type = ?;', ( ( clear_service_id, HC.SERVICE_INFO_NUM_DELETED_FILES ) for clear_service_id in clear_service_ids ) )
+                                    service_ids_to_nums_cleared = self.modules_files_storage.ClearLocalDeleteRecord()
                                     
                                 else:
                                     
                                     hash_ids = self.modules_hashes_local_cache.GetHashIds( hashes )
                                     
-                                    for clear_service_id in clear_service_ids:
-                                        
-                                        self._c.executemany( 'DELETE FROM deleted_files WHERE service_id = ? AND hash_id = ? AND hash_id NOT IN ( SELECT hash_id FROM current_files WHERE service_id = ? );', ( ( clear_service_id, hash_id, self.modules_services.trash_service_id ) for hash_id in hash_ids ) )
-                                        
-                                        num_cleared = HydrusDB.GetRowCount( self._c )
-                                        
-                                        self._c.execute( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', ( -num_cleared, clear_service_id, HC.SERVICE_INFO_NUM_DELETED_FILES ) )
-                                        
-                                    
-                                    self._c.executemany( 'DELETE FROM local_file_deletion_reasons WHERE hash_id = ? AND hash_id NOT IN ( SELECT hash_id FROM current_files WHERE service_id = ? );', ( ( hash_id, self.modules_services.trash_service_id ) for hash_id in hash_ids ) )
+                                    service_ids_to_nums_cleared = self.modules_files_storage.ClearLocalDeleteRecord( hash_ids )
                                     
                                 
+                                self._c.executemany( 'UPDATE service_info SET info = info + ? WHERE service_id = ? AND info_type = ?;', ( ( -num_cleared, clear_service_id, HC.SERVICE_INFO_NUM_DELETED_FILES ) for ( clear_service_id, num_cleared ) in service_ids_to_nums_cleared.items() ) )
+                                
                             
                         elif action == HC.CONTENT_UPDATE_ADD:
                             
@@ -14084,7 +14082,10 @@ class DB( HydrusDB.HydrusDB ):
                                     
                                     reason = content_update.GetReason()
                                     
-                                    self._SetLocalFileDeletionReason( hash_ids, reason )
+                                    if reason is not None:
+                                        
+                                        self.modules_files_storage.SetFileDeletionReason( hash_ids, reason )
+                                        
                                     
                                 
                                 if service_id == self.modules_services.trash_service_id:
@@ -14104,38 +14105,42 @@ class DB( HydrusDB.HydrusDB ):
                                 
                             elif action == HC.CONTENT_UPDATE_PEND:
                                 
-                                invalid_hash_ids = self._FilterHashIdsByFileServiceId( service_id, hash_ids )
+                                invalid_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, hash_ids )
                                 
                                 valid_hash_ids = hash_ids.difference( invalid_hash_ids )
                                 
-                                self._c.executemany( 'INSERT OR IGNORE INTO file_transfers ( service_id, hash_id ) VALUES ( ?, ? );', ( ( service_id, hash_id ) for hash_id in valid_hash_ids ) )
+                                self.modules_files_storage.PendFiles( service_id, valid_hash_ids )
                                 
-                                if service_key == CC.COMBINED_LOCAL_FILE_SERVICE_KEY: notify_new_downloads = True
-                                else: notify_new_pending = True
+                                if service_key == CC.COMBINED_LOCAL_FILE_SERVICE_KEY:
+                                    
+                                    notify_new_downloads = True
+                                    
+                                else:
+                                    
+                                    notify_new_pending = True
+                                    
                                 
                             elif action == HC.CONTENT_UPDATE_PETITION:
                                 
                                 reason = content_update.GetReason()
                                 
-                                valid_hash_ids = self._FilterHashIdsByFileServiceId( service_id, hash_ids )
-                                
                                 reason_id = self.modules_texts.GetTextId( reason )
                                 
-                                self._c.executemany( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in valid_hash_ids ) )
+                                valid_hash_ids = self.modules_files_storage.FilterCurrentHashIds( service_id, hash_ids )
                                 
-                                self._c.executemany( 'INSERT OR IGNORE INTO file_petitions ( service_id, hash_id, reason_id ) VALUES ( ?, ?, ? );', ( ( service_id, hash_id, reason_id ) for hash_id in valid_hash_ids ) )
+                                self.modules_files_storage.PetitionFiles( service_id, reason_id, valid_hash_ids )
                                 
                                 notify_new_pending = True
                                 
                             elif action == HC.CONTENT_UPDATE_RESCIND_PEND:
                                 
-                                self._c.executemany( 'DELETE FROM file_transfers WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in hash_ids ) )
+                                self.modules_files_storage.RescindPendFiles( service_id, hash_ids )
                                 
                                 notify_new_pending = True
                                 
                             elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
                                 
-                                self._c.executemany( 'DELETE FROM file_petitions WHERE service_id = ? AND hash_id = ?;', ( ( service_id, hash_id ) for hash_id in hash_ids ) )
+                                self.modules_files_storage.RescindPetitionFiles( service_id, hash_ids )
                                 
                                 notify_new_pending = True
                                 
@@ -14523,7 +14528,9 @@ class DB( HydrusDB.HydrusDB ):
                         
                         if action == 'delete_for_deleted_files':
                             
-                            self._c.execute( 'DELETE FROM local_ratings WHERE local_ratings.service_id = ? and hash_id IN ( SELECT hash_id FROM deleted_files WHERE deleted_files.service_id = ? );', ( service_id, self.modules_services.combined_local_file_service_id ) )
+                            deleted_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( self.modules_services.combined_local_file_service_id, HC.CONTENT_STATUS_DELETED )
+                            
+                            self._c.execute( 'DELETE FROM local_ratings WHERE service_id = ? and hash_id IN ( SELECT hash_id FROM {} );'.format( deleted_files_table_name ), ( service_id, ) )
                             
                             ratings_deleted = HydrusDB.GetRowCount( self._c )
                             
@@ -14531,7 +14538,9 @@ class DB( HydrusDB.HydrusDB ):
                             
                         elif action == 'delete_for_non_local_files':
                             
-                            self._c.execute( 'DELETE FROM local_ratings WHERE local_ratings.service_id = ? and hash_id NOT IN ( SELECT hash_id FROM current_files WHERE current_files.service_id = ? );', ( service_id, self.modules_services.combined_local_file_service_id ) )
+                            current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( self.modules_services.combined_local_file_service_id, HC.CONTENT_STATUS_CURRENT )
+                            
+                            self._c.execute( 'DELETE FROM local_ratings WHERE local_ratings.service_id = ? and hash_id NOT IN ( SELECT hash_id FROM {} );'.format( current_files_table_name ), ( service_id, ) )
                             
                             ratings_deleted = HydrusDB.GetRowCount( self._c )
                             
@@ -16661,16 +16670,6 @@ class DB( HydrusDB.HydrusDB ):
         self._c.execute( 'INSERT INTO last_shutdown_work_time ( last_shutdown_work_time ) VALUES ( ? );', ( timestamp, ) )
         
     
-    def _SetLocalFileDeletionReason( self, hash_ids, reason ):
-        
-        if reason is not None:
-            
-            reason_id = self.modules_texts.GetTextId( reason )
-            
-            self._c.executemany( 'REPLACE INTO local_file_deletion_reasons ( hash_id, reason_id ) VALUES ( ?, ? );', ( ( hash_id, reason_id ) for hash_id in hash_ids ) )
-            
-        
-    
     def _SetPassword( self, password ):
         
         if password is not None:
@@ -16788,63 +16787,60 @@ class DB( HydrusDB.HydrusDB ):
                 
                 if sort_data == CC.SORT_FILES_BY_IMPORT_TIME:
                     
-                    query = 'SELECT hash_id, timestamp FROM files_info NATURAL JOIN current_files WHERE hash_id = ? AND service_id = ?;'
+                    current_files_table_name = ClientDBFilesStorage.GenerateFilesTableName( file_service_id, HC.CONTENT_STATUS_CURRENT )
                     
-                    select_args_iterator = ( ( hash_id, file_service_id ) for hash_id in hash_ids )
+                    query = 'SELECT hash_id, timestamp FROM {} WHERE hash_id = ?;'.format( current_files_table_name )
                     
-                else:
+                elif sort_data == CC.SORT_FILES_BY_FILESIZE:
                     
-                    if sort_data == CC.SORT_FILES_BY_FILESIZE:
-                        
-                        query = 'SELECT hash_id, size FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_DURATION:
-                        
-                        query = 'SELECT hash_id, duration FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_FRAMERATE:
-                        
-                        query = 'SELECT hash_id, num_frames, duration FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_NUM_FRAMES:
-                        
-                        query = 'SELECT hash_id, num_frames FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_WIDTH:
-                        
-                        query = 'SELECT hash_id, width FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_HEIGHT:
-                        
-                        query = 'SELECT hash_id, height FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_RATIO:
-                        
-                        query = 'SELECT hash_id, width, height FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_NUM_PIXELS:
-                        
-                        query = 'SELECT hash_id, width, height FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_MEDIA_VIEWS:
-                        
-                        query = 'SELECT hash_id, media_views FROM file_viewing_stats WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_MEDIA_VIEWTIME:
-                        
-                        query = 'SELECT hash_id, media_viewtime FROM file_viewing_stats WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_APPROX_BITRATE:
-                        
-                        query = 'SELECT hash_id, duration, num_frames, size, width, height FROM files_info WHERE hash_id = ?;'
-                        
-                    elif sort_data == CC.SORT_FILES_BY_FILE_MODIFIED_TIMESTAMP:
-                        
-                        query = 'SELECT hash_id, file_modified_timestamp FROM file_modified_timestamps WHERE hash_id = ?;'
-                        
+                    query = 'SELECT hash_id, size FROM files_info WHERE hash_id = ?;'
                     
-                    select_args_iterator = ( ( hash_id, ) for hash_id in hash_ids )
+                elif sort_data == CC.SORT_FILES_BY_DURATION:
                     
+                    query = 'SELECT hash_id, duration FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_FRAMERATE:
+                    
+                    query = 'SELECT hash_id, num_frames, duration FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_NUM_FRAMES:
+                    
+                    query = 'SELECT hash_id, num_frames FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_WIDTH:
+                    
+                    query = 'SELECT hash_id, width FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_HEIGHT:
+                    
+                    query = 'SELECT hash_id, height FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_RATIO:
+                    
+                    query = 'SELECT hash_id, width, height FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_NUM_PIXELS:
+                    
+                    query = 'SELECT hash_id, width, height FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_MEDIA_VIEWS:
+                    
+                    query = 'SELECT hash_id, media_views FROM file_viewing_stats WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_MEDIA_VIEWTIME:
+                    
+                    query = 'SELECT hash_id, media_viewtime FROM file_viewing_stats WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_APPROX_BITRATE:
+                    
+                    query = 'SELECT hash_id, duration, num_frames, size, width, height FROM files_info WHERE hash_id = ?;'
+                    
+                elif sort_data == CC.SORT_FILES_BY_FILE_MODIFIED_TIMESTAMP:
+                    
+                    query = 'SELECT hash_id, file_modified_timestamp FROM file_modified_timestamps WHERE hash_id = ?;'
+                    
+                
+                select_args_iterator = ( ( hash_id, ) for hash_id in hash_ids )
                 
                 if sort_data == CC.SORT_FILES_BY_RATIO:
                     
@@ -16994,10 +16990,7 @@ class DB( HydrusDB.HydrusDB ):
     
     def _UndeleteFiles( self, service_id, hash_ids ):
         
-        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
-            
-            rows = self._c.execute( 'SELECT hash_id, original_timestamp FROM {} CROSS JOIN deleted_files USING ( hash_id ) WHERE service_id = ?;'.format( temp_hash_ids_table_name ), ( service_id, ) ).fetchall()
-            
+        rows = self.modules_files_storage.GetUndeleteRows( service_id, hash_ids )
         
         self._AddFiles( service_id, rows )
         
@@ -17016,714 +17009,16 @@ class DB( HydrusDB.HydrusDB ):
         
         self._controller.frame_splash_status.SetText( 'updating db to v' + str( version + 1 ) )
         
-        if version == 389:
+        if version < 411:
             
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'derpibooru.org file page parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
+            raise Exception( 'Sorry, your database version is {}, but hydrus 447 and above cannot update client databases before version 411! Please try installing 446 before going up to 447.'.format( version ) )
             
         
-        if version == 390:
+        if version < 436:
             
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'danbooru file page parser', 'danbooru file page parser - get webm ugoira' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-                #
-                
-                login_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER )
-                
-                login_manager.Initialise()
-                
-                #
-                
-                login_manager.OverwriteDefaultLoginScripts( ( 'e621.net login', ) )
-                
-                #
-                
-                login_manager.TryToLinkMissingLoginScripts( ( 'e621.net', ) )
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( login_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
+            message = 'Hey, your database version is {}, but hydrus 447 and above _may_ have a problem updating client databases before version 436. If you like, you can click ok and try it, but if it errors out, please try installing 446 before going up to 447. Also, sorry, but this message will spam until you are up to 436!'.format( version )
             
-        
-        if version == 391:
-            # pretty sure this won't work any more due to tag sibling application table being made later on. nbd, since we'll be reblatting them in one minute for v408 update
-            '''
-            tag_service_ids = self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES )
-            
-            for tag_service_id in tag_service_ids:
-                
-                self._CacheTagSiblingsDrop( tag_service_id )
-                self._CacheTagSiblingsGenerate( tag_service_id )
-                
-            '''
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'e621 file page parser', 'sankaku file page parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 392:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ 'deviant art file page extended_fetch api', 'deviant art file page', 'deviant art flash sandbox page' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'deviant art flash sandbox page parser', 'deviant art file extended_fetch parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 395:
-            
-            result = self._c.execute( 'SELECT 1 FROM external_master.sqlite_master WHERE name = ?;', ( 'labels', ) ).fetchone()
-            
-            if result is None:
-                
-                try:
-                    
-                    self._controller.frame_splash_status.SetSubtext( 'updating notes table' )
-                    
-                    self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.labels ( label_id INTEGER PRIMARY KEY, label TEXT UNIQUE );' )
-                    
-                    self._c.execute( 'CREATE TABLE IF NOT EXISTS external_master.notes ( note_id INTEGER PRIMARY KEY, note TEXT UNIQUE );' )
-                    self._c.execute( 'CREATE VIRTUAL TABLE IF NOT EXISTS external_caches.notes_fts4 USING fts4( note );' )
-                    
-                    self._c.execute( 'ALTER TABLE file_notes RENAME TO file_notes_old;' )
-                    
-                    self._c.execute( 'CREATE TABLE file_notes ( hash_id INTEGER, name_id INTEGER, note_id INTEGER, PRIMARY KEY ( hash_id, name_id ) );' )
-                    self._CreateIndex( 'file_notes', [ 'note_id' ] )
-                    
-                    all_data = self._c.execute( 'SELECT hash_id, notes FROM file_notes_old;' ).fetchall()
-                    
-                    name_id = self.modules_texts.GetLabelId( 'notes' )
-                    
-                    for ( hash_id, note ) in all_data:
-                        
-                        note_id = self.modules_texts.GetNoteId( note )
-                        
-                        self._c.execute( 'INSERT OR IGNORE INTO file_notes ( hash_id, name_id, note_id ) VALUES ( ?, ?, ? );', ( hash_id, name_id, note_id ) )
-                        
-                    
-                    self._c.execute( 'DROP TABLE file_notes_old;' )
-                    
-                    self._AnalyzeTable( 'file_notes' )
-                    self._AnalyzeTable( 'notes' )
-                    self._AnalyzeTable( 'notes_fts4' )
-                    self._AnalyzeTable( 'labels' )
-                    
-                except Exception as e:
-                    
-                    message = 'Trying to update the notes table failed! Please let hydrus dev know!'
-                    
-                    HydrusData.Print( message )
-                    BlockingSafeShowMessage( message )
-                    HydrusData.PrintException( e )
-                    
-                    raise
-                    
-                
-            
-        
-        if version == 397:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultGUGs( [ 'newgrounds artist art lookup', 'newgrounds artist games lookup', 'newgrounds artist movies lookup', 'newgrounds artist lookup' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ 'newgrounds movies gallery page', 'newgrounds games gallery page', 'newgrounds file page', 'newgrounds art', 'newgrounds art gallery page' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'gelbooru 0.2.x gallery page parser', 'newgrounds art parser', 'newgrounds file page parser', 'newgrounds gallery page parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 398:
-            
-            existing_shortcut_names = self.modules_serialisable.GetJSONDumpNames( HydrusSerialisable.SERIALISABLE_TYPE_SHORTCUT_SET )
-            
-            if 'media' in existing_shortcut_names:
-                
-                try:
-                    
-                    media_shortcuts = self.modules_serialisable.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_SHORTCUT_SET, dump_name = 'media' )
-                    
-                    from hydrus.client.gui import ClientGUIShortcuts
-                    
-                    delete_command = CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_DELETE_FILE )
-                    undelete_command = CAC.ApplicationCommand( CAC.APPLICATION_COMMAND_TYPE_SIMPLE, CAC.SIMPLE_UNDELETE_FILE )
-                    
-                    for delete_key in ClientGUIShortcuts.DELETE_KEYS_HYDRUS:
-                        
-                        shortcut = ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, delete_key, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [] )
-                        
-                        if media_shortcuts.GetCommand( shortcut ) is None:
-                            
-                            media_shortcuts.SetCommand( shortcut, delete_command )
-                            
-                        
-                        shortcut = ClientGUIShortcuts.Shortcut( ClientGUIShortcuts.SHORTCUT_TYPE_KEYBOARD_SPECIAL, delete_key, ClientGUIShortcuts.SHORTCUT_PRESS_TYPE_PRESS, [ ClientGUIShortcuts.SHORTCUT_MODIFIER_SHIFT ] )
-                        
-                        if media_shortcuts.GetCommand( shortcut ) is None:
-                            
-                            media_shortcuts.SetCommand( shortcut, undelete_command )
-                            
-                        
-                    
-                    self.modules_serialisable.SetJSONDump( media_shortcuts )
-                    
-                except:
-                    
-                    HydrusData.PrintException( e )
-                    
-                    message = 'Trying to update the media shortcuts failed! Please let hydrus dev know!'
-                    
-                    self.pub_initial_message( message )
-                    
-                
-            
-        
-        if version == 398:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ '8chan.moe thread', '8chan.moe thread json api' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ '8chan.moe thread api parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some parsers failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 399:
-            
-            try:
-                
-                legacy_subscription_names = self.modules_serialisable.GetJSONDumpNames( HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION_LEGACY )
-                
-                if len( legacy_subscription_names ) > 0:
-                    
-                    try:
-                        
-                        HydrusPaths.CheckHasSpaceForDBTransaction( self._db_dir, 500 * 1024 * 1024 )
-                        
-                    except:
-                        
-                        message = 'The big subscription update for v400 will now start. However this update is heavy and will also try to make a backup of your old subs, and it looks like your system drive or hydrus drive are a bit short on space right now. If your drives are truly currently real tight, please free up some space now. If you have thousands of subs with hundreds of thousands of URLs, you will need a few GB.'
-                        
-                        BlockingSafeShowMessage( message )
-                        
-                    
-                    sub_dir = os.path.join( self._db_dir, 'legacy_subscriptions_backup' )
-                    
-                    HydrusPaths.MakeSureDirectoryExists( sub_dir )
-                    
-                    for ( i, legacy_subscription_name ) in enumerate( legacy_subscription_names ):
-                        
-                        self._controller.frame_splash_status.SetSubtext( 'updating subscriptions: {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, len( legacy_subscription_names ) ) ) )
-                        
-                        legacy_subscription = self.modules_serialisable.GetJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION_LEGACY, legacy_subscription_name )
-                        
-                        backup_path = os.path.join( sub_dir, 'sub_{}.json'.format( i ) )
-                        
-                        with open( backup_path, 'w', encoding = 'utf-8' ) as f:
-                            
-                            f.write( legacy_subscription.DumpToString() )
-                            
-                        
-                        ( subscription, query_log_containers ) = ClientImportSubscriptionLegacy.ConvertLegacySubscriptionToNew( legacy_subscription )
-                        
-                        self.modules_serialisable.SetJSONDump( subscription )
-                        
-                        for query_log_container in query_log_containers:
-                            
-                            self.modules_serialisable.SetJSONDump( query_log_container )
-                            
-                        
-                        self.modules_serialisable.DeleteJSONDumpNamed( HydrusSerialisable.SERIALISABLE_TYPE_SUBSCRIPTION_LEGACY, legacy_subscription_name )
-                        
-                    
-                
-            except Exception as e:
-                
-                message = 'Damn, the big subscription update for v400 did not work for you! No changes have been saved, your database is still on v399. You will get an error next, please send it to hydrus dev and go back to using v399 for now!'
-                
-                BlockingSafeShowMessage( message )
-                
-                raise
-                
-            
-            #
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ 'twitter tweet', 'nitter tweet media', 'nitter tweet', 'nitter timeline', 'nitter media timeline', 'derpibooru gallery page', 'derpibooru gallery page api' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultGUGs( [ 'nitter media lookup', 'nitter retweets lookup', 'nitter media and retweets lookup', 'derpibooru tag search', 'derpibooru tag search - no filter' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'nitter media parser', 'nitter retweet parser', 'nitter tweet parser', 'nitter tweet parser (video from koto.reisen)', 'danbooru file page parser', 'danbooru file page parser - get webm ugoira', 'derpibooru gallery page api parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some downloaders failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 400:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ 'e621 gallery page (alternate format)' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'nitter tweet parser', 'nitter tweet parser (video from koto.reisen)', 'e621 gallery page parser', 'derpibooru gallery page api parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some downloaders failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 401:
-            
-            result = self._c.execute( 'SELECT 1 FROM sqlite_master WHERE name = ?;', ( 'remote_ratings', ) ).fetchone()
-            
-            if result is not None:
-                
-                try:
-                    
-                    # never used
-                    self._c.execute( 'DROP TABLE remote_ratings;' )
-                    
-                except Exception as e:
-                    
-                    HydrusData.PrintException( e )
-                    
-                
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultURLClasses( [ 'nitter tweet', 'twitter tweet' ] )
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'nitter media parser', 'nitter retweet parser', 'nitter tweet parser (video from koto.reisen)', 'nitter tweet parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some downloaders failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 402:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'danbooru file page parser', 'danbooru file page parser - get webm ugoira', 'deviant art file extended_fetch parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some downloaders failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 403:
-            
-            try:
-                
-                result = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_COLUMN_LIST_MANAGER )
-                
-                if result is None:
-                    
-                    from hydrus.client.gui.lists import ClientGUIListManager
-                    
-                    column_list_manager = ClientGUIListManager.ColumnListManager()
-                    
-                    self.modules_serialisable.SetJSONDump( column_list_manager )
-                    
-                
-            except:
-                
-                HydrusData.PrintException( e )
-                
-                raise Exception( 'Could not initialise the column list manager! Please let hydrus dev know. Error was written to log, it also follows: {}{}'.format( os.linesep * 2, e ) )
-                
-            
-        
-        if version == 404:
-            
-            try:
-                
-                domain_manager = self.modules_serialisable.GetJSONDump( HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_DOMAIN_MANAGER )
-                
-                domain_manager.Initialise()
-                
-                #
-                
-                domain_manager.OverwriteDefaultParsers( [ 'derpibooru.org file page parser' ] )
-                
-                #
-                
-                domain_manager.TryToLinkURLClassesAndParsers()
-                
-                #
-                
-                self.modules_serialisable.SetJSONDump( domain_manager )
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to update some downloaders failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
-            
-        
-        if version == 407:
-            
-            self._CreateIndex( 'current_files', [ 'hash_id' ] )
-            
-            self._AnalyzeTable( 'current_files' )
-            
-            result = self._c.execute( 'SELECT 1 FROM sqlite_master WHERE name = ?;', ( 'tag_sibling_application', ) ).fetchone()
-            
-            if result is None:
-                
-                real_tag_service_ids = self.modules_services.GetServiceIds( HC.REAL_TAG_SERVICES )
-                
-                repo_tag_service_ids = self.modules_services.GetServiceIds( ( HC.TAG_REPOSITORY, ) )
-                
-                if len( repo_tag_service_ids ) > 0:
-                    
-                    BlockingSafeShowMessage( 'Hey, if you sync to the PTR, the update step from 407->408 takes a few minutes on an SSD but will take much longer on an HDD, perhaps an hour or more. If you do not have time, close the program in task manager now.' )
-                    
-                
-                try:
-                    
-                    self._c.execute( 'CREATE TABLE IF NOT EXISTS tag_sibling_application ( master_service_id INTEGER, service_index INTEGER, application_service_id INTEGER, PRIMARY KEY ( master_service_id, service_index ) );' )
-                    
-                    inserts = [ ( tag_service_id, 0, tag_service_id ) for tag_service_id in real_tag_service_ids ]
-                    
-                    self._c.executemany( 'INSERT OR IGNORE INTO tag_sibling_application ( master_service_id, service_index, application_service_id ) VALUES ( ?, ?, ? );', inserts )
-                    
-                except Exception as e:
-                    
-                    HydrusData.PrintException( e )
-                    
-                    raise Exception( 'Could not create the new sibling options! Error was written to log, it also follows: {}{}'.format( os.linesep * 2, e ) )
-                    
-                
-                self._service_ids_to_sibling_applicable_service_ids = None
-                self._service_ids_to_sibling_interested_service_ids = None
-                self._service_ids_to_parent_applicable_service_ids = None
-                self._service_ids_to_parent_interested_service_ids = None
-                
-                #
-                
-                combined_tag_service_id = self.modules_services.GetServiceId( CC.COMBINED_TAG_SERVICE_KEY )
-                
-                bad_table_name = 'external_caches.tag_siblings_lookup_cache_{}'.format( combined_tag_service_id )
-                
-                self._c.execute( 'DROP TABLE IF EXISTS {};'.format( bad_table_name ) )
-                
-                #
-                
-                try:
-                    
-                    for ( i, tag_service_id ) in enumerate( real_tag_service_ids ):
-                        
-                        self._controller.frame_splash_status.SetSubtext( 'regenerating tag siblings lookup: {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, len( real_tag_service_ids ) ) ) )
-                        
-                        old_tag_siblings_lookup_base_name = 'tag_siblings_lookup_cache_{}'.format( tag_service_id )
-                        old_tag_siblings_lookup_name = 'external_caches.{}'.format( old_tag_siblings_lookup_base_name )
-                        
-                        self._c.execute( 'DROP TABLE IF EXISTS {};'.format( old_tag_siblings_lookup_name ) )
-                        
-                        self._CacheTagSiblingsDrop( tag_service_id )
-                        self._CacheTagSiblingsGenerate( tag_service_id )
-                        
-                    
-                except Exception as e:
-                    
-                    HydrusData.PrintException( e )
-                    
-                    raise Exception( 'Could not create the new tag sibling lookup caches! Error was written to log, it also follows: {}{}'.format( os.linesep * 2, e ) )
-                    
-                
-                file_service_ids = self.modules_services.GetServiceIds( HC.AUTOCOMPLETE_CACHE_SPECIFIC_FILE_SERVICES )
-                
-                num_to_do = len( file_service_ids ) * len( real_tag_service_ids )
-                
-                try:
-                    
-                    for ( i, ( file_service_id, tag_service_id ) ) in enumerate( itertools.product( file_service_ids, real_tag_service_ids ) ):
-                        
-                        message = 'generating specific tag display cache: {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, num_to_do ) )
-                        
-                        self._controller.frame_splash_status.SetSubtext( message )
-                        
-                        def status_hook_1( s: str ):
-                            
-                            self._controller.frame_splash_status.SetSubtext( '{} - {}'.format( message, s ) )
-                            
-                        
-                        self._CacheSpecificDisplayMappingsDrop( file_service_id, tag_service_id )
-                        self._CacheSpecificDisplayMappingsGenerate( file_service_id, tag_service_id, status_hook = status_hook_1 )
-                        
-                    
-                    self._controller.frame_splash_status.SetSubtext( '' )
-                    
-                except Exception as e:
-                    
-                    HydrusData.PrintException( e )
-                    
-                    raise Exception( 'Could not create the new specific tag display caches! Error was written to log, it also follows: {}{}'.format( os.linesep * 2, e ) )
-                    
-                
-                try:
-                    
-                    for ( i, tag_service_id ) in enumerate( real_tag_service_ids ):
-                        
-                        message = 'generating combined tag display cache: {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, len( real_tag_service_ids ) ) )
-                        
-                        self._controller.frame_splash_status.SetSubtext( message )
-                        
-                        def status_hook_2( s: str ):
-                            
-                            self._controller.frame_splash_status.SetSubtext( '{} - {}'.format( message, s ) )
-                            
-                        
-                        self._CacheCombinedFilesDisplayMappingsDrop( tag_service_id )
-                        self._CacheCombinedFilesDisplayMappingsGenerate( tag_service_id, status_hook = status_hook_2 )
-                        
-                    
-                    self._controller.frame_splash_status.SetSubtext( '' )
-                    
-                except Exception as e:
-                    
-                    HydrusData.PrintException( e )
-                    
-                    raise Exception( 'Could not create the new combined tag display caches! Error was written to log, it also follows: {}{}'.format( os.linesep * 2, e ) )
-                    
-                
+            BlockingSafeShowMessage( message )
             
         
         if version == 411:
@@ -18441,18 +17736,7 @@ class DB( HydrusDB.HydrusDB ):
                 raise Exception( 'Could not create the new local hashes cache! The error has been printed to the log, please let hydev know!' )
                 
             
-            try:
-                
-                self._RegenerateLocalHashCache()
-                
-            except Exception as e:
-                
-                HydrusData.PrintException( e )
-                
-                message = 'Trying to populate the new local hashes cache failed! Please let hydrus dev know!'
-                
-                self.pub_initial_message( message )
-                
+            # took out local hash regen here due to later file service splitting, which regens local hash cache anyway
             
         
         if version == 429:
@@ -18984,6 +18268,77 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
+        if version == 446:
+            
+            result = self._c.execute( 'SELECT 1 FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;', ( 32, 'gelbooru md5' ) ).fetchone()
+            
+            if result is not None:
+                
+                try:
+                    
+                    self._c.execute( 'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;', ( 32, 'gelbooru md5' ) )
+                    
+                    script_info = ( 32, 'gelbooru md5', 2, HydrusData.GetNow(), '''["http://gelbooru.com/index.php", 0, 1, [55, 1, [[[4, "hex"]], "some hash bytes"]], "md5", {"s": "list", "page": "post"}, [[30, 6, ["we got sent back to main gallery page -- title test", 8, [27, 7, [[26, 1, [[62, 2, [0, "head", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "title", {}, 0, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], [true, [51, 1, [2, "Image List", null, null, "Image List"]]]]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-general"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], ""]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-copyright"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "series"]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-artist"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "creator"]], [30, 6, ["", 0, [27, 7, [[26, 1, [[62, 2, [0, "li", {"class": "tag-type-character"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, 1, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 1, "", [84, 1, [26, 1, []]]]], "character"]], [30, 6, ["we got sent back to main gallery page -- page links exist", 8, [27, 7, [[26, 1, [[62, 2, [0, "div", {"id": "paginator"}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]], [62, 2, [0, "a", {}, null, null, false, [51, 1, [3, "", null, null, "example string"]]]]]], 2, "class", [84, 1, [26, 1, []]]]], [true, [51, 1, [3, "", null, null, "pagination"]]]]]]]''' )
+                    
+                    self._c.execute( 'REPLACE INTO json_dumps_named VALUES ( ?, ?, ?, ?, ? );', script_info )
+                    
+                except Exception as e:
+                    
+                    HydrusData.PrintException( e )
+                    
+                    message = 'Trying to update gelbooru file lookup script failed! Please let hydrus dev know!'
+                    
+                    self.pub_initial_message( message )
+                    
+                
+            
+            #
+            
+            result = self._c.execute( 'SELECT 1 FROM sqlite_master WHERE name = ?;', ( 'current_files', ) ).fetchone()
+            
+            if result is not None:
+                
+                try:
+                    
+                    service_ids = self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES )
+                    
+                    for ( i, service_id ) in enumerate( service_ids ):
+                        
+                        self._controller.frame_splash_status.SetSubtext( 'reorganising file storage {}'.format( HydrusData.ConvertValueRangeToPrettyString( i + 1, len( service_ids ) ) ) )
+                        
+                        self.modules_files_storage.GenerateFilesTables( service_id )
+                        
+                        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = ClientDBFilesStorage.GenerateFilesTableNames( service_id )
+                        
+                        self._c.execute( 'INSERT INTO {} ( hash_id, timestamp ) SELECT hash_id, timestamp FROM current_files WHERE service_id = ?;'.format( current_files_table_name ), ( service_id, ) )
+                        self._c.execute( 'INSERT INTO {} ( hash_id, timestamp, original_timestamp ) SELECT hash_id, timestamp, original_timestamp FROM deleted_files WHERE service_id = ?;'.format( deleted_files_table_name ), ( service_id, ) )
+                        self._c.execute( 'INSERT INTO {} ( hash_id ) SELECT hash_id FROM file_transfers WHERE service_id = ?;'.format( pending_files_table_name ), ( service_id, ) )
+                        self._c.execute( 'INSERT INTO {} ( hash_id, reason_id ) SELECT hash_id, reason_id FROM file_petitions WHERE service_id = ?;'.format( petitioned_files_table_name ), ( service_id, ) )
+                        
+                        self._AnalyzeTable( current_files_table_name )
+                        self._AnalyzeTable( deleted_files_table_name )
+                        self._AnalyzeTable( pending_files_table_name )
+                        self._AnalyzeTable( petitioned_files_table_name )
+                        
+                    
+                    self._c.execute( 'DROP TABLE current_files;' )
+                    self._c.execute( 'DROP TABLE deleted_files;' )
+                    self._c.execute( 'DROP TABLE file_transfers;' )
+                    self._c.execute( 'DROP TABLE file_petitions;' )
+                    
+                except Exception as e:
+                    
+                    HydrusData.PrintException( e )
+                    
+                    raise Exception( 'Unfortunately, hydrus was unable to update your file storage to the new system! The error has been written to your log, please roll back to v446 and let hydev know!' )
+                    
+                
+            
+            #
+            
+            self._CacheLocalHashIdsGenerate()
+            
+        
         self._controller.frame_splash_status.SetTitleText( 'updated db to v{}'.format( HydrusData.ToHumanInt( version + 1 ) ) )
         
         self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
diff --git a/hydrus/client/db/ClientDBFilesStorage.py b/hydrus/client/db/ClientDBFilesStorage.py
new file mode 100644
index 00000000..1d3fd8b2
--- /dev/null
+++ b/hydrus/client/db/ClientDBFilesStorage.py
@@ -0,0 +1,665 @@
+import collections
+import sqlite3
+import typing
+
+from hydrus.core import HydrusConstants as HC
+from hydrus.core import HydrusData
+from hydrus.core import HydrusDB
+from hydrus.core import HydrusDBModule
+
+from hydrus.client.db import ClientDBMaster
+from hydrus.client.db import ClientDBServices
+
+def GenerateFilesTableNames( service_id: int ) -> typing.Tuple[ str, str, str, str ]:
+    
+    suffix = str( service_id )
+    
+    current_files_table_name = 'current_files_{}'.format( suffix )
+    
+    deleted_files_table_name = 'deleted_files_{}'.format( suffix )
+    
+    pending_files_table_name = 'pending_files_{}'.format( suffix )
+    
+    petitioned_files_table_name = 'petitioned_files_{}'.format( suffix )
+    
+    return ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name )
+    
+def GenerateFilesTableName( service_id: int, status: int ) -> str:
+    
+    ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+    
+    if status == HC.CONTENT_STATUS_CURRENT:
+        
+        return current_files_table_name
+        
+    elif status == HC.CONTENT_STATUS_DELETED:
+        
+        return deleted_files_table_name
+        
+    elif status == HC.CONTENT_STATUS_PENDING:
+        
+        return pending_files_table_name
+        
+    else:
+        
+        return petitioned_files_table_name
+        
+    
+class ClientDBFilesStorage( HydrusDBModule.HydrusDBModule ):
+    
+    def __init__( self, cursor: sqlite3.Cursor, modules_services: ClientDBServices.ClientDBMasterServices, modules_texts: ClientDBMaster.ClientDBMasterTexts ):
+        
+        self.modules_services = modules_services
+        self.modules_texts = modules_texts
+        
+        HydrusDBModule.HydrusDBModule.__init__( self, 'client files storage', cursor )
+        
+    
+    def _GetInitialIndexGenerationTuples( self ):
+        
+        index_generation_tuples = []
+        
+        return index_generation_tuples
+        
+    
+    def AddFiles( self, service_id, insert_rows ):
+        
+        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+        
+        self._c.executemany( 'INSERT OR IGNORE INTO {} VALUES ( ?, ? );'.format( current_files_table_name ), ( ( hash_id, timestamp ) for ( hash_id, timestamp ) in insert_rows ) )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( pending_files_table_name ), ( ( hash_id, ) for ( hash_id, timestamp ) in insert_rows ) )
+        
+        pending_changed = HydrusDB.GetRowCount( self._c ) > 0
+        
+        return pending_changed
+        
+    
+    def ClearDeleteRecord( self, service_id, hash_ids ):
+        
+        deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( deleted_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+        num_deleted = HydrusDB.GetRowCount( self._c )
+        
+        return num_deleted
+        
+    
+    def ClearLocalDeleteRecord( self, hash_ids = None ):
+        
+        # we delete from everywhere, but not for files currently in the trash
+        
+        service_ids_to_nums_cleared = {}
+        
+        local_non_trash_service_ids = self.modules_services.GetServiceIds( ( HC.COMBINED_LOCAL_FILE, HC.LOCAL_FILE_DOMAIN ) )
+        
+        if hash_ids is None:
+            
+            trash_current_files_table_name = GenerateFilesTableName( self.modules_services.trash_service_id, HC.CONTENT_STATUS_CURRENT )
+            
+            for service_id in local_non_trash_service_ids:
+                
+                deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+                
+                self._c.execute( 'DELETE FROM {} WHERE hash_id NOT IN ( SELECT hash_id FROM {} );'.format( deleted_files_table_name, trash_current_files_table_name ) )
+                
+                num_cleared = HydrusDB.GetRowCount( self._c )
+                
+                service_ids_to_nums_cleared[ service_id ] = num_cleared
+                
+            
+            self._c.execute( 'DELETE FROM local_file_deletion_reasons WHERE hash_id NOT IN ( SELECT hash_id FROM {} );'.format( trash_current_files_table_name ) )
+            
+        else:
+            
+            trashed_hash_ids = self.FilterCurrentHashIds( self.modules_services.trash_service_id, hash_ids )
+            
+            ok_to_clear_hash_ids = set( hash_ids ).difference( trashed_hash_ids )
+            
+            if len( ok_to_clear_hash_ids ) > 0:
+                
+                for service_id in local_non_trash_service_ids:
+                    
+                    deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+                    
+                    self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( deleted_files_table_name ), ( ( hash_id, ) for hash_id in ok_to_clear_hash_ids ) )
+                    
+                    num_cleared = HydrusDB.GetRowCount( self._c )
+                    
+                    service_ids_to_nums_cleared[ service_id ] = num_cleared
+                    
+                
+                self._c.executemany( 'DELETE FROM local_file_deletion_reasons WHERE hash_id = ?;', ( ( hash_id, ) for hash_id in ok_to_clear_hash_ids ) )
+                
+            
+        
+        return service_ids_to_nums_cleared
+        
+    
+    def CreateInitialTables( self ):
+        
+        self._c.execute( 'CREATE TABLE local_file_deletion_reasons ( hash_id INTEGER PRIMARY KEY, reason_id INTEGER );' )
+        
+    
+    def FilterAllCurrentHashIds( self, hash_ids, just_these_service_ids = None ):
+        
+        if just_these_service_ids is None:
+            
+            service_ids = self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES )
+            
+        else:
+            
+            service_ids = just_these_service_ids
+            
+        
+        current_hash_ids = set()
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            for service_id in service_ids:
+                
+                current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+                
+                hash_id_iterator = self._STI( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ) )
+                
+                current_hash_ids.update( hash_id_iterator )
+                
+            
+        
+        return current_hash_ids
+        
+    
+    def FilterAllPendingHashIds( self, hash_ids, just_these_service_ids = None ):
+        
+        if just_these_service_ids is None:
+            
+            service_ids = self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES )
+            
+        else:
+            
+            service_ids = just_these_service_ids
+            
+        
+        pending_hash_ids = set()
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            for service_id in service_ids:
+                
+                pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+                
+                hash_id_iterator = self._STI( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, pending_files_table_name ) ) )
+                
+                pending_hash_ids.update( hash_id_iterator )
+                
+            
+        
+        return pending_hash_ids
+        
+    
+    def FilterCurrentHashIds( self, service_id, hash_ids ):
+        
+        if service_id == self.modules_services.combined_file_service_id:
+            
+            return set( hash_ids )
+            
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+            
+            current_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ) )
+            
+        
+        return current_hash_ids
+        
+    
+    def FilterPendingHashIds( self, service_id, hash_ids ):
+        
+        if service_id == self.modules_services.combined_file_service_id:
+            
+            return set( hash_ids )
+            
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+            
+            pending_hash_ids = self._STS( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, pending_files_table_name ) ) )
+            
+        
+        return pending_hash_ids
+        
+    
+    def DeletePending( self, service_id: int ):
+        
+        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+        
+        self._c.execute( 'DELETE FROM {};'.format( pending_files_table_name ) )
+        self._c.execute( 'DELETE FROM {};'.format( petitioned_files_table_name ) )
+        
+    
+    def DropFilesTables( self, service_id: int ):
+        
+        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+        
+        self._c.execute( 'DROP TABLE IF EXISTS {};'.format( current_files_table_name ) )
+        self._c.execute( 'DROP TABLE IF EXISTS {};'.format( deleted_files_table_name ) )
+        self._c.execute( 'DROP TABLE IF EXISTS {};'.format( pending_files_table_name ) )
+        self._c.execute( 'DROP TABLE IF EXISTS {};'.format( petitioned_files_table_name ) )
+        
+    
+    def GenerateFilesTables( self, service_id: int ):
+        
+        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+        
+        self._c.execute( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER PRIMARY KEY, timestamp INTEGER );'.format( current_files_table_name ) )
+        self._CreateIndex( current_files_table_name, [ 'timestamp' ] )
+        
+        self._c.execute( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER PRIMARY KEY, timestamp INTEGER, original_timestamp INTEGER );'.format( deleted_files_table_name ) )
+        self._CreateIndex( deleted_files_table_name, [ 'timestamp' ] )
+        self._CreateIndex( deleted_files_table_name, [ 'original_timestamp' ] )
+        
+        self._c.execute( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER PRIMARY KEY );'.format( pending_files_table_name ) )
+        
+        self._c.execute( 'CREATE TABLE IF NOT EXISTS {} ( hash_id INTEGER PRIMARY KEY, reason_id INTEGER );'.format( petitioned_files_table_name ) )
+        self._CreateIndex( petitioned_files_table_name, [ 'reason_id' ] )
+        
+    
+    def GetAPendingHashId( self, service_id ):
+        
+        pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+        
+        result = self._c.execute( 'SELECT hash_id FROM {};'.format( pending_files_table_name ) ).fetchone()
+        
+        if result is None:
+            
+            return None
+            
+        else:
+            
+            ( hash_id, ) = result
+            
+            return hash_id
+            
+        
+    
+    def GetAPetitionedHashId( self, service_id ):
+        
+        petitioned_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PETITIONED )
+        
+        result = self._c.execute( 'SELECT hash_id FROM {};'.format( petitioned_files_table_name ) ).fetchone()
+        
+        if result is None:
+            
+            return None
+            
+        else:
+            
+            ( hash_id, ) = result
+            
+            return hash_id
+            
+        
+    
+    def GetCurrentFilesCount( self, service_id, only_viewable = False ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        if only_viewable:
+            
+            # hashes to mimes
+            result = self._c.execute( 'SELECT COUNT( * ) FROM {} CROSS JOIN files_info USING ( hash_id ) WHERE mime IN {};'.format( current_files_table_name, HydrusData.SplayListForDB( HC.SEARCHABLE_MIMES ) ) ).fetchone()
+            
+        else:
+            
+            result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( current_files_table_name ) ).fetchone()
+            
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetCurrentFilesInboxCount( self, service_id ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {} CROSS JOIN file_inbox USING ( hash_id );'.format( current_files_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetCurrentHashIdsList( self, service_id ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        hash_ids = self._STL( self._c.execute( 'SELECT hash_id FROM {};'.format( current_files_table_name ) ) )
+        
+        return hash_ids
+        
+    
+    def GetCurrentFilesTotalSize( self, service_id ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        # hashes to size
+        result = self._c.execute( 'SELECT SUM( size ) FROM {} CROSS JOIN files_info USING ( hash_id );'.format( current_files_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetCurrentHashIdsToTimestamps( self, service_id, hash_ids ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            rows = dict( self._c.execute( 'SELECT hash_id, timestamp FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ) )
+            
+        
+        return rows
+        
+    
+    def GetCurrentTableJoinPhrase( self, service_id, table_name ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        return '{} CROSS JOIN {} USING ( hash_id )'.format( table_name, current_files_table_name )
+        
+    
+    def GetCurrentTimestamp( self, service_id: int, hash_id: int ):
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        result = self._c.execute( 'SELECT timestamp FROM {} WHERE hash_id = ?;'.format( current_files_table_name ), ( hash_id, ) ).fetchone()
+        
+        if result is None:
+            
+            return None
+            
+        else:
+            
+            ( timestamp, ) = result
+            
+            return timestamp
+            
+        
+    
+    def GetDeletedFilesCount( self, service_id: int ) -> int:
+        
+        deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( deleted_files_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetDeletionStatus( self, service_id, hash_id ):
+        
+        # can have a value here and just be in trash, so we fetch it whatever the end result
+        result = self._c.execute( 'SELECT reason_id FROM local_file_deletion_reasons WHERE hash_id = ?;', ( hash_id, ) ).fetchone()
+        
+        if result is None:
+            
+            file_deletion_reason = 'Unknown deletion reason.'
+            
+        else:
+            
+            ( reason_id, ) = result
+            
+            file_deletion_reason = self.modules_texts.GetText( reason_id )
+            
+        
+        deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+        
+        is_deleted = False
+        timestamp = None
+        
+        result = self._c.execute( 'SELECT timestamp FROM {} WHERE hash_id = ?;'.format( deleted_files_table_name ), ( hash_id, ) ).fetchone()
+        
+        if result is not None:
+            
+            is_deleted = True
+            
+            ( timestamp, ) = result
+            
+        
+        return ( is_deleted, timestamp, file_deletion_reason )
+        
+    
+    def GetExpectedTableNames( self ) -> typing.Collection[ str ]:
+        
+        expected_table_names = [
+            'local_file_deletion_reasons',
+        ]
+        
+        return expected_table_names
+        
+    
+    def GetHashIdsToCurrentServiceIds( self, temp_hash_ids_table_name ):
+        
+        hash_ids_to_current_file_service_ids = collections.defaultdict( list )
+        
+        for service_id in self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES ):
+            
+            current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+            
+            for hash_id in self._STI( self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ) ):
+                
+                hash_ids_to_current_file_service_ids[ hash_id ].append( service_id )
+                
+            
+        
+        return hash_ids_to_current_file_service_ids
+        
+    
+    def GetHashIdsToServiceInfoDicts( self, temp_hash_ids_table_name ):
+        
+        hash_ids_to_current_file_service_ids_and_timestamps = collections.defaultdict( list )
+        hash_ids_to_deleted_file_service_ids_and_timestamps = collections.defaultdict( list )
+        hash_ids_to_pending_file_service_ids = collections.defaultdict( list )
+        hash_ids_to_petitioned_file_service_ids = collections.defaultdict( list )
+        
+        for service_id in self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES ):
+            
+            ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+            
+            for ( hash_id, timestamp ) in self._c.execute( 'SELECT hash_id, timestamp FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ):
+                
+                hash_ids_to_current_file_service_ids_and_timestamps[ hash_id ].append( ( service_id, timestamp ) )
+                
+            
+            for ( hash_id, timestamp, original_timestamp ) in self._c.execute( 'SELECT hash_id, timestamp, original_timestamp FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, deleted_files_table_name ) ):
+                
+                hash_ids_to_deleted_file_service_ids_and_timestamps[ hash_id ].append( ( service_id, timestamp, original_timestamp ) )
+                
+            
+            for hash_id in self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, pending_files_table_name ) ):
+                
+                hash_ids_to_pending_file_service_ids[ hash_id ].append( service_id )
+                
+            
+            for hash_id in self._c.execute( 'SELECT hash_id FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, petitioned_files_table_name ) ):
+                
+                hash_ids_to_petitioned_file_service_ids[ hash_id ].append( service_id )
+                
+            
+        
+        return (
+            hash_ids_to_current_file_service_ids_and_timestamps,
+            hash_ids_to_deleted_file_service_ids_and_timestamps,
+            hash_ids_to_pending_file_service_ids,
+            hash_ids_to_petitioned_file_service_ids
+        )
+        
+    
+    def GetNumLocal( self, service_id: int ) -> int:
+        
+        current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+        combined_local_current_files_table_name = GenerateFilesTableName( self.modules_services.combined_local_file_service_id, HC.CONTENT_STATUS_CURRENT )
+        
+        ( num_local, ) = self._c.execute( 'SELECT COUNT( * ) FROM {} CROSS JOIN {} USING ( hash_id );'.format( current_files_table_name, combined_local_current_files_table_name ) ).fetchone()
+        
+        return num_local
+        
+    
+    def GetPendingFilesCount( self, service_id: int ) -> int:
+        
+        pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( pending_files_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetPetitionedFilesCount( self, service_id: int ) -> int:
+        
+        petitioned_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PETITIONED )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( petitioned_files_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetServiceIdCounts( self, hash_ids ) -> typing.Dict[ int, int ]:
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            service_ids_to_counts = {}
+            
+            for service_id in self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES ):
+                
+                current_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_CURRENT )
+                
+                # temp hashes to files
+                ( count, ) = self._c.execute( 'SELECT COUNT( * ) FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, current_files_table_name ) ).fetchone()
+                
+                service_ids_to_counts[ service_id ] = count
+                
+            
+        
+        return service_ids_to_counts
+        
+    
+    def GetSomePetitionedRows( self, service_id: int ):
+        
+        petitioned_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PETITIONED )
+        
+        petitioned_rows = list( HydrusData.BuildKeyToListDict( self._c.execute( 'SELECT reason_id, hash_id FROM {} ORDER BY reason_id LIMIT 100;'.format( petitioned_files_table_name ) ) ).items() )
+        
+        return petitioned_rows
+        
+    
+    def GetTablesAndColumnsThatUseDefinitions( self, content_type: int ) -> typing.List[ typing.Tuple[ str, str ] ]:
+        
+        if HC.CONTENT_TYPE_HASH:
+            
+            tables_and_columns = []
+            
+            for service_id in self.modules_services.GetServiceIds( HC.SPECIFIC_FILE_SERVICES ):
+                
+                ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+                
+                tables_and_columns.extend( [
+                    ( current_files_table_name, 'hash_id' ),
+                    ( deleted_files_table_name, 'hash_id' ),
+                    ( pending_files_table_name, 'hash_id' ),
+                    ( petitioned_files_table_name, 'hash_id' )
+                ] )
+                
+            
+            return tables_and_columns
+            
+        
+        return []
+        
+    
+    def GetUndeleteRows( self, service_id, hash_ids ):
+        
+        deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+        
+        with HydrusDB.TemporaryIntegerTable( self._c, hash_ids, 'hash_id' ) as temp_hash_ids_table_name:
+            
+            rows = self._c.execute( 'SELECT hash_id, original_timestamp FROM {} CROSS JOIN {} USING ( hash_id );'.format( temp_hash_ids_table_name, deleted_files_table_name ) ).fetchall()
+            
+        
+        return rows
+        
+    
+    def PendFiles( self, service_id, hash_ids ):
+        
+        pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+        
+        self._c.executemany( 'INSERT OR IGNORE INTO {} ( hash_id ) VALUES ( ? );'.format( pending_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+    
+    def PetitionFiles( self, service_id, reason_id, hash_ids ):
+        
+        petitioned_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PETITIONED )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( petitioned_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+        self._c.executemany( 'INSERT OR IGNORE INTO {} ( hash_id, reason_id ) VALUES ( ?, ? );'.format( petitioned_files_table_name ), ( ( hash_id, reason_id ) for hash_id in hash_ids ) )
+        
+    
+    def RecordDeleteFiles( self, service_id, insert_rows ):
+        
+        deleted_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_DELETED )
+        
+        now = HydrusData.GetNow()
+        
+        self._c.executemany(
+            'INSERT OR IGNORE INTO {} ( hash_id, timestamp, original_timestamp ) VALUES ( ?, ?, ? );'.format( deleted_files_table_name ),
+            ( ( hash_id, now, original_timestamp ) for ( hash_id, original_timestamp ) in insert_rows )
+        )
+        
+        num_new_deleted_files = HydrusDB.GetRowCount( self._c )
+        
+        return num_new_deleted_files
+        
+    
+    def RescindPendFiles( self, service_id, hash_ids ):
+        
+        pending_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PENDING )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( pending_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+    
+    def RescindPetitionFiles( self, service_id, hash_ids ):
+        
+        petitioned_files_table_name = GenerateFilesTableName( service_id, HC.CONTENT_STATUS_PETITIONED )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( petitioned_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+    
+    def RemoveFiles( self, service_id, hash_ids ):
+        
+        ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = GenerateFilesTableNames( service_id )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( current_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+        self._c.executemany( 'DELETE FROM {} WHERE hash_id = ?;'.format( petitioned_files_table_name ), ( ( hash_id, ) for hash_id in hash_ids ) )
+        
+        pending_changed = HydrusDB.GetRowCount( self._c ) > 0
+        
+        return pending_changed
+        
+    
+    def SetFileDeletionReason( self, hash_ids, reason ):
+        
+        reason_id = self.modules_texts.GetTextId( reason )
+        
+        self._c.executemany( 'REPLACE INTO local_file_deletion_reasons ( hash_id, reason_id ) VALUES ( ?, ? );', ( ( hash_id, reason_id ) for hash_id in hash_ids ) )
+        
diff --git a/hydrus/client/db/ClientDBMappingsStorage.py b/hydrus/client/db/ClientDBMappingsStorage.py
index 22361eb6..b563cc01 100644
--- a/hydrus/client/db/ClientDBMappingsStorage.py
+++ b/hydrus/client/db/ClientDBMappingsStorage.py
@@ -75,22 +75,48 @@ class ClientDBMappingsStorage( HydrusDBModule.HydrusDBModule ):
         self._CreateIndex( petitioned_mappings_table_name, [ 'hash_id', 'tag_id' ], unique = True )
         
     
-    def GetPendingCount( self, service_id ):
+    def GetCurrentFilesCount( self, service_id: int ) -> int:
+        
+        ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
+        
+        result = self._c.execute( 'SELECT COUNT( DISTINCT hash_id ) FROM {};'.format( current_mappings_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetDeletedMappingsCount( self, service_id: int ) -> int:
+        
+        ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( deleted_mappings_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetPendingMappingsCount( self, service_id: int ) -> int:
         
         ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
         
         result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( pending_mappings_table_name ) ).fetchone()
         
-        if result is None:
-            
-            return 0
-            
-        else:
-            
-            ( count, ) = result
-            
-            return count
-            
+        ( count, ) = result
+        
+        return count
+        
+    
+    def GetPetitionedMappingsCount( self, service_id: int ) -> int:
+        
+        ( current_mappings_table_name, deleted_mappings_table_name, pending_mappings_table_name, petitioned_mappings_table_name ) = GenerateMappingsTableNames( service_id )
+        
+        result = self._c.execute( 'SELECT COUNT( * ) FROM {};'.format( petitioned_mappings_table_name ) ).fetchone()
+        
+        ( count, ) = result
+        
+        return count
         
     
     def GetTablesAndColumnsThatUseDefinitions( self, content_type: int ) -> typing.List[ typing.Tuple[ str, str ] ]:
diff --git a/hydrus/client/db/ClientDBSimilarFiles.py b/hydrus/client/db/ClientDBSimilarFiles.py
index 7eb689c9..f06d91be 100644
--- a/hydrus/client/db/ClientDBSimilarFiles.py
+++ b/hydrus/client/db/ClientDBSimilarFiles.py
@@ -10,10 +10,15 @@ from hydrus.core import HydrusDBModule
 from hydrus.core import HydrusGlobals as HG
 
 from hydrus.client import ClientThreading
+from hydrus.client.db import ClientDBFilesStorage
+from hydrus.client.db import ClientDBServices
 
 class ClientDBSimilarFiles( HydrusDBModule.HydrusDBModule ):
     
-    def __init__( self, cursor: sqlite3.Cursor ):
+    def __init__( self, cursor: sqlite3.Cursor, modules_services: ClientDBServices.ClientDBMasterServices, modules_files_storage: ClientDBFilesStorage.ClientDBFilesStorage ):
+        
+        self.modules_services = modules_services
+        self.modules_files_storage = modules_files_storage
         
         HydrusDBModule.HydrusDBModule.__init__( self, 'client similar files', cursor )
         
@@ -548,7 +553,9 @@ class ClientDBSimilarFiles( HydrusDBModule.HydrusDBModule ):
             
             job_key.SetVariable( 'popup_text_1', 'purging search info of orphans' )
             
-            self._c.execute( 'DELETE FROM shape_perceptual_hash_map WHERE hash_id NOT IN ( SELECT hash_id FROM current_files );' )
+            ( current_files_table_name, deleted_files_table_name, pending_files_table_name, petitioned_files_table_name ) = ClientDBFilesStorage.GenerateFilesTableNames( self.modules_services.combined_local_file_service_id )
+            
+            self._c.execute( 'DELETE FROM shape_perceptual_hash_map WHERE hash_id NOT IN ( SELECT hash_id FROM {} );'.format( current_files_table_name ) )
             
             job_key.SetVariable( 'popup_text_1', 'gathering all leaves' )
             
diff --git a/hydrus/client/gui/ClientGUI.py b/hydrus/client/gui/ClientGUI.py
index f04c6408..678c661b 100644
--- a/hydrus/client/gui/ClientGUI.py
+++ b/hydrus/client/gui/ClientGUI.py
@@ -4759,18 +4759,10 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
             HG.cache_report_mode = not HG.cache_report_mode
             
-        elif name == 'callto_profile_mode':
-            
-            HG.callto_profile_mode = not HG.callto_profile_mode
-            
         elif name == 'db_report_mode':
             
             HG.db_report_mode = not HG.db_report_mode
             
-        elif name == 'db_profile_mode':
-            
-            HG.db_profile_mode = not HG.db_profile_mode
-            
         elif name == 'db_ui_hang_relief_mode':
             
             HG.db_ui_hang_relief_mode = not HG.db_ui_hang_relief_mode
@@ -4795,10 +4787,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
             HG.media_load_report_mode = not HG.media_load_report_mode
             
-        elif name == 'menu_profile_mode':
-            
-            HG.menu_profile_mode = not HG.menu_profile_mode
-            
         elif name == 'network_report_mode':
             
             HG.network_report_mode = not HG.network_report_mode
@@ -4807,18 +4795,40 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
             HG.phash_generation_report_mode = not HG.phash_generation_report_mode
             
+        elif name == 'profile_mode':
+            
+            if not HG.profile_mode:
+                
+                now = HydrusData.GetNow()
+                
+                with HG.profile_counter_lock:
+                    
+                    HG.profile_start_time = now
+                    HG.profile_slow_count = 0
+                    HG.profile_fast_count = 0
+                    
+                
+                
+                HG.profile_mode = True
+                
+                HydrusData.ShowText( 'Profile mode on!' )
+                
+            else:
+                
+                HG.profile_mode = False
+                
+                with HG.profile_counter_lock:
+                    
+                    ( slow, fast ) = ( HG.profile_slow_count, HG.profile_fast_count )
+                    
+                
+                HydrusData.ShowText( 'Profiling done: {} slow jobs, {} fast jobs'.format( HydrusData.ToHumanInt( slow ), HydrusData.ToHumanInt( fast ) ) )
+                
+            
         elif name == 'pubsub_report_mode':
             
             HG.pubsub_report_mode = not HG.pubsub_report_mode
             
-        elif name == 'pubsub_profile_mode':
-            
-            HG.pubsub_profile_mode = not HG.pubsub_profile_mode
-            
-        elif name == 'server_profile_mode':
-            
-            HG.server_profile_mode = not HG.server_profile_mode
-            
         elif name == 'shortcut_report_mode':
             
             HG.shortcut_report_mode = not HG.shortcut_report_mode
@@ -4835,19 +4845,6 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
             HG.thumbnail_debug_mode = not HG.thumbnail_debug_mode
             
-        elif name == 'ui_timer_profile_mode':
-            
-            HG.ui_timer_profile_mode = not HG.ui_timer_profile_mode
-            
-            if HG.ui_timer_profile_mode:
-                
-                HydrusData.ShowText( 'ui timer profile mode activated' )
-                
-            else:
-                
-                HydrusData.ShowText( 'ui timer profile mode deactivated' )
-                
-            
         elif name == 'force_idle_mode':
             
             HG.force_idle_mode = not HG.force_idle_mode
@@ -5339,11 +5336,11 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
                 
                 try:
                     
-                    if HG.ui_timer_profile_mode:
+                    if HG.profile_mode:
                         
                         summary = 'Profiling animation timer: ' + repr( window )
                         
-                        HydrusData.Profile( summary, 'window.TIMERAnimationUpdate()', globals(), locals(), min_duration_ms = 3, show_summary = True )
+                        HydrusData.Profile( summary, 'window.TIMERAnimationUpdate()', globals(), locals(), min_duration_ms = HG.ui_timer_profile_min_job_time_ms )
                         
                     else:
                         
@@ -5943,23 +5940,18 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
             ClientGUIMenus.AppendMenu( debug, debug_modes, 'debug modes' )
             
-            profile_modes = QW.QMenu( debug )
+            profiling = QW.QMenu( debug )
             
-            profile_mode_message = 'If something is running slow, you can turn on one of these modes to have hydrus gather information on how long each part takes to run. You probably want \'db profile mode\'.'
+            profile_mode_message = 'If something is running slow, you can turn on profile mode to have hydrus gather information on how long many jobs take to run.'
             profile_mode_message += os.linesep * 2
-            profile_mode_message += 'Turn the mode on, do the slow thing for a bit, and then turn it off. In your database directory will be a new profile log, which is really helpful for hydrus dev to figure out what in your case is running slow.'
+            profile_mode_message += 'Turn the mode on, do the slow thing for a bit, and then turn it off. In your database directory will be a new profile log, which is really helpful for hydrus dev to figure out what is running slow for you and how to fix it.'
             profile_mode_message += os.linesep * 2
             profile_mode_message += 'More information is available in the help, under \'reducing program lag\'.'
             
-            ClientGUIMenus.AppendMenuItem( profile_modes, 'what is this?', 'Show profile info.', QW.QMessageBox.information, self, 'Profile modes', profile_mode_message )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'callto profile mode', 'Run detailed \'profiles\' on most threaded jobs and dump this information to the log (this is very useful for hydrus dev to have, if something is running slow for you in UI!).', HG.callto_profile_mode, self._SwitchBoolean, 'callto_profile_mode' )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'client api profile mode', 'Run detailed \'profiles\' on every client api query and dump this information to the log (this is very useful for hydrus dev to have, if something is running slow for you!).', HG.server_profile_mode, self._SwitchBoolean, 'server_profile_mode' )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'db profile mode', 'Run detailed \'profiles\' on every database query and dump this information to the log (this is very useful for hydrus dev to have, if something is running slow for you in the DB!).', HG.db_profile_mode, self._SwitchBoolean, 'db_profile_mode' )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'menu profile mode', 'Run detailed \'profiles\' on menu actions.', HG.menu_profile_mode, self._SwitchBoolean, 'menu_profile_mode' )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'pubsub profile mode', 'Run detailed \'profiles\' on every internal publisher/subscriber message and dump this information to the log. This can hammer your log with dozens of large dumps every second. Don\'t run it unless you know you need to.', HG.pubsub_profile_mode, self._SwitchBoolean, 'pubsub_profile_mode' )
-            ClientGUIMenus.AppendMenuCheckItem( profile_modes, 'ui timer profile mode', 'Run detailed \'profiles\' on every ui timer update. This will likely spam you!', HG.ui_timer_profile_mode, self._SwitchBoolean, 'ui_timer_profile_mode' )
+            ClientGUIMenus.AppendMenuItem( profiling, 'what is this?', 'Show profile info.', QW.QMessageBox.information, self, 'Profile modes', profile_mode_message )
+            ClientGUIMenus.AppendMenuCheckItem( profiling, 'profile mode', 'Run detailed \'profiles\'.', HG.profile_mode, self._SwitchBoolean, 'profile_mode' )
             
-            ClientGUIMenus.AppendMenu( debug, profile_modes, 'profile modes' )
+            ClientGUIMenus.AppendMenu( debug, profiling, 'profiling' )
             
             report_modes = QW.QMenu( debug )
             
@@ -7184,11 +7176,11 @@ Try to keep this below 10 million!'''
         
         if page is not None:
             
-            if HG.ui_timer_profile_mode:
+            if HG.profile_mode:
                 
                 summary = 'Profiling page timer: ' + repr( page )
                 
-                HydrusData.Profile( summary, 'page.REPEATINGPageUpdate()', globals(), locals(), min_duration_ms = 3, show_summary = True )
+                HydrusData.Profile( summary, 'page.REPEATINGPageUpdate()', globals(), locals(), min_duration_ms = HG.ui_timer_profile_min_job_time_ms )
                 
             else:
                 
@@ -7229,11 +7221,11 @@ Try to keep this below 10 million!'''
             
             try:
                 
-                if HG.ui_timer_profile_mode:
+                if HG.profile_mode:
                     
                     summary = 'Profiling ui update timer: ' + repr( window )
                     
-                    HydrusData.Profile( summary, 'window.TIMERUIUpdate()', globals(), locals(), min_duration_ms = 3, show_summary = True )
+                    HydrusData.Profile( summary, 'window.TIMERUIUpdate()', globals(), locals(), min_duration_ms = HG.ui_timer_profile_min_job_time_ms )
                     
                 else:
                     
diff --git a/hydrus/client/gui/ClientGUIDragDrop.py b/hydrus/client/gui/ClientGUIDragDrop.py
index 34758c17..fe81bf81 100644
--- a/hydrus/client/gui/ClientGUIDragDrop.py
+++ b/hydrus/client/gui/ClientGUIDragDrop.py
@@ -7,6 +7,7 @@ from qtpy import QtWidgets as QW
 from hydrus.core import HydrusConstants as HC
 from hydrus.core import HydrusGlobals as HG
 from hydrus.core import HydrusPaths
+from hydrus.core import HydrusText
 
 from hydrus.client import ClientExporting
 from hydrus.client.gui import ClientGUIFunctions
@@ -250,47 +251,65 @@ class FileDropTarget( QC.QObject ):
 
             result = QC.Qt.MoveAction
             '''
-        elif urls_dnd and self._filenames_callable is not None:
+        elif urls_dnd or text_dnd:
             
             paths = []
             urls = []
             
-            for url in mime_data.urls():
+            if urls_dnd:
                 
-                if url.isLocalFile():
+                dnd_items = mime_data.urls()
+                
+                for dnd_item in dnd_items:
                     
-                    paths.append( os.path.normpath( url.toLocalFile() ) )
+                    if dnd_item.isLocalFile():
+                        
+                        paths.append( os.path.normpath( dnd_item.toLocalFile() ) )
+                        
+                    else:
+                        
+                        urls.append( dnd_item.url() )
+                        
                     
-                else:
+                
+            else:
+                
+                text = mime_data.text()
+                
+                text_lines = HydrusText.DeserialiseNewlinedTexts( text )
+                
+                for text_line in text_lines:
                     
-                    urls.append( url.url() )
+                    if text_line.startswith( 'http' ):
+                        
+                        urls.append( text_line )
+                        
+                        # ignore 'paths'
+                        
                     
                 
             
-            if len( paths ) > 0:
+            if self._filenames_callable is not None:
                 
-                QP.CallAfter( self._filenames_callable, paths ) # callafter to terminate dnd event now
+                if len( paths ) > 0:
+                    
+                    QP.CallAfter( self._filenames_callable, paths ) # callafter to terminate dnd event now
+                    
                 
             
-            if len( urls ) > 0:
+            if self._url_callable is not None:
                 
-                for url in urls:
+                if len( urls ) > 0:
                     
-                    QP.CallAfter( self._url_callable, url ) # callafter to terminate dnd event now
+                    for url in urls:
+                        
+                        QP.CallAfter( self._url_callable, url ) # callafter to terminate dnd event now
+                        
                     
                 
             
             result = QC.Qt.IgnoreAction
             
-        elif text_dnd and self._url_callable is not None:
-            
-            text = mime_data.text()
-            
-            QP.CallAfter( self._url_callable, text ) # callafter to terminate dnd event now
-            
-            result = QC.Qt.CopyAction
-            
-            
         else:
             
             result = QC.Qt.IgnoreAction
diff --git a/hydrus/client/gui/ClientGUIFileSeedCache.py b/hydrus/client/gui/ClientGUIFileSeedCache.py
index 9f7407fd..6a980b4b 100644
--- a/hydrus/client/gui/ClientGUIFileSeedCache.py
+++ b/hydrus/client/gui/ClientGUIFileSeedCache.py
@@ -28,6 +28,16 @@ from hydrus.client.gui.widgets import ClientGUICommon
 from hydrus.client.importing import ClientImportFileSeeds
 from hydrus.client.importing.options import FileImportOptions
 
+def GetRetryIgnoredParam( window ):
+    
+    choice_tuples = [
+        ( 'retry all', None, 'retry all' ),
+        ( 'retry 404s', '^404', 'retry all 404s' ),
+        ( 'retry blacklisted', 'blacklisted!$', 'retry all blacklisted' )
+    ]
+    
+    return ClientGUIDialogsQuick.SelectFromListButtons( window, 'select what to retry', choice_tuples )
+    
 class EditFileSeedCachePanel( ClientGUIScrolledPanels.EditPanel ):
     
     def __init__( self, parent, controller, file_seed_cache ):
@@ -523,16 +533,18 @@ class FileSeedCacheButton( ClientGUICommon.BetterButton ):
     
     def _RetryIgnored( self ):
         
-        message = 'Are you sure you want to retry all the files that were ignored/vetoed?'
+        try:
+            
+            ignored_regex = GetRetryIgnoredParam( self )
+            
+        except HydrusExceptions.CancelledException:
+            
+            return
+            
         
-        result = ClientGUIDialogsQuick.GetYesNo( self, message )
+        file_seed_cache = self._file_seed_cache_get_callable()
         
-        if result == QW.QDialog.Accepted:
-            
-            file_seed_cache = self._file_seed_cache_get_callable()
-            
-            file_seed_cache.RetryIgnored()
-            
+        file_seed_cache.RetryIgnored( ignored_regex = ignored_regex )
         
     
     def _ShowFileSeedCacheFrame( self ):
diff --git a/hydrus/client/gui/ClientGUIFunctions.py b/hydrus/client/gui/ClientGUIFunctions.py
index 88cbd755..64c8bfd3 100644
--- a/hydrus/client/gui/ClientGUIFunctions.py
+++ b/hydrus/client/gui/ClientGUIFunctions.py
@@ -1,4 +1,4 @@
-import collections
+import typing
 
 from qtpy import QtCore as QC
 from qtpy import QtWidgets as QW
@@ -22,9 +22,6 @@ def ClientToScreen( win: QW.QWidget, pos: QC.QPoint ) -> QC.QPoint:
         return QC.QPoint( 50, 50 )
         
     
-
-MAGIC_TEXT_PADDING = 1.1
-
 def ColourIsBright( colour: QG.QColor ):
     
     it_is_bright = colour.valueF() > 0.75
@@ -37,29 +34,50 @@ def ColourIsGreyish( colour: QG.QColor ):
     
     return it_is_greyish
     
-def ConvertPixelsToTextWidth( window, pixels, round_down = False ):
+# OK, so we now have a fixed block for width, which we sometimes want to calculate in both directions.
+# by normalising our 'one character' width, the inverse calculation uses the same coefficient and we aren't losing so much in rounding
+NUM_CHARS_FOR_WIDTH_CALCULATIONS = 32
+MAGIC_TEXT_PADDING = 1.1
+
+def GetOneCharacterPixelHeight( window ) -> float:
     
-    twenty_chars_in_pixels = int( window.fontMetrics().boundingRect( 20 * 'x' ).width() * MAGIC_TEXT_PADDING )
-    one_char_in_pixels = twenty_chars_in_pixels / 20
+    return window.fontMetrics().height() * MAGIC_TEXT_PADDING
+    
+def GetOneCharacterPixelWidth( window ) -> float:
+    
+    char_block_width = window.fontMetrics().boundingRect( NUM_CHARS_FOR_WIDTH_CALCULATIONS * 'x' ).width() * MAGIC_TEXT_PADDING
+    
+    one_char_width = char_block_width / NUM_CHARS_FOR_WIDTH_CALCULATIONS
+    
+    return one_char_width
+    
+def ConvertPixelsToTextWidth( window, pixels, round_down = False ) -> int:
+    
+    one_char_width = GetOneCharacterPixelWidth( window )
     
     if round_down:
         
-        return int( pixels // one_char_in_pixels )
+        return int( pixels // one_char_width )
         
     else:
         
-        return round( pixels / one_char_in_pixels )
+        return round( pixels / one_char_width )
         
     
-def ConvertTextToPixels( window, char_dimensions ):
+def ConvertTextToPixels( window, char_dimensions ) -> typing.Tuple[ int, int ]:
     
     ( char_cols, char_rows ) = char_dimensions
     
-    return ( int( window.fontMetrics().boundingRect( char_cols * 'x' ).width() * MAGIC_TEXT_PADDING ), int( char_rows * window.fontMetrics().height() * MAGIC_TEXT_PADDING ) )
+    one_char_width = GetOneCharacterPixelWidth( window )
+    one_char_height = GetOneCharacterPixelHeight( window )
     
-def ConvertTextToPixelWidth( window, char_cols ):
+    return ( round( char_cols * one_char_width ), round( char_rows * one_char_height ) )
     
-    return int( window.fontMetrics().boundingRect( char_cols * 'x' ).width() * MAGIC_TEXT_PADDING )
+def ConvertTextToPixelWidth( window, char_cols ) -> int:
+    
+    one_char_width = GetOneCharacterPixelWidth( window )
+    
+    return round( char_cols * one_char_width )
     
 def DialogIsOpen():
     
diff --git a/hydrus/client/gui/ClientGUIMenus.py b/hydrus/client/gui/ClientGUIMenus.py
index 68abe40a..9d839b05 100644
--- a/hydrus/client/gui/ClientGUIMenus.py
+++ b/hydrus/client/gui/ClientGUIMenus.py
@@ -163,11 +163,11 @@ def GetEventCallable( callable, *args, **kwargs ):
     
     def event_callable( checked_state ):
         
-        if HG.menu_profile_mode:
+        if HG.profile_mode:
             
             summary = 'Profiling menu: ' + repr( callable )
             
-            HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = 3, show_summary = True )
+            HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = HG.menu_profile_min_job_time_ms )
             
         else:
             
diff --git a/hydrus/client/gui/ClientGUISubscriptions.py b/hydrus/client/gui/ClientGUISubscriptions.py
index 719a8910..e6a3fa0a 100644
--- a/hydrus/client/gui/ClientGUISubscriptions.py
+++ b/hydrus/client/gui/ClientGUISubscriptions.py
@@ -998,16 +998,25 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
     
     def _STARTRetryIgnored( self ):
         
+        try:
+            
+            ignored_regex = ClientGUIFileSeedCache.GetRetryIgnoredParam( self )
+            
+        except HydrusExceptions.CancelledException:
+            
+            return
+            
+        
         selected_query_headers = self._query_headers.GetData( only_selected = True )
         
         query_headers = [ query_header for query_header in selected_query_headers if query_header.CanRetryIgnored() ]
         
-        call = HydrusData.Call( self._RetryIgnored, query_headers )
+        call = HydrusData.Call( self._RetryIgnored, query_headers, ignored_regex )
         
         self._DoAsyncGetQueryLogContainers( query_headers, call )
         
     
-    def _RetryIgnored( self, query_headers: typing.Collection[ ClientImportSubscriptionQuery.SubscriptionQueryHeader ] ):
+    def _RetryIgnored( self, query_headers: typing.Collection[ ClientImportSubscriptionQuery.SubscriptionQueryHeader ], ignored_regex = typing.Optional[ str ] ):
         
         for query_header in query_headers:
             
@@ -1020,7 +1029,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
             
             query_log_container = self._names_to_edited_query_log_containers[ query_log_container_name ]
             
-            query_log_container.GetFileSeedCache().RetryIgnored()
+            query_log_container.GetFileSeedCache().RetryIgnored( ignored_regex = ignored_regex )
             
             query_header.UpdateFileStatus( query_log_container )
             
@@ -1889,6 +1898,15 @@ class EditSubscriptionsPanel( ClientGUIScrolledPanels.EditPanel ):
     
     def _STARTRetryIgnored( self ):
         
+        try:
+            
+            ignored_regex = ClientGUIFileSeedCache.GetRetryIgnoredParam( self )
+            
+        except HydrusExceptions.CancelledException:
+            
+            return
+            
+        
         query_headers = []
         
         subscriptions = self._subscriptions.GetData( only_selected = True )
@@ -1900,12 +1918,12 @@ class EditSubscriptionsPanel( ClientGUIScrolledPanels.EditPanel ):
         
         query_headers = [ query_header for query_header in query_headers if query_header.CanRetryIgnored() ]
         
-        call = HydrusData.Call( self._RetryIgnored, query_headers )
+        call = HydrusData.Call( self._RetryIgnored, query_headers, ignored_regex )
         
         self._DoAsyncGetQueryLogContainers( query_headers, call )
         
     
-    def _RetryIgnored( self, query_headers: typing.Iterable[ ClientImportSubscriptionQuery.SubscriptionQueryHeader ] ):
+    def _RetryIgnored( self, query_headers: typing.Iterable[ ClientImportSubscriptionQuery.SubscriptionQueryHeader ], ignored_regex: typing.Optional[ str ] ):
         
         for query_header in query_headers:
             
@@ -1918,7 +1936,7 @@ class EditSubscriptionsPanel( ClientGUIScrolledPanels.EditPanel ):
             
             query_log_container = self._names_to_edited_query_log_containers[ query_log_container_name ]
             
-            query_log_container.GetFileSeedCache().RetryIgnored()
+            query_log_container.GetFileSeedCache().RetryIgnored( ignored_regex = ignored_regex )
             
             query_header.UpdateFileStatus( query_log_container )
             
diff --git a/hydrus/client/gui/ClientGUITags.py b/hydrus/client/gui/ClientGUITags.py
index 0c6d2a9d..c8194ef0 100644
--- a/hydrus/client/gui/ClientGUITags.py
+++ b/hydrus/client/gui/ClientGUITags.py
@@ -2895,11 +2895,11 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ):
             
             self._show_all = QW.QCheckBox( self )
             
-            listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
+            self._listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
             
-            self._tag_parents = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, CGLC.COLUMN_LIST_TAG_PARENTS.ID, 8, self._ConvertPairToListCtrlTuples, delete_key_callback = self._ListCtrlActivated, activation_callback = self._ListCtrlActivated )
+            self._tag_parents = ClientGUIListCtrl.BetterListCtrl( self._listctrl_panel, CGLC.COLUMN_LIST_TAG_PARENTS.ID, 8, self._ConvertPairToListCtrlTuples, delete_key_callback = self._ListCtrlActivated, activation_callback = self._ListCtrlActivated )
             
-            listctrl_panel.SetListCtrl( self._tag_parents )
+            self._listctrl_panel.SetListCtrl( self._tag_parents )
             
             self._tag_parents.Sort()
             
@@ -2910,14 +2910,16 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ):
             menu_items.append( ( 'normal', 'from .txt file', 'Load parents from a .txt file.', HydrusData.Call( self._ImportFromTXT, False ) ) )
             menu_items.append( ( 'normal', 'from .txt file (only add pairs--no deletions)', 'Load parents from a .txt file.', HydrusData.Call( self._ImportFromTXT, True ) ) )
             
-            listctrl_panel.AddMenuButton( 'import', menu_items )
+            self._listctrl_panel.AddMenuButton( 'import', menu_items )
             
             menu_items = []
             
             menu_items.append( ( 'normal', 'to clipboard', 'Save selected parents to your clipboard.', self._ExportToClipboard ) )
             menu_items.append( ( 'normal', 'to .txt file', 'Save selected parents to a .txt file.', self._ExportToTXT ) )
             
-            listctrl_panel.AddMenuButton( 'export', menu_items, enabled_only_on_selection = True )
+            self._listctrl_panel.AddMenuButton( 'export', menu_items, enabled_only_on_selection = True )
+            
+            self._listctrl_panel.setEnabled( False )
             
             self._children = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self, self._service_key, ClientTags.TAG_DISPLAY_ACTUAL )
             self._parents = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self, self._service_key, ClientTags.TAG_DISPLAY_ACTUAL )
@@ -2972,7 +2974,7 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ):
             QP.AddToLayout( vbox, self._sync_status_st, CC.FLAGS_EXPAND_PERPENDICULAR )
             QP.AddToLayout( vbox, self._count_st, CC.FLAGS_EXPAND_PERPENDICULAR )
             QP.AddToLayout( vbox, ClientGUICommon.WrapInText(self._show_all,self,'show all pairs'), CC.FLAGS_EXPAND_PERPENDICULAR )
-            QP.AddToLayout( vbox, listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
+            QP.AddToLayout( vbox, self._listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
             QP.AddToLayout( vbox, self._add, CC.FLAGS_ON_RIGHT )
             QP.AddToLayout( vbox, tags_box, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
             QP.AddToLayout( vbox, input_box, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
@@ -3314,8 +3316,6 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ):
                 
                 QW.QMessageBox.information( self, 'Information', 'Uneven number of tags in clipboard!' )
                 
-                return
-                
             
             pairs = []
             
@@ -3704,6 +3704,7 @@ class ManageTagParents( ClientGUIScrolledPanels.ManagePanel ):
                 
                 self._count_st.setText( 'Starting with '+HydrusData.ToHumanInt(len(original_statuses_to_pairs[HC.CONTENT_STATUS_CURRENT]))+' pairs.' )
                 
+                self._listctrl_panel.setEnabled( True )
                 self._child_input.setEnabled( True )
                 self._parent_input.setEnabled( True )
                 
@@ -3858,11 +3859,11 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ):
             
             self._show_all = QW.QCheckBox( self )
             
-            listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
+            self._listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
             
-            self._tag_siblings = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, CGLC.COLUMN_LIST_TAG_SIBLINGS.ID, 8, self._ConvertPairToListCtrlTuples, delete_key_callback = self._ListCtrlActivated, activation_callback = self._ListCtrlActivated )
+            self._tag_siblings = ClientGUIListCtrl.BetterListCtrl( self._listctrl_panel, CGLC.COLUMN_LIST_TAG_SIBLINGS.ID, 8, self._ConvertPairToListCtrlTuples, delete_key_callback = self._ListCtrlActivated, activation_callback = self._ListCtrlActivated )
             
-            listctrl_panel.SetListCtrl( self._tag_siblings )
+            self._listctrl_panel.SetListCtrl( self._tag_siblings )
             
             self._tag_siblings.Sort()
             
@@ -3873,14 +3874,16 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ):
             menu_items.append( ( 'normal', 'from .txt file', 'Load siblings from a .txt file.', HydrusData.Call( self._ImportFromTXT, False ) ) )
             menu_items.append( ( 'normal', 'from .txt file (only add pairs--no deletions)', 'Load siblings from a .txt file.', HydrusData.Call( self._ImportFromTXT, True ) ) )
             
-            listctrl_panel.AddMenuButton( 'import', menu_items )
+            self._listctrl_panel.AddMenuButton( 'import', menu_items )
             
             menu_items = []
             
             menu_items.append( ( 'normal', 'to clipboard', 'Save selected siblings to your clipboard.', self._ExportToClipboard ) )
             menu_items.append( ( 'normal', 'to .txt file', 'Save selected siblings to a .txt file.', self._ExportToTXT ) )
             
-            listctrl_panel.AddMenuButton( 'export', menu_items, enabled_only_on_selection = True )
+            self._listctrl_panel.AddMenuButton( 'export', menu_items, enabled_only_on_selection = True )
+            
+            self._listctrl_panel.setEnabled( False )
             
             self._old_siblings = ClientGUIListBoxes.ListBoxTagsStringsAddRemove( self, self._service_key, ClientTags.TAG_DISPLAY_ACTUAL )
             self._new_sibling = ClientGUICommon.BetterStaticText( self )
@@ -3934,7 +3937,7 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ):
             QP.AddToLayout( vbox, self._sync_status_st, CC.FLAGS_EXPAND_PERPENDICULAR )
             QP.AddToLayout( vbox, self._count_st, CC.FLAGS_EXPAND_PERPENDICULAR )
             QP.AddToLayout( vbox, ClientGUICommon.WrapInText(self._show_all,self,'show all pairs'), CC.FLAGS_EXPAND_PERPENDICULAR )
-            QP.AddToLayout( vbox, listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
+            QP.AddToLayout( vbox, self._listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
             QP.AddToLayout( vbox, self._add, CC.FLAGS_ON_RIGHT )
             QP.AddToLayout( vbox, text_box, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
             QP.AddToLayout( vbox, input_box, CC.FLAGS_EXPAND_SIZER_PERPENDICULAR )
@@ -4733,6 +4736,8 @@ class ManageTagSiblings( ClientGUIScrolledPanels.ManagePanel ):
                 
                 self._count_st.setText( 'Starting with '+HydrusData.ToHumanInt(len(original_statuses_to_pairs[HC.CONTENT_STATUS_CURRENT]))+' pairs.' )
                 
+                self._listctrl_panel.setEnabled( True )
+                
                 self._old_input.setEnabled( True )
                 self._new_input.setEnabled( True )
                 
diff --git a/hydrus/client/gui/QtPorting.py b/hydrus/client/gui/QtPorting.py
index 403a7a28..d0fc9d27 100644
--- a/hydrus/client/gui/QtPorting.py
+++ b/hydrus/client/gui/QtPorting.py
@@ -1246,11 +1246,11 @@ class CallAfterEventCatcher( QC.QObject ):
         
         if event.type() == CallAfterEventType and isinstance( event, CallAfterEvent ):
             
-            if HG.callto_profile_mode:
+            if HG.profile_mode:
                 
                 summary = 'Profiling CallAfter Event: {}'.format( event._fn )
                 
-                HydrusData.Profile( summary, 'event.Execute()', globals(), locals(), min_duration_ms = 3, show_summary = True )
+                HydrusData.Profile( summary, 'event.Execute()', globals(), locals(), min_duration_ms = HG.callto_profile_min_job_time_ms )
                 
             else:
                 
diff --git a/hydrus/client/gui/canvas/ClientGUICanvas.py b/hydrus/client/gui/canvas/ClientGUICanvas.py
index 68cec7bc..c0147473 100644
--- a/hydrus/client/gui/canvas/ClientGUICanvas.py
+++ b/hydrus/client/gui/canvas/ClientGUICanvas.py
@@ -1819,12 +1819,21 @@ class CanvasPanel( Canvas ):
         self.ShowMenu()
         
     
+    def ClearMedia( self ):
+        
+        self._hidden_page_current_media = None
+        
+        Canvas.ClearMedia( self )
+        
+    
     def PageHidden( self ):
         
-        self._hidden_page_current_media = self._current_media
+        hidden_page_current_media = self._current_media
         
         self.ClearMedia()
         
+        self._hidden_page_current_media = hidden_page_current_media
+        
     
     def PageShown( self ):
         
diff --git a/hydrus/client/gui/lists/ClientGUIListCtrl.py b/hydrus/client/gui/lists/ClientGUIListCtrl.py
index a57df8c1..62300f8f 100644
--- a/hydrus/client/gui/lists/ClientGUIListCtrl.py
+++ b/hydrus/client/gui/lists/ClientGUIListCtrl.py
@@ -242,6 +242,8 @@ class BetterListCtrl( QW.QTreeWidget ):
         
         LAST_COLUMN_SNAP_DISTANCE_CHARS = 5
         
+        total_fixed_columns_width = 0
+        
         for visual_index in range( num_columns ):
             
             logical_index = header.logicalIndex( visual_index )
@@ -252,18 +254,22 @@ class BetterListCtrl( QW.QTreeWidget ):
             
             if visual_index == last_column_index:
                 
-                if self.verticalScrollBar().isVisible():
-                    
-                    width_pixels += max( 0, min( self.verticalScrollBar().width(), 20 ) )
-                    
+                # testing if scrollbar is visible is unreliable, since we don't know if it is laid out correct yet (we could be doing that now!)
+                # so let's just hack it
+                
+                width_pixels = self.width() - ( self.frameWidth() * 2 ) - total_fixed_columns_width
+                
+            else:
+                
+                total_fixed_columns_width += width_pixels
                 
             
             width_chars = ClientGUIFunctions.ConvertPixelsToTextWidth( main_tlw, width_pixels )
             
             if visual_index == last_column_index:
                 
-                # here's the snap magic
-                width_chars = round( width_chars // LAST_COLUMN_SNAP_DISTANCE_CHARS ) * LAST_COLUMN_SNAP_DISTANCE_CHARS
+                # here's the snap magic. final width_chars is always a multiple of 5
+                width_chars = round( width_chars / LAST_COLUMN_SNAP_DISTANCE_CHARS ) * LAST_COLUMN_SNAP_DISTANCE_CHARS
                 
             
             columns.append( ( column_type, width_chars, shown ) )
@@ -751,6 +757,8 @@ class BetterListCtrl( QW.QTreeWidget ):
         
         width = 0
         
+        width += self.frameWidth() * 2
+        
         # all but last column
         
         for i in range( self.columnCount() - 1 ):
@@ -772,6 +780,10 @@ class BetterListCtrl( QW.QTreeWidget ):
             
             width += self.columnWidth( self.columnCount() - 1 )
             
+            # this is a hack to stop the thing suddenly growing to screen width in a weird resize loop
+            # I couldn't reproduce this error, so I assume it is a QSS or whatever font/style/scrollbar on some systems that caused inaccurate columnWidth result
+            width = min( width, self.width() )
+            
         else:
             
             last_column_chars = self._original_column_list_status.GetColumnWidth( last_column_type )
@@ -783,8 +795,6 @@ class BetterListCtrl( QW.QTreeWidget ):
         
         #
         
-        width += self.frameWidth() * 2
-        
         if self._forced_height_num_chars is None:
             
             num_rows = self._initial_height_num_chars
diff --git a/hydrus/client/gui/pages/ClientGUIManagement.py b/hydrus/client/gui/pages/ClientGUIManagement.py
index 0a759882..0e5ca9d5 100644
--- a/hydrus/client/gui/pages/ClientGUIManagement.py
+++ b/hydrus/client/gui/pages/ClientGUIManagement.py
@@ -2136,9 +2136,18 @@ class ManagementPanelImporterMultipleGallery( ManagementPanelImporter ):
     
     def _RetryIgnored( self ):
         
+        try:
+            
+            ignored_regex = ClientGUIFileSeedCache.GetRetryIgnoredParam( self )
+            
+        except HydrusExceptions.CancelledException:
+            
+            return
+            
+        
         for gallery_import in self._gallery_importers_listctrl.GetData( only_selected = True ):
             
-            gallery_import.RetryIgnored()
+            gallery_import.RetryIgnored( ignored_regex = ignored_regex )
             
         
     
@@ -2967,9 +2976,18 @@ class ManagementPanelImporterMultipleWatcher( ManagementPanelImporter ):
     
     def _RetryIgnored( self ):
         
+        try:
+            
+            ignored_regex = ClientGUIFileSeedCache.GetRetryIgnoredParam( self )
+            
+        except HydrusExceptions.CancelledException:
+            
+            return
+            
+        
         for watcher in self._watchers_listctrl.GetData( only_selected = True ):
             
-            watcher.RetryIgnored()
+            watcher.RetryIgnored( ignored_regex = ignored_regex )
             
         
     
diff --git a/hydrus/client/gui/pages/ClientGUIPages.py b/hydrus/client/gui/pages/ClientGUIPages.py
index e620ffa8..44710964 100644
--- a/hydrus/client/gui/pages/ClientGUIPages.py
+++ b/hydrus/client/gui/pages/ClientGUIPages.py
@@ -811,20 +811,6 @@ class Page( QW.QSplitter ):
             
         
     
-    def ShowHideSplit( self ):
-        
-        if QP.SplitterVisibleCount( self ) > 1:
-            
-            QP.Unsplit( self, self._search_preview_split )
-            
-            self._media_panel.SetFocusedMedia( None )
-            
-        else:
-            
-            self.SetSplitterPositions()
-            
-        
-    
     def SetMediaFocus( self ):
         
         self._media_panel.setFocus( QC.Qt.OtherFocusReason )
@@ -873,6 +859,20 @@ class Page( QW.QSplitter ):
             
         
     
+    def ShowHideSplit( self ):
+        
+        if QP.SplitterVisibleCount( self ) > 1:
+            
+            QP.Unsplit( self, self._search_preview_split )
+            
+            self._media_panel.SetFocusedMedia( None )
+            
+        else:
+            
+            self.SetSplitterPositions()
+            
+        
+    
     def _StartInitialMediaResultsLoad( self ):
         
         def qt_code_status( status ):
diff --git a/hydrus/client/importing/ClientImportFileSeeds.py b/hydrus/client/importing/ClientImportFileSeeds.py
index 00dd95a9..bac5b4f4 100644
--- a/hydrus/client/importing/ClientImportFileSeeds.py
+++ b/hydrus/client/importing/ClientImportFileSeeds.py
@@ -2,6 +2,7 @@ import collections
 import itertools
 import os
 import random
+import re
 import threading
 import time
 import traceback
@@ -91,11 +92,14 @@ class FileSeed( HydrusSerialisable.SerialisableBase ):
     
     def _CheckTagsVeto( self, tags, tag_import_options: TagImportOptions.TagImportOptions ):
         
-        tags_to_siblings = HG.client_controller.Read( 'tag_siblings_lookup', CC.COMBINED_TAG_SERVICE_KEY, tags )
-        
-        all_chain_tags = set( itertools.chain.from_iterable( tags_to_siblings.values() ) )
-        
-        tag_import_options.CheckTagsVeto( tags, all_chain_tags )
+        if len( tags ) > 0:
+            
+            tags_to_siblings = HG.client_controller.Read( 'tag_siblings_lookup', CC.COMBINED_TAG_SERVICE_KEY, tags )
+            
+            all_chain_tags = set( itertools.chain.from_iterable( tags_to_siblings.values() ) )
+            
+            tag_import_options.CheckTagsVeto( tags, all_chain_tags )
+            
         
     
     def _GetSerialisableInfo( self ):
@@ -2330,7 +2334,7 @@ class FileSeedCache( HydrusSerialisable.SerialisableBase ):
         self.NotifyFileSeedsUpdated( failed_file_seeds )
         
     
-    def RetryIgnored( self ):
+    def RetryIgnored( self, ignored_regex = None ):
         
         with self._lock:
             
@@ -2338,6 +2342,14 @@ class FileSeedCache( HydrusSerialisable.SerialisableBase ):
             
             for file_seed in ignored_file_seeds:
                 
+                if ignored_regex is not None:
+                    
+                    if re.search( ignored_regex, file_seed.note ) is None:
+                        
+                        continue
+                        
+                    
+                
                 file_seed.SetStatus( CC.STATUS_UNKNOWN )
                 
             
diff --git a/hydrus/client/importing/ClientImportGallery.py b/hydrus/client/importing/ClientImportGallery.py
index 26d1ddd0..ca281a56 100644
--- a/hydrus/client/importing/ClientImportGallery.py
+++ b/hydrus/client/importing/ClientImportGallery.py
@@ -734,11 +734,11 @@ class GalleryImport( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def RetryIgnored( self ):
+    def RetryIgnored( self, ignored_regex = None ):
         
         with self._lock:
             
-            self._file_seed_cache.RetryIgnored()
+            self._file_seed_cache.RetryIgnored( ignored_regex = ignored_regex )
             
         
     
diff --git a/hydrus/client/importing/ClientImportSubscriptions.py b/hydrus/client/importing/ClientImportSubscriptions.py
index f1803670..cb4eb7bb 100644
--- a/hydrus/client/importing/ClientImportSubscriptions.py
+++ b/hydrus/client/importing/ClientImportSubscriptions.py
@@ -1349,11 +1349,11 @@ class Subscription( HydrusSerialisable.SerialisableBaseNamed ):
             
         
     
-    def RetryIgnored( self ):
+    def RetryIgnored( self, ignored_regex = None ):
         
         for query_header in self._query_headers:
             
-            query_header.RetryIgnored()
+            query_header.RetryIgnored( ignored_regex = ignored_regex )
             
         
     
diff --git a/hydrus/client/importing/ClientImportWatchers.py b/hydrus/client/importing/ClientImportWatchers.py
index 11f622d1..6d0b4938 100644
--- a/hydrus/client/importing/ClientImportWatchers.py
+++ b/hydrus/client/importing/ClientImportWatchers.py
@@ -1443,11 +1443,11 @@ class WatcherImport( HydrusSerialisable.SerialisableBase ):
             
         
     
-    def RetryIgnored( self ):
+    def RetryIgnored( self, ignored_regex = None ):
         
         with self._lock:
             
-            self._file_seed_cache.RetryIgnored()
+            self._file_seed_cache.RetryIgnored( ignored_regex = ignored_regex )
             
         
     
diff --git a/hydrus/client/media/ClientMedia.py b/hydrus/client/media/ClientMedia.py
index 65a78984..646dbf23 100644
--- a/hydrus/client/media/ClientMedia.py
+++ b/hydrus/client/media/ClientMedia.py
@@ -1,4 +1,5 @@
 import collections
+import itertools
 import random
 import typing
 
@@ -74,25 +75,6 @@ def GetDuplicateComparisonScore( shown_media, comparison_media ):
     
     return total_score
     
-NICE_RESOLUTIONS = {}
-
-NICE_RESOLUTIONS[ ( 640, 480 ) ] = '480p'
-NICE_RESOLUTIONS[ ( 1280, 720 ) ] = '720p'
-NICE_RESOLUTIONS[ ( 1920, 1080 ) ] = '1080p'
-NICE_RESOLUTIONS[ ( 3840, 2060 ) ] = '4k'
-
-NICE_RATIOS = {}
-
-NICE_RATIOS[ 1 ] = '1:1'
-NICE_RATIOS[ 4 / 3 ] = '4:3'
-NICE_RATIOS[ 5 / 4 ] = '5:4'
-NICE_RATIOS[ 16 / 9 ] = '16:9'
-NICE_RATIOS[ 21 / 9 ] = '21:9'
-NICE_RATIOS[ 47 / 20 ] = '2.35:1'
-NICE_RATIOS[ 9 / 16 ] = '9:16'
-NICE_RATIOS[ 2 / 3 ] = '2:3'
-NICE_RATIOS[ 4 / 5 ] = '4:5'
-
 def GetDuplicateComparisonStatements( shown_media, comparison_media ):
     
     new_options = HG.client_controller.new_options
@@ -252,9 +234,9 @@ def GetDuplicateComparisonStatements( shown_media, comparison_media ):
             score = -duplicate_comparison_score_higher_resolution
             
         
-        if s_res in NICE_RESOLUTIONS:
+        if s_res in HC.NICE_RESOLUTIONS:
             
-            s_string = NICE_RESOLUTIONS[ s_res ]
+            s_string = HC.NICE_RESOLUTIONS[ s_res ]
             
         else:
             
@@ -266,9 +248,9 @@ def GetDuplicateComparisonStatements( shown_media, comparison_media ):
                 
             
         
-        if c_res in NICE_RESOLUTIONS:
+        if c_res in HC.NICE_RESOLUTIONS:
             
-            c_string = NICE_RESOLUTIONS[ c_res ]
+            c_string = HC.NICE_RESOLUTIONS[ c_res ]
             
         else:
             
@@ -289,14 +271,14 @@ def GetDuplicateComparisonStatements( shown_media, comparison_media ):
         s_ratio = s_w / s_h
         c_ratio = c_w / c_h
         
-        s_nice = s_ratio in NICE_RATIOS
-        c_nice = c_ratio in NICE_RATIOS
+        s_nice = s_ratio in HC.NICE_RATIOS
+        c_nice = c_ratio in HC.NICE_RATIOS
         
         if s_nice or c_nice:
             
             if s_nice:
                 
-                s_string = NICE_RATIOS[ s_ratio ]
+                s_string = HC.NICE_RATIOS[ s_ratio ]
                 
             else:
                 
@@ -305,7 +287,7 @@ def GetDuplicateComparisonStatements( shown_media, comparison_media ):
             
             if c_nice:
                 
-                c_string = NICE_RATIOS[ c_ratio ]
+                c_string = HC.NICE_RATIOS[ c_ratio ]
                 
             else:
                 
@@ -413,7 +395,7 @@ def GetDuplicateComparisonStatements( shown_media, comparison_media ):
             score = 0
             
         
-        statement = '{} {} {}'.format( ClientData.TimestampToPrettyTimeDelta( s_ts ), operator, ClientData.TimestampToPrettyTimeDelta( c_ts ) )
+        statement = '{}, {} {}'.format( ClientData.TimestampToPrettyTimeDelta( s_ts, history_suffix = ' old' ), operator, ClientData.TimestampToPrettyTimeDelta( c_ts, history_suffix = ' old' ) )
         
         statements_and_scores[ 'time_imported' ] = ( statement, score )
         
@@ -814,6 +796,7 @@ class MediaList( object ):
         self._file_service_key = file_service_key
         
         self._hashes = set()
+        self._hashes_ordered = []
         
         self._hashes_to_singleton_media = {}
         self._hashes_to_collected_media = {}
@@ -985,29 +968,34 @@ class MediaList( object ):
     def _RecalcHashes( self ):
         
         self._hashes = set()
+        self._hashes_ordered = []
         
         self._hashes_to_singleton_media = {}
         self._hashes_to_collected_media = {}
         
-        for media in self._collected_media:
+        for m in self._sorted_media:
             
-            hashes = media.GetHashes()
-            
-            self._hashes.update( hashes )
-            
-            for hash in hashes:
+            if isinstance( m, MediaCollection ):
                 
-                self._hashes_to_collected_media[ hash ] = media
+                hashes = m.GetHashes( ordered = True )
+                
+                self._hashes.update( hashes )
+                self._hashes_ordered.extend( hashes )
+                
+                for hash in hashes:
+                    
+                    self._hashes_to_collected_media[ hash ] = m
+                    
+                
+            else:
+                
+                hash = m.GetHash()
+                
+                self._hashes.add( hash )
+                self._hashes_ordered.append( hash )
+                
+                self._hashes_to_singleton_media[ hash ] = m
                 
-            
-        
-        for media in self._singleton_media:
-            
-            hash = media.GetHash()
-            
-            self._hashes.add( hash )
-            
-            self._hashes_to_singleton_media[ hash ] = media
             
         
     
@@ -1068,6 +1056,7 @@ class MediaList( object ):
             addable_media.append( media )
             
             self._hashes.add( hash )
+            self._hashes_ordered.append( hash )
             
             self._hashes_to_singleton_media[ hash ] = media
             
@@ -1490,9 +1479,16 @@ class MediaList( object ):
     
     def GetHashes( self, has_location = None, discriminant = None, not_uploaded_to = None, ordered = False ):
         
-        if has_location is None and discriminant is None and not_uploaded_to is None and not ordered:
+        if has_location is None and discriminant is None and not_uploaded_to is None:
             
-            return self._hashes
+            if ordered:
+                
+                return self._hashes_ordered
+                
+            else:
+                
+                return self._hashes
+                
             
         else:
             
@@ -1563,7 +1559,7 @@ class MediaList( object ):
         return self._sorted_media
         
     
-    def HasAnyOfTheseHashes( self, hashes ):
+    def HasAnyOfTheseHashes( self, hashes: set ):
         
         return not hashes.isdisjoint( self._hashes )
         
@@ -2431,7 +2427,10 @@ class MediaSingleton( Media ):
         
         info_string = HydrusData.ToHumanBytes( size ) + ' ' + HC.mime_string_lookup[ mime ]
         
-        if width is not None and height is not None: info_string += ' (' + HydrusData.ToHumanInt( width ) + 'x' + HydrusData.ToHumanInt( height ) + ')'
+        if width is not None and height is not None:
+            
+            info_string += ' ({})'.format( HydrusData.ConvertResolutionToPrettyString( ( width, height ) ) )
+            
         
         if duration is not None:
             
diff --git a/hydrus/core/HydrusConstants.py b/hydrus/core/HydrusConstants.py
index 3334a49f..ae099eac 100644
--- a/hydrus/core/HydrusConstants.py
+++ b/hydrus/core/HydrusConstants.py
@@ -81,7 +81,7 @@ options = {}
 # Misc
 
 NETWORK_VERSION = 20
-SOFTWARE_VERSION = 446
+SOFTWARE_VERSION = 447
 CLIENT_API_VERSION = 17
 
 SERVER_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@@ -286,6 +286,28 @@ MAINTENANCE_SHUTDOWN = 1
 MAINTENANCE_FORCED = 2
 MAINTENANCE_ACTIVE = 3
 
+NICE_RESOLUTIONS = {}
+
+NICE_RESOLUTIONS[ ( 640, 480 ) ] = '480p'
+NICE_RESOLUTIONS[ ( 1280, 720 ) ] = '720p'
+NICE_RESOLUTIONS[ ( 1920, 1080 ) ] = '1080p'
+NICE_RESOLUTIONS[ ( 3840, 2060 ) ] = '4k'
+NICE_RESOLUTIONS[ ( 720, 1280 ) ] = 'vertical 720p'
+NICE_RESOLUTIONS[ ( 1080, 1920 ) ] = 'vertical 1080p'
+NICE_RESOLUTIONS[ ( 2060, 3840 ) ] = 'vertical 4k'
+
+NICE_RATIOS = {}
+
+NICE_RATIOS[ 1 ] = '1:1'
+NICE_RATIOS[ 4 / 3 ] = '4:3'
+NICE_RATIOS[ 5 / 4 ] = '5:4'
+NICE_RATIOS[ 16 / 9 ] = '16:9'
+NICE_RATIOS[ 21 / 9 ] = '21:9'
+NICE_RATIOS[ 47 / 20 ] = '2.35:1'
+NICE_RATIOS[ 9 / 16 ] = '9:16'
+NICE_RATIOS[ 2 / 3 ] = '2:3'
+NICE_RATIOS[ 4 / 5 ] = '4:5'
+
 GET_DATA = 0
 POST_DATA = 1
 POST_PETITIONS = 2
@@ -407,6 +429,7 @@ REAL_TAG_SERVICES = ( LOCAL_TAG, TAG_REPOSITORY )
 ADDREMOVABLE_SERVICES = ( LOCAL_TAG, LOCAL_RATING_LIKE, LOCAL_RATING_NUMERICAL, FILE_REPOSITORY, TAG_REPOSITORY, SERVER_ADMIN, IPFS )
 MUST_HAVE_AT_LEAST_ONE_SERVICES = ( LOCAL_TAG, )
 NONEDITABLE_SERVICES = ( LOCAL_FILE_DOMAIN, LOCAL_FILE_TRASH_DOMAIN, COMBINED_FILE, COMBINED_TAG, COMBINED_LOCAL_FILE )
+SPECIFIC_FILE_SERVICES = ( LOCAL_FILE_DOMAIN, LOCAL_FILE_TRASH_DOMAIN, COMBINED_LOCAL_FILE, FILE_REPOSITORY, IPFS )
 AUTOCOMPLETE_CACHE_SPECIFIC_FILE_SERVICES = ( LOCAL_FILE_DOMAIN, LOCAL_FILE_TRASH_DOMAIN, COMBINED_LOCAL_FILE, FILE_REPOSITORY )
 TAG_CACHE_SPECIFIC_FILE_SERVICES = ( COMBINED_LOCAL_FILE, FILE_REPOSITORY )
 ALL_SERVICES = REMOTE_SERVICES + LOCAL_SERVICES + ( COMBINED_FILE, COMBINED_TAG )
diff --git a/hydrus/core/HydrusController.py b/hydrus/core/HydrusController.py
index bc283857..81ff9082 100644
--- a/hydrus/core/HydrusController.py
+++ b/hydrus/core/HydrusController.py
@@ -628,9 +628,9 @@ class HydrusController( object ):
     
     def PrintProfile( self, summary, profile_text ):
         
-        boot_pretty_timestamp = time.strftime( '%Y-%m-%d %H-%M-%S', time.localtime( self.GetBootTime() ) )
+        pretty_timestamp = time.strftime( '%Y-%m-%d %H-%M-%S', time.localtime( HG.profile_start_time ) )
         
-        profile_log_filename = self._name + ' profile - ' + boot_pretty_timestamp + '.log'
+        profile_log_filename = '{} profile - {}.log'.format( self._name, pretty_timestamp )
         
         profile_log_path = os.path.join( self.db_dir, profile_log_filename )
         
diff --git a/hydrus/core/HydrusDB.py b/hydrus/core/HydrusDB.py
index d0518eb6..bd48af85 100644
--- a/hydrus/core/HydrusDB.py
+++ b/hydrus/core/HydrusDB.py
@@ -731,13 +731,13 @@ class HydrusDB( object ):
                 result = self._Write( action, *args, **kwargs )
                 
             
-            self._cursor_transaction_wrapper.Save()
-            
             if job.IsSynchronous():
                 
                 job.PutResult( result )
                 
             
+            self._cursor_transaction_wrapper.Save()
+            
             if self._cursor_transaction_wrapper.TimeToCommit():
                 
                 self._current_status = 'db committing'
@@ -997,16 +997,16 @@ class HydrusDB( object ):
                     
                     if HG.db_report_mode:
                         
-                        summary = 'Running ' + job.ToString()
+                        summary = 'Running db job: ' + job.ToString()
                         
                         HydrusData.ShowText( summary )
                         
                     
-                    if HG.db_profile_mode:
+                    if HG.profile_mode:
                         
-                        summary = 'Profiling ' + job.ToString()
+                        summary = 'Profiling db job: ' + job.ToString()
                         
-                        HydrusData.Profile( summary, 'self._ProcessJob( job )', globals(), locals(), show_summary = True )
+                        HydrusData.Profile( summary, 'self._ProcessJob( job )', globals(), locals(), min_duration_ms = HG.db_profile_min_job_time_ms )
                         
                     else:
                         
diff --git a/hydrus/core/HydrusData.py b/hydrus/core/HydrusData.py
index 3f52da04..58a4f20c 100644
--- a/hydrus/core/HydrusData.py
+++ b/hydrus/core/HydrusData.py
@@ -244,9 +244,14 @@ def ConvertPrettyStringsToUglyNamespaces( pretty_strings ):
     
 def ConvertResolutionToPrettyString( resolution ):
     
+    if resolution in HC.NICE_RESOLUTIONS:
+        
+        return HC.NICE_RESOLUTIONS[ resolution ]
+        
+    
     ( width, height ) = resolution
     
-    return ToHumanInt( width ) + 'x' + ToHumanInt( height )
+    return '{}x{}'.format( ToHumanInt( width ), ToHumanInt( height ) )
     
 def ConvertStatusToPrefix( status ):
     
@@ -429,7 +434,7 @@ def ConvertTimestampToPrettyTime( timestamp, in_utc = False, include_24h_time =
         return 'unparseable time {}'.format( timestamp )
         
     
-def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'now', just_now_threshold = 3, show_seconds = True, no_prefix = False ):
+def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'now', just_now_threshold = 3, history_suffix = ' ago', show_seconds = True, no_prefix = False ):
     
     if timestamp is None:
         
@@ -454,7 +459,7 @@ def TimestampToPrettyTimeDelta( timestamp, just_now_string = 'now', just_now_thr
         
         if TimeHasPassed( timestamp ):
             
-            return time_delta_string + ' ago'
+            return '{}{}'.format( time_delta_string, history_suffix )
             
         else:
             
@@ -1210,20 +1215,31 @@ def Profile( summary, code, g, l, min_duration_ms = 20, show_summary = False ):
         
         details = output.read()
         
+        with HG.profile_counter_lock:
+            
+            HG.profile_slow_count += 1
+            
+        
         if show_summary:
             
             ShowText( summary )
             
         
+        HG.controller.PrintProfile( summary, details )
+        
     else:
         
-        summary += ' - It took ' + TimeDeltaToPrettyTimeDelta( time_took ) + '.'
+        with HG.profile_counter_lock:
+            
+            HG.profile_fast_count += 1
+            
         
-        details = ''
+        if show_summary:
+            
+            HG.controller.PrintProfile( summary, details )
+            
         
     
-    HG.controller.PrintProfile( summary, details )
-    
 def PullNFromIterator( iterator, n ):
     
     chunk = []
diff --git a/hydrus/core/HydrusGlobals.py b/hydrus/core/HydrusGlobals.py
index e500ebcb..b2beb5cf 100644
--- a/hydrus/core/HydrusGlobals.py
+++ b/hydrus/core/HydrusGlobals.py
@@ -1,3 +1,4 @@
+import collections
 import threading
 
 controller = None
@@ -26,12 +27,23 @@ db_synchronous = 2
 import_folders_running = False
 export_folders_running = False
 
+profile_mode = False
+
+db_profile_min_job_time_ms = 16
+callto_profile_min_job_time_ms = 5
+server_profile_min_job_time_ms = 5
+menu_profile_min_job_time_ms = 16
+pubsub_profile_min_job_time_ms = 5
+ui_timer_profile_min_job_time_ms = 5
+
+profile_start_time = 0
+profile_slow_count = 0
+profile_fast_count = 0
+profile_counter_lock = threading.Lock()
+
 db_ui_hang_relief_mode = False
 callto_report_mode = False
-server_profile_mode = False
 db_report_mode = False
-db_profile_mode = False
-callto_profile_mode = False
 file_report_mode = False
 media_load_report_mode = False
 gui_report_mode = False
@@ -42,11 +54,8 @@ subscription_report_mode = False
 hover_window_report_mode = False
 file_import_report_mode = False
 phash_generation_report_mode = False
-menu_profile_mode = False
 network_report_mode = False
 pubsub_report_mode = False
-pubsub_profile_mode = False
-ui_timer_profile_mode = False
 daemon_report_mode = False
 force_idle_mode = False
 no_page_limit_mode = False
diff --git a/hydrus/core/HydrusPubSub.py b/hydrus/core/HydrusPubSub.py
index f4f687dd..72ec882c 100644
--- a/hydrus/core/HydrusPubSub.py
+++ b/hydrus/core/HydrusPubSub.py
@@ -113,19 +113,15 @@ class HydrusPubSub( object ):
                         HydrusData.ShowText( ( topic, args, kwargs, callable_tuples ) )
                         
                     
-                    if HG.pubsub_profile_mode and not_a_report:
+                    if HG.profile_mode and not_a_report:
                         
-                        summary = 'Profiling ' + HydrusData.ToHumanInt( len( callable_tuples ) ) + ' x ' + topic
-                        
-                        HydrusData.ShowText( summary )
-                        
-                        per_summary = 'Profiling ' + topic
+                        summary = 'Profiling pubsub: {}'.format( topic )
                         
                         for ( obj, callable ) in callable_tuples:
                             
                             try:
                                 
-                                HydrusData.Profile( per_summary, 'callable( *args, **kwargs )', globals(), locals() )
+                                HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = HG.pubsub_profile_min_job_time_ms )
                                 
                             except HydrusExceptions.ShutdownException:
                                 
diff --git a/hydrus/core/HydrusThreading.py b/hydrus/core/HydrusThreading.py
index ec30fcf7..eb572369 100644
--- a/hydrus/core/HydrusThreading.py
+++ b/hydrus/core/HydrusThreading.py
@@ -390,11 +390,11 @@ class THREADCallToThread( DAEMON ):
                     
                     self._callable = ( callable, args, kwargs )
                     
-                    if HG.callto_profile_mode:
+                    if HG.profile_mode:
                         
                         summary = 'Profiling CallTo Job: {}'.format( callable )
                         
-                        HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = 3, show_summary = True )
+                        HydrusData.Profile( summary, 'callable( *args, **kwargs )', globals(), locals(), min_duration_ms = HG.callto_profile_min_job_time_ms )
                         
                     else:
                         
diff --git a/hydrus/core/networking/HydrusServerResources.py b/hydrus/core/networking/HydrusServerResources.py
index 79dee53a..d90a815d 100644
--- a/hydrus/core/networking/HydrusServerResources.py
+++ b/hydrus/core/networking/HydrusServerResources.py
@@ -301,7 +301,7 @@ class HydrusResource( Resource ):
             return request
             
         
-        if HG.server_profile_mode:
+        if HG.profile_mode:
             
             d = deferToThread( self._profileJob, self._threadDoGETJob, request )
             
@@ -324,7 +324,7 @@ class HydrusResource( Resource ):
             return request
             
         
-        if HG.server_profile_mode:
+        if HG.profile_mode:
             
             d = deferToThread( self._profileJob, self._threadDoOPTIONSJob, request )
             
@@ -347,7 +347,7 @@ class HydrusResource( Resource ):
             return request
             
         
-        if HG.server_profile_mode:
+        if HG.profile_mode:
             
             d = deferToThread( self._profileJob, self._threadDoPOSTJob, request )
             
@@ -590,7 +590,7 @@ class HydrusResource( Resource ):
     
     def _profileJob( self, call, request: HydrusServerRequest.HydrusRequest ):
         
-        HydrusData.Profile( 'client api {}'.format( request.path ), 'request.result_lmao = call( request )', globals(), locals(), min_duration_ms = 3, show_summary = True )
+        HydrusData.Profile( 'Profiling client api: {}'.format( request.path ), 'request.result_lmao = call( request )', globals(), locals(), min_duration_ms = HG.server_profile_min_job_time_ms )
         
         return request.result_lmao