From 8a84453f892d598e6233706eda39b046d3fc3740 Mon Sep 17 00:00:00 2001
From: Hydrus <hydrus.admin@gmail.com>
Date: Wed, 25 Nov 2015 16:00:57 -0600
Subject: [PATCH] Version 183

---
 bin/upnpc.txt                     |  19 -
 client.pyw                        |   2 +
 help/changelog.html               |  34 ++
 include/ClientCaches.py           | 842 ++++++++++++++++++++++--------
 include/ClientController.py       |  53 +-
 include/ClientDB.py               | 101 +++-
 include/ClientData.py             | 299 ++---------
 include/ClientFiles.py            |  50 +-
 include/ClientGUI.py              |  26 +-
 include/ClientGUICanvas.py        |  41 +-
 include/ClientGUICommon.py        | 205 ++++----
 include/ClientGUIDialogsManage.py |   6 +-
 include/ClientGUIHoverFrames.py   |  22 +-
 include/ClientGUIMedia.py         |  13 +-
 include/ClientGUIMessages.py      |  16 +-
 include/ClientGUIPages.py         |  43 +-
 include/ClientRendering.py        |   9 +-
 include/ClientVideoHandling.py    |   2 +-
 include/HydrusConstants.py        |   4 +-
 include/HydrusDB.py               |   8 +-
 include/HydrusData.py             | 105 ++--
 include/HydrusFileHandling.py     |  80 ++-
 include/HydrusFlashHandling.py    |  25 +
 include/HydrusGlobals.py          |   1 +
 include/HydrusImageHandling.py    |   5 +-
 include/HydrusPaths.py            |  69 +++
 include/HydrusPubSub.py           |  21 +-
 include/HydrusServerResources.py  |  14 +-
 include/HydrusVideoHandling.py    |   1 +
 include/ServerDB.py               |  44 +-
 include/TestClientConstants.py    |   5 +-
 include/TestHydrusTags.py         |   4 +-
 server.py                         |   2 +
 test.py                           |  16 +-
 34 files changed, 1339 insertions(+), 848 deletions(-)
 delete mode 100644 bin/upnpc.txt

diff --git a/bin/upnpc.txt b/bin/upnpc.txt
deleted file mode 100644
index d0f65408..00000000
--- a/bin/upnpc.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-upnpc : miniupnpc library test client. (c) 2005-2013 Thomas Bernard
-Go to http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/
-for more information.
-List of UPNP devices found on the network :
- desc: http://192.168.0.1/root.sxml
- st: urn:schemas-upnp-org:device:InternetGatewayDevice:1
-
-Found valid IGD : http://192.168.0.1:4444/wipconn
-Local LAN ip address : 192.168.0.195
-Connection Type : IP_Routed
-Status : Connected, uptime=83462s, LastConnectionError : ERROR_NONE
-  Time started : Mon Aug 10 14:49:40 2015
-MaxBitRateDown : 1000000000 bps (1000.0 Mbps)   MaxBitRateUp 1000000000 bps (1000.0 Mbps)
-ExternalIPAddress = 98.213.207.109
- i protocol exPort->inAddr:inPort description remoteHost leaseTime
- 0 UDP 45871->192.168.0.103:45871 'Public Tags' '' 0
- 1 UDP 45872->192.168.0.103:45872 'File Repo' '' 0
- 2 UDP 45873->192.168.0.103:45873 'Message Depot' '' 0
-GetGenericPortMappingEntry() returned 713 (SpecifiedArrayIndexInvalid)
diff --git a/client.pyw b/client.pyw
index ded4a759..4257f6e3 100755
--- a/client.pyw
+++ b/client.pyw
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+
 # This program is free software. It comes without any warranty, to
 # the extent permitted by applicable law. You can redistribute it
 # and/or modify it under the terms of the Do What The Fuck You Want
diff --git a/help/changelog.html b/help/changelog.html
index c615ad24..c2b81be8 100755
--- a/help/changelog.html
+++ b/help/changelog.html
@@ -8,6 +8,40 @@
 		<div class="content">
 			<h3>changelog</h3>
 			<ul>
+				<li><h3>version 183</h3></li>
+				<ul>
+					<li>added swf thumbnail support--it works ok for most swfs!</li>
+					<li>thumbs for existing swf files will generate on db update for both client and server</li>
+					<li>the server will also generate thumbnails for video on update, which it seems it was not doing before</li>
+					<li>rewrote some hash comparisons in thumbnail downloading and thumbnail counting code to be a lot faster and memory efficient</li>
+					<li>fixed thumbnail count invalidation after service sync</li>
+					<li>in certain cases, fetching autocomplete tag results from the db will be slightly less laggy</li>
+					<li>if the autocomplete dropdown has cached or otherwise quickly fetchable results, it will refilter for current input as you type, ignoring the normal character delay (i.e. typing through autocomplete is far less laggy now)</li>
+					<li>fixed a couple of autocomplete tag-parsing bugs for tags with more than one colon character</li>
+					<li>fixed some key event selection bugs in the autocomplete taglist when there are no results in the list</li>
+					<li>if there are no results in the autocomplete taglist, the typically caret-moving key events up/down/home/end will move the caret (rather than being uselessly passed on to the empty taglist)</li>
+					<li>all profiling now goes through a single location</li>
+					<li>profiling now also prints extensive information on explicit code callers</li>
+					<li>fixed db profile mode printing display for the new logger</li>
+					<li>added new pubsub_profile_mode (AKA log-killer)</li>
+					<li>all menu popup display and explicit memory cleanup is done through a single location</li>
+					<li>hover windows will now not show whenever a menu is open</li>
+					<li>hover windows will now not hide whenever a menu is open (useful when trying to go 'copy tag' off the hover taglist's menu!)</li>
+					<li>a keyerror bug when force_idle_mode was hit right at program start is fixed</li>
+					<li>unexpected serverside client disconnections are now caught and further request processing is cancelled</li>
+					<li>middle clicking on an 'empty' system predicate in the a/c dropdown now will throw up the dialog to enter a value and then pass that filled in system pred onto the new search page</li>
+					<li>duplicate parents (which can occur with overlapping grandparents) are now collapsed</li>
+					<li>the upload pending popup message cancels properly after 'server busy' events</li>
+					<li>whenever the client's services are iterated at the view-level, it will now be in random order, to stop sync bottlenecks choking other services (and anything else like this that might have been occuring)</li>
+					<li>fixed a bug with double-clicking a tag selection on the the tag censorship taglist</li>
+					<li>fixed the too-fast frame timings on some videos opencv had a problem parsing</li>
+					<li>loading many video types will now be just a little faster</li>
+					<li>remote files will no longer present the right-click menu option to share->copy->file</li>
+					<li>hitting a shortcut for manage ratings from the media viewer canvas will no longer work if you have no ratings services</li>
+					<li>the base scripts (client.pyw, server.py and test.py) now have a shebang line to facilitate non-Windows use</li>
+					<li>laid groundwork for external client_files locations</li>
+					<li>plenty of misc cleaning and refactoring</li>
+				</ul>
 				<li><h3>version 182</h3></li>
 				<ul>
 					<li>all printing to the log should now be unicode safe</li>
diff --git a/include/ClientCaches.py b/include/ClientCaches.py
index d1cbb853..4725a24d 100644
--- a/include/ClientCaches.py
+++ b/include/ClientCaches.py
@@ -8,6 +8,7 @@ import HydrusFileHandling
 import HydrusImageHandling
 import HydrusPaths
 import HydrusSessions
+import itertools
 import os
 import random
 import Queue
@@ -24,10 +25,272 @@ import HydrusTags
 import itertools
 import ClientSearch
 
+# important thing here, and reason why it is recursive, is because we want to preserve the parent-grandparent interleaving
+def BuildServiceKeysToChildrenToParents( service_keys_to_simple_children_to_parents ):
+    
+    def AddParents( simple_children_to_parents, children_to_parents, child, parents ):
+        
+        for parent in parents:
+            
+            if parent not in children_to_parents[ child ]:
+                
+                children_to_parents[ child ].append( parent )
+                
+            
+            if parent in simple_children_to_parents:
+                
+                grandparents = simple_children_to_parents[ parent ]
+                
+                AddParents( simple_children_to_parents, children_to_parents, child, grandparents )
+                
+            
+        
+    
+    service_keys_to_children_to_parents = collections.defaultdict( HydrusData.default_dict_list )
+    
+    for ( service_key, simple_children_to_parents ) in service_keys_to_simple_children_to_parents.items():
+        
+        children_to_parents = service_keys_to_children_to_parents[ service_key ]
+        
+        for ( child, parents ) in simple_children_to_parents.items():
+            
+            AddParents( simple_children_to_parents, children_to_parents, child, parents )
+            
+        
+    
+    return service_keys_to_children_to_parents
+    
+def BuildServiceKeysToSimpleChildrenToParents( service_keys_to_pairs_flat ):
+    
+    service_keys_to_simple_children_to_parents = collections.defaultdict( HydrusData.default_dict_set )
+    
+    for ( service_key, pairs ) in service_keys_to_pairs_flat.items():
+        
+        service_keys_to_simple_children_to_parents[ service_key ] = BuildSimpleChildrenToParents( pairs )
+        
+    
+    return service_keys_to_simple_children_to_parents
+    
+def BuildSimpleChildrenToParents( pairs ):
+    
+    simple_children_to_parents = HydrusData.default_dict_set()
+    
+    for ( child, parent ) in pairs:
+        
+        if child == parent: continue
+        
+        if LoopInSimpleChildrenToParents( simple_children_to_parents, child, parent ): continue
+        
+        simple_children_to_parents[ child ].add( parent )
+        
+    
+    return simple_children_to_parents
+    
+def CollapseTagSiblingChains( processed_siblings ):
+    
+    # now to collapse chains
+    # A -> B and B -> C goes to A -> C and B -> C
+    
+    siblings = {}
+    
+    for ( old_tag, new_tag ) in processed_siblings.items():
+        
+        # adding A -> B
+        
+        if new_tag in siblings:
+            
+            # B -> F already calculated and added, so add A -> F
+            
+            siblings[ old_tag ] = siblings[ new_tag ]
+            
+        else:
+            
+            while new_tag in processed_siblings: new_tag = processed_siblings[ new_tag ] # pursue endpoint F
+            
+            siblings[ old_tag ] = new_tag
+            
+        
+    
+    reverse_lookup = collections.defaultdict( list )
+    
+    for ( old_tag, new_tag ) in siblings.items():
+        
+        reverse_lookup[ new_tag ].append( old_tag )
+        
+    
+    return ( siblings, reverse_lookup )
+    
+def CombineTagSiblingPairs( service_keys_to_statuses_to_pairs ):
+    
+    # first combine the services
+    # if A map already exists, don't overwrite
+    # if A -> B forms a loop, don't write it
+    
+    processed_siblings = {}
+    current_deleted_pairs = set()
+    
+    for ( service_key, statuses_to_pairs ) in service_keys_to_statuses_to_pairs.items():
+        
+        pairs = statuses_to_pairs[ HC.CURRENT ].union( statuses_to_pairs[ HC.PENDING ] )
+        
+        for ( old, new ) in pairs:
+            
+            if old == new: continue
+            
+            if old not in processed_siblings:
+                
+                next_new = new
+                
+                we_have_a_loop = False
+                
+                while next_new in processed_siblings:
+                    
+                    next_new = processed_siblings[ next_new ]
+                    
+                    if next_new == old:
+                        
+                        we_have_a_loop = True
+                        
+                        break
+                        
+                    
+                
+                if not we_have_a_loop: processed_siblings[ old ] = new
+                
+            
+        
+    
+    return processed_siblings
+    
+def LoopInSimpleChildrenToParents( simple_children_to_parents, child, parent ):
+    
+    potential_loop_paths = { parent }
+    
+    while len( potential_loop_paths.intersection( simple_children_to_parents.keys() ) ) > 0:
+        
+        new_potential_loop_paths = set()
+        
+        for potential_loop_path in potential_loop_paths.intersection( simple_children_to_parents.keys() ):
+            
+            new_potential_loop_paths.update( simple_children_to_parents[ potential_loop_path ] )
+            
+        
+        potential_loop_paths = new_potential_loop_paths
+        
+        if child in potential_loop_paths: return True
+        
+    
+    return False
+    
+class ClientFilesManager( object ):
+    
+    def __init__( self, controller ):
+        
+        self._controller = controller
+        
+        self._lock = threading.Lock()
+        
+        # fetch the current exact mapping from the db
+        
+        self._Reinit()
+        
+    
+    def _Reinit( self ):
+        
+        self._prefixes_to_locations = self._controller.Read( 'client_files_locations' )
+        
+    
+    def _GetRebalanceTuple( self ):
+        
+        paths_to_ideal_weights = self._controller.GetNewOptions().GetClientFilesLocationsToIdealWeights()
+        
+        total_weight = sum( paths_to_ideal_weights.values() )
+        
+        paths_to_normalised_ideal_weights = { path : weight / total_weight for ( path, weight ) in paths_to_ideal_weights.items() }
+        
+        current_paths_to_normalised_weights = {}
+        
+        for ( prefix, path ) in self._prefixes_to_locations.items():
+            
+            current_paths_to_normalised_weights[ path ] += 1.0 / 256
+            
+        
+        #
+        
+        overweight_paths = []
+        underweight_paths = []
+        
+        for ( path, ideal_weight ) in paths_to_normalised_ideal_weights.items():
+            
+            if path in current_paths_to_normalised_weights:
+                
+                current_weight = current_paths_to_normalised_weights[ path ]
+                
+                if current_weight < ideal_weight:
+                    
+                    underweight_paths.append( path )
+                    
+                elif current_weight >= ideal_weight + 1.0 / 256:
+                    
+                    overweight_paths.append( path )
+                    
+                
+            else:
+                
+                underweight_paths.append( path )
+                
+            
+        
+        #
+        
+        if len( underweight_paths ) == 0 or len( overweight_paths ) == 0:
+            
+            return None
+            
+        else:
+            
+            overweight_path = overweight_paths.pop( 0 )
+            underweight_path = underweight_paths.pop( 0 )
+            
+            for ( prefix, path ) in self._prefixes_to_locations.items():
+                
+                if path == overweight_path:
+                    
+                    return ( prefix, overweight_path, underweight_path )
+                    
+                
+            
+        
+    
+    def Rebalance( self, partial = True ):
+        
+        with self._lock:
+            
+            rebalance_tuple = self._GetRebalanceTuple()
+            
+            while rebalance_tuple is not None:
+                
+                ( prefix, overweight_path, underweight_path ) = rebalance_tuple
+                
+                self._controller.Write( 'move_client_files', prefix, overweight_path, underweight_path )
+                
+                self._Reinit()
+                
+                if partial:
+                    
+                    break
+                    
+                
+                rebalance_tuple = self._GetRebalanceTuple()
+                
+            
+        
+    
 class DataCache( object ):
     
-    def __init__( self, cache_size_key ):
+    def __init__( self, controller, cache_size_key ):
         
+        self._controller = controller
         self._cache_size_key = cache_size_key
         
         self._keys_to_data = {}
@@ -68,7 +331,7 @@ class DataCache( object ):
             
             if key not in self._keys_to_data:
                 
-                options = HydrusGlobals.client_controller.GetOptions()
+                options = self._controller.GetOptions()
                 
                 while self._total_estimated_memory_footprint > options[ self._cache_size_key ]:
                     
@@ -138,14 +401,16 @@ class DataCache( object ):
     
 class LocalBooruCache( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
+        
+        self._controller = controller
         
         self._lock = threading.Lock()
         
         self._RefreshShares()
         
-        HydrusGlobals.client_controller.sub( self, 'RefreshShares', 'refresh_local_booru_shares' )
-        HydrusGlobals.client_controller.sub( self, 'RefreshShares', 'restart_booru' )
+        self._controller.sub( self, 'RefreshShares', 'refresh_local_booru_shares' )
+        self._controller.sub( self, 'RefreshShares', 'restart_booru' )
         
     
     def _CheckDataUsage( self ):
@@ -185,13 +450,13 @@ class LocalBooruCache( object ):
         
         if info is None:
             
-            info = HydrusGlobals.client_controller.Read( 'local_booru_share', share_key )
+            info = self._controller.Read( 'local_booru_share', share_key )
             
             hashes = info[ 'hashes' ]
             
             info[ 'hashes_set' ] = set( hashes )
             
-            media_results = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
+            media_results = self._controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
             
             info[ 'media_results' ] = media_results
             
@@ -207,11 +472,11 @@ class LocalBooruCache( object ):
     
     def _RefreshShares( self ):
         
-        self._local_booru_service = HydrusGlobals.client_controller.GetServicesManager().GetService( CC.LOCAL_BOORU_SERVICE_KEY )
+        self._local_booru_service = self._controller.GetServicesManager().GetService( CC.LOCAL_BOORU_SERVICE_KEY )
         
         self._keys_to_infos = {}
         
-        share_keys = HydrusGlobals.client_controller.Read( 'local_booru_share_keys' )
+        share_keys = self._controller.Read( 'local_booru_share_keys' )
         
         for share_key in share_keys: self._keys_to_infos[ share_key ] = None
         
@@ -280,6 +545,62 @@ class LocalBooruCache( object ):
             
         
     
+class HydrusSessionManager( object ):
+    
+    def __init__( self, controller ):
+        
+        self._controller = controller
+        
+        existing_sessions = self._controller.Read( 'hydrus_sessions' )
+        
+        self._service_keys_to_sessions = { service_key : ( session_key, expires ) for ( service_key, session_key, expires ) in existing_sessions }
+        
+        self._lock = threading.Lock()
+        
+    
+    def DeleteSessionKey( self, service_key ):
+        
+        with self._lock:
+            
+            self._controller.Write( 'delete_hydrus_session_key', service_key )
+            
+            if service_key in self._service_keys_to_sessions: del self._service_keys_to_sessions[ service_key ]
+            
+        
+    
+    def GetSessionKey( self, service_key ):
+        
+        now = HydrusData.GetNow()
+        
+        with self._lock:
+            
+            if service_key in self._service_keys_to_sessions:
+                
+                ( session_key, expires ) = self._service_keys_to_sessions[ service_key ]
+                
+                if now + 600 > expires: del self._service_keys_to_sessions[ service_key ]
+                else: return session_key
+                
+            
+            # session key expired or not found
+            
+            service = self._controller.GetServicesManager().GetService( service_key )
+            
+            ( response_gumpf, cookies ) = service.Request( HC.GET, 'session_key', return_cookies = True )
+            
+            try: session_key = cookies[ 'session_key' ].decode( 'hex' )
+            except: raise Exception( 'Service did not return a session key!' )
+            
+            expires = now + HydrusSessions.HYDRUS_SESSION_LIFETIME
+            
+            self._service_keys_to_sessions[ service_key ] = ( session_key, expires )
+            
+            self._controller.Write( 'hydrus_session', service_key, session_key, expires )
+            
+            return session_key
+            
+        
+    
 class MenuEventIdToActionCache( object ):
     
     def __init__( self ):
@@ -377,18 +698,19 @@ MENU_EVENT_ID_TO_ACTION_CACHE = MenuEventIdToActionCache()
 
 class RenderedImageCache( object ):
     
-    def __init__( self, cache_type ):
+    def __init__( self, controller, cache_type ):
         
+        self._controller = controller
         self._type = cache_type
         
-        if self._type == 'fullscreen': self._data_cache = DataCache( 'fullscreen_cache_size' )
-        elif self._type == 'preview': self._data_cache = DataCache( 'preview_cache_size' )
+        if self._type == 'fullscreen': self._data_cache = DataCache( self._controller, 'fullscreen_cache_size' )
+        elif self._type == 'preview': self._data_cache = DataCache( self._controller, 'preview_cache_size' )
         
         self._total_estimated_memory_footprint = 0
         
         self._keys_being_rendered = {}
         
-        HydrusGlobals.client_controller.sub( self, 'FinishedRendering', 'finished_rendering' )
+        self._controller.sub( self, 'FinishedRendering', 'finished_rendering' )
         
     
     def Clear( self ): self._data_cache.Clear()
@@ -434,9 +756,10 @@ class RenderedImageCache( object ):
     
 class ThumbnailCache( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
         
-        self._data_cache = DataCache( 'thumbnail_cache_size' )
+        self._controller = controller
+        self._data_cache = DataCache( self._controller, 'thumbnail_cache_size' )
         
         self._lock = threading.Lock()
         
@@ -451,7 +774,7 @@ class ThumbnailCache( object ):
         
         threading.Thread( target = self.DAEMONWaterfall, name = 'Waterfall Daemon' ).start()
         
-        HydrusGlobals.client_controller.sub( self, 'Clear', 'thumbnail_resize' )
+        self._controller.sub( self, 'Clear', 'thumbnail_resize' )
         
     
     def _RecalcWaterfallQueueRandom( self ):
@@ -487,7 +810,7 @@ class ThumbnailCache( object ):
                 
                 path = os.path.join( HC.STATIC_DIR, name + '.png' )
                 
-                options = HydrusGlobals.client_controller.GetOptions()
+                options = self._controller.GetOptions()
                 
                 thumbnail = HydrusFileHandling.GenerateThumbnail( path, options[ 'thumbnail_dimensions' ] )
                 
@@ -613,7 +936,7 @@ class ThumbnailCache( object ):
                 
                 self.GetThumbnail( media ) # to load it
                 
-                HydrusGlobals.client_controller.pub( 'waterfall_thumbnail', page_key, media )
+                self._controller.pub( 'waterfall_thumbnail', page_key, media )
                 
                 if HydrusData.GetNowPrecise() - last_paused > 0.005:
                     
@@ -629,218 +952,69 @@ class ThumbnailCache( object ):
             
         
     
-def LoopInSimpleChildrenToParents( simple_children_to_parents, child, parent ):
+class ServicesManager( object ):
     
-    potential_loop_paths = { parent }
-    
-    while len( potential_loop_paths.intersection( simple_children_to_parents.keys() ) ) > 0:
+    def __init__( self, controller ):
         
-        new_potential_loop_paths = set()
-        
-        for potential_loop_path in potential_loop_paths.intersection( simple_children_to_parents.keys() ):
-            
-            new_potential_loop_paths.update( simple_children_to_parents[ potential_loop_path ] )
-            
-        
-        potential_loop_paths = new_potential_loop_paths
-        
-        if child in potential_loop_paths: return True
-        
-    
-    return False
-    
-def BuildSimpleChildrenToParents( pairs ):
-    
-    simple_children_to_parents = HydrusData.default_dict_set()
-    
-    for ( child, parent ) in pairs:
-        
-        if child == parent: continue
-        
-        if LoopInSimpleChildrenToParents( simple_children_to_parents, child, parent ): continue
-        
-        simple_children_to_parents[ child ].add( parent )
-        
-    
-    return simple_children_to_parents
-    
-def CollapseTagSiblingChains( processed_siblings ):
-    
-    # now to collapse chains
-    # A -> B and B -> C goes to A -> C and B -> C
-    
-    siblings = {}
-    
-    for ( old_tag, new_tag ) in processed_siblings.items():
-        
-        # adding A -> B
-        
-        if new_tag in siblings:
-            
-            # B -> F already calculated and added, so add A -> F
-            
-            siblings[ old_tag ] = siblings[ new_tag ]
-            
-        else:
-            
-            while new_tag in processed_siblings: new_tag = processed_siblings[ new_tag ] # pursue endpoint F
-            
-            siblings[ old_tag ] = new_tag
-            
-        
-    
-    reverse_lookup = collections.defaultdict( list )
-    
-    for ( old_tag, new_tag ) in siblings.items():
-        
-        reverse_lookup[ new_tag ].append( old_tag )
-        
-    
-    return ( siblings, reverse_lookup )
-    
-def CombineTagSiblingPairs( service_keys_to_statuses_to_pairs ):
-    
-    # first combine the services
-    # if A map already exists, don't overwrite
-    # if A -> B forms a loop, don't write it
-    
-    processed_siblings = {}
-    current_deleted_pairs = set()
-    
-    for ( service_key, statuses_to_pairs ) in service_keys_to_statuses_to_pairs.items():
-        
-        pairs = statuses_to_pairs[ HC.CURRENT ].union( statuses_to_pairs[ HC.PENDING ] )
-        
-        for ( old, new ) in pairs:
-            
-            if old == new: continue
-            
-            if old not in processed_siblings:
-                
-                next_new = new
-                
-                we_have_a_loop = False
-                
-                while next_new in processed_siblings:
-                    
-                    next_new = processed_siblings[ next_new ]
-                    
-                    if next_new == old:
-                        
-                        we_have_a_loop = True
-                        
-                        break
-                        
-                    
-                
-                if not we_have_a_loop: processed_siblings[ old ] = new
-                
-            
-        
-    
-    return processed_siblings
-    
-# important thing here, and reason why it is recursive, is because we want to preserve the parent-grandparent interleaving
-def BuildServiceKeysToChildrenToParents( service_keys_to_simple_children_to_parents ):
-    
-    def AddParents( simple_children_to_parents, children_to_parents, child, parents ):
-        
-        for parent in parents:
-            
-            children_to_parents[ child ].append( parent )
-            
-            if parent in simple_children_to_parents:
-                
-                grandparents = simple_children_to_parents[ parent ]
-                
-                AddParents( simple_children_to_parents, children_to_parents, child, grandparents )
-                
-            
-        
-    
-    service_keys_to_children_to_parents = collections.defaultdict( HydrusData.default_dict_list )
-    
-    for ( service_key, simple_children_to_parents ) in service_keys_to_simple_children_to_parents.items():
-        
-        children_to_parents = service_keys_to_children_to_parents[ service_key ]
-        
-        for ( child, parents ) in simple_children_to_parents.items(): AddParents( simple_children_to_parents, children_to_parents, child, parents )
-        
-    
-    return service_keys_to_children_to_parents
-    
-def BuildServiceKeysToSimpleChildrenToParents( service_keys_to_pairs_flat ):
-    
-    service_keys_to_simple_children_to_parents = collections.defaultdict( HydrusData.default_dict_set )
-    
-    for ( service_key, pairs ) in service_keys_to_pairs_flat.items():
-        
-        service_keys_to_simple_children_to_parents[ service_key ] = BuildSimpleChildrenToParents( pairs )
-        
-    
-    return service_keys_to_simple_children_to_parents
-    
-class HydrusSessionManagerClient( object ):
-    
-    def __init__( self ):
-        
-        existing_sessions = HydrusGlobals.client_controller.Read( 'hydrus_sessions' )
-        
-        self._service_keys_to_sessions = { service_key : ( session_key, expires ) for ( service_key, session_key, expires ) in existing_sessions }
+        self._controller = controller
         
         self._lock = threading.Lock()
+        self._keys_to_services = {}
+        self._services_sorted = []
+        
+        self.RefreshServices()
+        
+        self._controller.sub( self, 'RefreshServices', 'notify_new_services_data' )
         
     
-    def DeleteSessionKey( self, service_key ):
+    def GetService( self, service_key ):
         
         with self._lock:
             
-            HydrusGlobals.client_controller.Write( 'delete_hydrus_session_key', service_key )
-            
-            if service_key in self._service_keys_to_sessions: del self._service_keys_to_sessions[ service_key ]
+            try: return self._keys_to_services[ service_key ]
+            except KeyError: raise HydrusExceptions.NotFoundException( 'That service was not found!' )
             
         
     
-    def GetSessionKey( self, service_key ):
-        
-        now = HydrusData.GetNow()
+    def GetServices( self, types = HC.ALL_SERVICES, randomised = True ):
         
         with self._lock:
             
-            if service_key in self._service_keys_to_sessions:
+            services = [ service for service in self._services_sorted if service.GetServiceType() in types ]
+            
+            if randomised:
                 
-                ( session_key, expires ) = self._service_keys_to_sessions[ service_key ]
-                
-                if now + 600 > expires: del self._service_keys_to_sessions[ service_key ]
-                else: return session_key
+                random.shuffle( services )
                 
             
-            # session key expired or not found
+            return services
             
-            service = HydrusGlobals.client_controller.GetServicesManager().GetService( service_key )
+        
+    
+    def RefreshServices( self ):
+        
+        with self._lock:
             
-            ( response_gumpf, cookies ) = service.Request( HC.GET, 'session_key', return_cookies = True )
+            services = self._controller.Read( 'services' )
             
-            try: session_key = cookies[ 'session_key' ].decode( 'hex' )
-            except: raise Exception( 'Service did not return a session key!' )
+            self._keys_to_services = { service.GetServiceKey() : service for service in services }
             
-            expires = now + HydrusSessions.HYDRUS_SESSION_LIFETIME
+            compare_function = lambda a, b: cmp( a.GetName(), b.GetName() )
             
-            self._service_keys_to_sessions[ service_key ] = ( session_key, expires )
-            
-            HydrusGlobals.client_controller.Write( 'hydrus_session', service_key, session_key, expires )
-            
-            return session_key
+            self._services_sorted = list( services )
+            self._services_sorted.sort( cmp = compare_function )
             
         
     
 class TagCensorshipManager( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
+        
+        self._controller = controller
         
         self.RefreshData()
         
-        HydrusGlobals.client_controller.sub( self, 'RefreshData', 'notify_new_tag_censorship' )
+        self._controller.sub( self, 'RefreshData', 'notify_new_tag_censorship' )
         
     
     def GetInfo( self, service_key ):
@@ -851,7 +1025,7 @@ class TagCensorshipManager( object ):
     
     def RefreshData( self ):
         
-        info = HydrusGlobals.client_controller.Read( 'tag_censorship' )
+        info = self._controller.Read( 'tag_censorship' )
         
         self._service_keys_to_info = {}
         self._service_keys_to_predicates = {}
@@ -921,7 +1095,9 @@ class TagCensorshipManager( object ):
     
 class TagParentsManager( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
+        
+        self._controller = controller
         
         self._service_keys_to_children_to_parents = collections.defaultdict( HydrusData.default_dict_list )
         
@@ -929,16 +1105,16 @@ class TagParentsManager( object ):
         
         self._lock = threading.Lock()
         
-        HydrusGlobals.client_controller.sub( self, 'RefreshParents', 'notify_new_parents' )
+        self._controller.sub( self, 'RefreshParents', 'notify_new_parents' )
         
     
     def _RefreshParents( self ):
         
-        service_keys_to_statuses_to_pairs = HydrusGlobals.client_controller.Read( 'tag_parents' )
+        service_keys_to_statuses_to_pairs = self._controller.Read( 'tag_parents' )
         
         # first collapse siblings
         
-        sibling_manager = HydrusGlobals.client_controller.GetManager( 'tag_siblings' )
+        sibling_manager = self._controller.GetManager( 'tag_siblings' )
         
         collapsed_service_keys_to_statuses_to_pairs = collections.defaultdict( HydrusData.default_dict_set )
         
@@ -985,7 +1161,7 @@ class TagParentsManager( object ):
     
     def ExpandPredicates( self, service_key, predicates ):
         
-        new_options = HydrusGlobals.client_controller.GetNewOptions()
+        new_options = self._controller.GetNewOptions()
         
         if new_options.GetBoolean( 'apply_all_parents_to_all_services' ):
             
@@ -1021,7 +1197,7 @@ class TagParentsManager( object ):
     
     def ExpandTags( self, service_key, tags ):
         
-        new_options = HydrusGlobals.client_controller.GetNewOptions()
+        new_options = self._controller.GetNewOptions()
         
         if new_options.GetBoolean( 'apply_all_parents_to_all_services' ):
             
@@ -1043,7 +1219,7 @@ class TagParentsManager( object ):
     
     def GetParents( self, service_key, tag ):
         
-        new_options = HydrusGlobals.client_controller.GetNewOptions()
+        new_options = self._controller.GetNewOptions()
         
         if new_options.GetBoolean( 'apply_all_parents_to_all_services' ):
             
@@ -1066,13 +1242,15 @@ class TagParentsManager( object ):
     
 class TagSiblingsManager( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
+        
+        self._controller = controller
         
         self._RefreshSiblings()
         
         self._lock = threading.Lock()
         
-        HydrusGlobals.client_controller.sub( self, 'RefreshSiblings', 'notify_new_siblings' )
+        self._controller.sub( self, 'RefreshSiblings', 'notify_new_siblings' )
         
     
     def _CollapseTags( self, tags ):
@@ -1082,13 +1260,13 @@ class TagSiblingsManager( object ):
     
     def _RefreshSiblings( self ):
         
-        service_keys_to_statuses_to_pairs = HydrusGlobals.client_controller.Read( 'tag_siblings' )
+        service_keys_to_statuses_to_pairs = self._controller.Read( 'tag_siblings' )
         
         processed_siblings = CombineTagSiblingPairs( service_keys_to_statuses_to_pairs )
         
         ( self._siblings, self._reverse_lookup ) = CollapseTagSiblingChains( processed_siblings )
         
-        HydrusGlobals.client_controller.pub( 'new_siblings_gui' )
+        self._controller.pub( 'new_siblings_gui' )
         
     
     def GetAutocompleteSiblings( self, half_complete_tag ):
@@ -1281,11 +1459,237 @@ class TagSiblingsManager( object ):
             
         
     
+class UndoManager( object ):
+    
+    def __init__( self, controller ):
+        
+        self._controller = controller
+        
+        self._commands = []
+        self._inverted_commands = []
+        self._current_index = 0
+        
+        self._lock = threading.Lock()
+        
+        self._controller.sub( self, 'Undo', 'undo' )
+        self._controller.sub( self, 'Redo', 'redo' )
+        
+    
+    def _FilterServiceKeysToContentUpdates( self, service_keys_to_content_updates ):
+        
+        filtered_service_keys_to_content_updates = {}
+        
+        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
+            
+            filtered_content_updates = []
+            
+            for content_update in content_updates:
+                
+                ( data_type, action, row ) = content_update.ToTuple()
+                
+                if data_type == HC.CONTENT_TYPE_FILES:
+                    if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE, HC.CONTENT_UPDATE_UNDELETE, HC.CONTENT_UPDATE_RESCIND_PETITION ): continue
+                elif data_type == HC.CONTENT_TYPE_MAPPINGS:
+                    
+                    if action in ( HC.CONTENT_UPDATE_RESCIND_PETITION, HC.CONTENT_UPDATE_ADVANCED ): continue
+                    
+                else: continue
+                
+                filtered_content_update = HydrusData.ContentUpdate( data_type, action, row )
+                
+                filtered_content_updates.append( filtered_content_update )
+                
+            
+            if len( filtered_content_updates ) > 0:
+                
+                filtered_service_keys_to_content_updates[ service_key ] = filtered_content_updates
+                
+            
+        
+        return filtered_service_keys_to_content_updates
+        
+    
+    def _InvertServiceKeysToContentUpdates( self, service_keys_to_content_updates ):
+        
+        inverted_service_keys_to_content_updates = {}
+        
+        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
+            
+            inverted_content_updates = []
+            
+            for content_update in content_updates:
+                
+                ( data_type, action, row ) = content_update.ToTuple()
+                
+                inverted_row = row
+                
+                if data_type == HC.CONTENT_TYPE_FILES:
+                    
+                    if action == HC.CONTENT_UPDATE_ARCHIVE: inverted_action = HC.CONTENT_UPDATE_INBOX
+                    elif action == HC.CONTENT_UPDATE_INBOX: inverted_action = HC.CONTENT_UPDATE_ARCHIVE
+                    elif action == HC.CONTENT_UPDATE_PEND: inverted_action = HC.CONTENT_UPDATE_RESCIND_PEND
+                    elif action == HC.CONTENT_UPDATE_RESCIND_PEND: inverted_action = HC.CONTENT_UPDATE_PEND
+                    elif action == HC.CONTENT_UPDATE_PETITION:
+                        
+                        inverted_action = HC.CONTENT_UPDATE_RESCIND_PETITION
+                        
+                        ( hashes, reason ) = row
+                        
+                        inverted_row = hashes
+                        
+                    
+                elif data_type == HC.CONTENT_TYPE_MAPPINGS:
+                    
+                    if action == HC.CONTENT_UPDATE_ADD: inverted_action = HC.CONTENT_UPDATE_DELETE
+                    elif action == HC.CONTENT_UPDATE_DELETE: inverted_action = HC.CONTENT_UPDATE_ADD
+                    elif action == HC.CONTENT_UPDATE_PEND: inverted_action = HC.CONTENT_UPDATE_RESCIND_PEND
+                    elif action == HC.CONTENT_UPDATE_RESCIND_PEND: inverted_action = HC.CONTENT_UPDATE_PEND
+                    elif action == HC.CONTENT_UPDATE_PETITION:
+                        
+                        inverted_action = HC.CONTENT_UPDATE_RESCIND_PETITION
+                        
+                        ( tag, hashes, reason ) = row
+                        
+                        inverted_row = ( tag, hashes )
+                        
+                    
+                
+                inverted_content_update = HydrusData.ContentUpdate( data_type, inverted_action, inverted_row )
+                
+                inverted_content_updates.append( inverted_content_update )
+                
+            
+            inverted_service_keys_to_content_updates[ service_key ] = inverted_content_updates
+            
+        
+        return inverted_service_keys_to_content_updates
+        
+    
+    def AddCommand( self, action, *args, **kwargs ):
+        
+        with self._lock:
+            
+            inverted_action = action
+            inverted_args = args
+            inverted_kwargs = kwargs
+            
+            if action == 'content_updates':
+                
+                ( service_keys_to_content_updates, ) = args
+                
+                service_keys_to_content_updates = self._FilterServiceKeysToContentUpdates( service_keys_to_content_updates )
+                
+                if len( service_keys_to_content_updates ) == 0: return
+                
+                inverted_service_keys_to_content_updates = self._InvertServiceKeysToContentUpdates( service_keys_to_content_updates )
+                
+                if len( inverted_service_keys_to_content_updates ) == 0: return
+                
+                inverted_args = ( inverted_service_keys_to_content_updates, )
+                
+            else: return
+            
+            self._commands = self._commands[ : self._current_index ]
+            self._inverted_commands = self._inverted_commands[ : self._current_index ]
+            
+            self._commands.append( ( action, args, kwargs ) )
+            
+            self._inverted_commands.append( ( inverted_action, inverted_args, inverted_kwargs ) )
+            
+            self._current_index += 1
+            
+            self._controller.pub( 'notify_new_undo' )
+            
+        
+    
+    def GetUndoRedoStrings( self ):
+        
+        with self._lock:
+            
+            ( undo_string, redo_string ) = ( None, None )
+            
+            if self._current_index > 0:
+                
+                undo_index = self._current_index - 1
+                
+                ( action, args, kwargs ) = self._commands[ undo_index ]
+                
+                if action == 'content_updates':
+                    
+                    ( service_keys_to_content_updates, ) = args
+                    
+                    undo_string = 'undo ' + ClientData.ConvertServiceKeysToContentUpdatesToPrettyString( service_keys_to_content_updates )
+                    
+                
+            
+            if len( self._commands ) > 0 and self._current_index < len( self._commands ):
+                
+                redo_index = self._current_index
+                
+                ( action, args, kwargs ) = self._commands[ redo_index ]
+                
+                if action == 'content_updates':
+                    
+                    ( service_keys_to_content_updates, ) = args
+                    
+                    redo_string = 'redo ' + ClientData.ConvertServiceKeysToContentUpdatesToPrettyString( service_keys_to_content_updates )
+                    
+                
+            
+            return ( undo_string, redo_string )
+            
+        
+    
+    def Undo( self ):
+        
+        action = None
+        
+        with self._lock:
+            
+            if self._current_index > 0:
+                
+                self._current_index -= 1
+                
+                ( action, args, kwargs ) = self._inverted_commands[ self._current_index ]
+                
+        
+        if action is not None:
+            
+            self._controller.WriteSynchronous( action, *args, **kwargs )
+            
+            self._controller.pub( 'notify_new_undo' )
+            
+        
+    
+    def Redo( self ):
+        
+        action = None
+        
+        with self._lock:
+            
+            if len( self._commands ) > 0 and self._current_index < len( self._commands ):
+                
+                ( action, args, kwargs ) = self._commands[ self._current_index ]
+                
+                self._current_index += 1
+                
+            
+        
+        if action is not None:
+            
+            self._controller.WriteSynchronous( action, *args, **kwargs )
+            
+            self._controller.pub( 'notify_new_undo' )
+            
+        
+    
 class WebSessionManagerClient( object ):
     
-    def __init__( self ):
+    def __init__( self, controller ):
         
-        existing_sessions = HydrusGlobals.client_controller.Read( 'web_sessions' )
+        self._controller = controller
+        
+        existing_sessions = self._controller.Read( 'web_sessions' )
         
         self._names_to_sessions = { name : ( cookies, expires ) for ( name, cookies, expires ) in existing_sessions }
         
@@ -1310,13 +1714,13 @@ class WebSessionManagerClient( object ):
             
             if name == 'deviant art':
                 
-                ( response_gumpf, cookies ) = HydrusGlobals.client_controller.DoHTTP( HC.GET, 'http://www.deviantart.com/', return_cookies = True )
+                ( response_gumpf, cookies ) = self._controller.DoHTTP( HC.GET, 'http://www.deviantart.com/', return_cookies = True )
                 
                 expires = now + 30 * 86400
                 
             if name == 'hentai foundry':
                 
-                ( response_gumpf, cookies ) = HydrusGlobals.client_controller.DoHTTP( HC.GET, 'http://www.hentai-foundry.com/?enterAgree=1', return_cookies = True )
+                ( response_gumpf, cookies ) = self._controller.DoHTTP( HC.GET, 'http://www.hentai-foundry.com/?enterAgree=1', return_cookies = True )
                 
                 raw_csrf = cookies[ 'YII_CSRF_TOKEN' ] # 19b05b536885ec60b8b37650a32f8deb11c08cd1s%3A40%3A%222917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32%22%3B
                 
@@ -1334,13 +1738,13 @@ class WebSessionManagerClient( object ):
                 ClientNetworking.AddCookiesToHeaders( cookies, request_headers )
                 request_headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
                 
-                HydrusGlobals.client_controller.DoHTTP( HC.POST, 'http://www.hentai-foundry.com/site/filters', request_headers = request_headers, body = body )
+                self._controller.DoHTTP( HC.POST, 'http://www.hentai-foundry.com/site/filters', request_headers = request_headers, body = body )
                 
                 expires = now + 60 * 60
                 
             elif name == 'pixiv':
                 
-                ( id, password ) = HydrusGlobals.client_controller.Read( 'pixiv_account' )
+                ( id, password ) = self._controller.Read( 'pixiv_account' )
                 
                 if id == '' and password == '':
                     
@@ -1359,7 +1763,7 @@ class WebSessionManagerClient( object ):
                 headers = {}
                 headers[ 'Content-Type' ] = 'application/x-www-form-urlencoded'
                 
-                ( response_gumpf, cookies ) = HydrusGlobals.client_controller.DoHTTP( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
+                ( response_gumpf, cookies ) = self._controller.DoHTTP( HC.POST, 'http://www.pixiv.net/login.php', request_headers = headers, body = body, return_cookies = True )
                 
                 # _ only given to logged in php sessions
                 if 'PHPSESSID' not in cookies or '_' not in cookies[ 'PHPSESSID' ]: raise Exception( 'Pixiv login credentials not accepted!' )
@@ -1369,7 +1773,7 @@ class WebSessionManagerClient( object ):
             
             self._names_to_sessions[ name ] = ( cookies, expires )
             
-            HydrusGlobals.client_controller.Write( 'web_session', name, cookies, expires )
+            self._controller.Write( 'web_session', name, cookies, expires )
             
             return cookies
             
diff --git a/include/ClientController.py b/include/ClientController.py
index 4e69c089..7dd1f1e4 100755
--- a/include/ClientController.py
+++ b/include/ClientController.py
@@ -44,6 +44,7 @@ class Controller( HydrusController.HydrusController ):
         HydrusGlobals.client_controller = self
         
         self._last_mouse_position = None
+        self._menu_open = False
         
     
     def _InitDB( self ):
@@ -361,8 +362,15 @@ class Controller( HydrusController.HydrusController ):
     
     def ForceIdle( self ):
         
-        del self._timestamps[ 'last_user_action' ]
-        del self._timestamps[ 'last_mouse_action' ]
+        if 'last_user_action' in self._timestamps:
+            
+            del self._timestamps[ 'last_user_action' ]
+            
+        
+        if 'last_mouse_action' in self._timestamps:
+            
+            del self._timestamps[ 'last_mouse_action' ]
+            
         
         self._last_mouse_position = None
         
@@ -410,15 +418,15 @@ class Controller( HydrusController.HydrusController ):
         
         HC.options = self._options
         
-        self._services_manager = ClientData.ServicesManager()
+        self._services_manager = ClientCaches.ServicesManager( self )
         
-        self._managers[ 'hydrus_sessions' ] = ClientCaches.HydrusSessionManagerClient()
-        self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache()
-        self._managers[ 'tag_censorship' ] = ClientCaches.TagCensorshipManager()
-        self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager()
-        self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager()
-        self._managers[ 'undo' ] = ClientData.UndoManager()
-        self._managers[ 'web_sessions' ] = ClientCaches.WebSessionManagerClient()
+        self._managers[ 'hydrus_sessions' ] = ClientCaches.HydrusSessionManager( self )
+        self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache( self )
+        self._managers[ 'tag_censorship' ] = ClientCaches.TagCensorshipManager( self )
+        self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
+        self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
+        self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
+        self._managers[ 'web_sessions' ] = ClientCaches.WebSessionManagerClient( self )
         
         if HC.options[ 'proxy' ] is not None:
             
@@ -429,9 +437,9 @@ class Controller( HydrusController.HydrusController ):
         
         def wx_code():
             
-            self._caches[ 'fullscreen' ] = ClientCaches.RenderedImageCache( 'fullscreen' )
-            self._caches[ 'preview' ] = ClientCaches.RenderedImageCache( 'preview' )
-            self._caches[ 'thumbnail' ] = ClientCaches.ThumbnailCache()
+            self._caches[ 'fullscreen' ] = ClientCaches.RenderedImageCache( self, 'fullscreen' )
+            self._caches[ 'preview' ] = ClientCaches.RenderedImageCache( self, 'preview' )
+            self._caches[ 'thumbnail' ] = ClientCaches.ThumbnailCache( self )
             
             CC.GlobalBMPs.STATICInitialise()
             
@@ -555,6 +563,11 @@ class Controller( HydrusController.HydrusController ):
             
         
     
+    def MenuIsOpen( self ):
+        
+        return self._menu_open
+        
+    
     def NotifyPubSubs( self ):
         
         wx.CallAfter( self.ProcessPubSub )
@@ -570,6 +583,20 @@ class Controller( HydrusController.HydrusController ):
         return self._gui.PageHidden( page_key )
         
     
+    def PopupMenu( self, window, menu ):
+        
+        if menu.GetMenuItemCount() > 0:
+            
+            self._menu_open = True
+            
+            window.PopupMenu( menu )
+            
+            self._menu_open = False
+            
+        
+        wx.CallAfter( menu.Destroy )
+        
+    
     def PrepStringForDisplay( self, text ):
         
         if self._options[ 'gui_capitalisation' ]: return text
diff --git a/include/ClientDB.py b/include/ClientDB.py
index 1e51fabc..d61cb3ee 100755
--- a/include/ClientDB.py
+++ b/include/ClientDB.py
@@ -1155,6 +1155,8 @@ class DB( HydrusDB.HydrusDB ):
             self._c.execute( 'INSERT OR REPLACE INTO perceptual_hashes ( hash_id, phash ) VALUES ( ?, ? );', ( hash_id, sqlite3.Binary( phash ) ) )
             
         
+        self._c.execute( 'DELETE FROM service_info WHERE info_type = ?;', ( HC.SERVICE_INFO_NUM_THUMBNAILS_LOCAL, ) )
+        
         hashes = { hash for ( hash, thumbnail ) in thumbnails }
         
         self.pub_after_commit( 'new_thumbnails', hashes )
@@ -2186,12 +2188,15 @@ class DB( HydrusDB.HydrusDB ):
             current_counts = collections.defaultdict( zero )
             pending_counts = collections.defaultdict( zero )
             
-            current_counts.update( { tag_id : count for ( tag_id, count ) in self._c.execute( count_phrase + table_phrase + predicates_phrase + 'tag_id IN ' + HydrusData.SplayListForDB( tag_ids ) + ' GROUP BY tag_id;', ( HC.CURRENT, namespace_id ) ) } )
-            pending_counts.update( { tag_id : count for ( tag_id, count ) in self._c.execute( count_phrase + table_phrase + predicates_phrase + 'tag_id IN ' + HydrusData.SplayListForDB( tag_ids ) + ' GROUP BY tag_id;', ( HC.PENDING, namespace_id ) ) } )
+            for sub_tag_ids in HydrusData.SplitListIntoChunks( tag_ids, 50 ):
+                
+                current_counts.update( { tag_id : count for ( tag_id, count ) in self._c.execute( count_phrase + table_phrase + predicates_phrase + 'tag_id IN ' + HydrusData.SplayListForDB( sub_tag_ids ) + ' GROUP BY tag_id;', ( HC.CURRENT, namespace_id ) ) } )
+                pending_counts.update( { tag_id : count for ( tag_id, count ) in self._c.execute( count_phrase + table_phrase + predicates_phrase + 'tag_id IN ' + HydrusData.SplayListForDB( sub_tag_ids ) + ' GROUP BY tag_id;', ( HC.PENDING, namespace_id ) ) } )
+                
             
-            self._c.executemany( 'INSERT OR IGNORE INTO autocomplete_tags_cache ( file_service_id, tag_service_id, namespace_id, tag_id, current_count, pending_count ) VALUES ( ?, ?, ?, ?, ?, ? );', [ ( file_service_id, tag_service_id, namespace_id, tag_id, current_counts[ tag_id ], pending_counts[ tag_id ] ) for tag_id in tag_ids ] )
+            self._c.executemany( 'INSERT OR IGNORE INTO autocomplete_tags_cache ( file_service_id, tag_service_id, namespace_id, tag_id, current_count, pending_count ) VALUES ( ?, ?, ?, ?, ?, ? );', ( ( file_service_id, tag_service_id, namespace_id, tag_id, current_counts[ tag_id ], pending_counts[ tag_id ] ) for tag_id in tag_ids ) )
             
-            cache_results.extend( [ ( namespace_id, tag_id, current_counts[ tag_id ], pending_counts[ tag_id ] ) for tag_id in tag_ids ] )
+            cache_results.extend( ( ( namespace_id, tag_id, current_counts[ tag_id ], pending_counts[ tag_id ] ) for tag_id in tag_ids ) )
             
         
         #
@@ -2847,10 +2852,6 @@ class DB( HydrusDB.HydrusDB ):
         
         tags = siblings_manager.GetAllSiblings( tag )
         
-        tag_censorship_manager = self._controller.GetManager( 'tag_censorship' )
-        
-        tags = tag_censorship_manager.FilterTags( tag_service_key, tags )
-        
         hash_ids = set()
         
         predicates = []
@@ -3690,15 +3691,23 @@ class DB( HydrusDB.HydrusDB ):
                     elif info_type == HC.SERVICE_INFO_NUM_THUMBNAILS: result = self._c.execute( 'SELECT COUNT( * ) FROM files_info WHERE service_id = ? AND mime IN ' + HydrusData.SplayListForDB( HC.MIMES_WITH_THUMBNAILS ) + ';', ( service_id, ) ).fetchone()
                     elif info_type == HC.SERVICE_INFO_NUM_THUMBNAILS_LOCAL:
                         
-                        thumbnails_i_have = ClientFiles.GetAllThumbnailHashes()
-                        
                         hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM files_info WHERE mime IN ' + HydrusData.SplayListForDB( HC.MIMES_WITH_THUMBNAILS ) + ' AND service_id = ?;', ( service_id, ) ) ]
                         
                         thumbnails_i_should_have = self._GetHashes( hash_ids )
                         
-                        thumbnails_i_have.intersection_update( thumbnails_i_should_have )
+                        num_local = 0
                         
-                        result = ( len( thumbnails_i_have ), )
+                        for hash in thumbnails_i_should_have:
+                            
+                            path = ClientFiles.GetExpectedThumbnailPath( hash )
+                            
+                            if os.path.exists( path ):
+                                
+                                num_local += 1
+                                
+                            
+                        
+                        result = ( num_local, )
                         
                     elif info_type == HC.SERVICE_INFO_NUM_INBOX: result = self._c.execute( 'SELECT COUNT( * ) FROM file_inbox, files_info USING ( hash_id ) WHERE service_id = ?;', ( service_id, ) ).fetchone()
                     
@@ -4550,7 +4559,10 @@ class DB( HydrusDB.HydrusDB ):
                                 
                                 ( new_namespace_id, new_tag_id ) = self._GetNamespaceIdTagId( new_tag )
                                 
-                            except HydrusExceptions.SizeException: continue
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
                             
                             self._c.execute( 'DELETE FROM tag_siblings WHERE service_id = ? AND old_namespace_id = ? AND old_tag_id = ?;', ( service_id, old_namespace_id, old_tag_id ) )
                             self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND old_namespace_id = ? AND old_tag_id = ? AND status = ?;', ( service_id, old_namespace_id, old_tag_id, deletee_status ) )
@@ -4570,7 +4582,10 @@ class DB( HydrusDB.HydrusDB ):
                                 
                                 ( new_namespace_id, new_tag_id ) = self._GetNamespaceIdTagId( new_tag )
                                 
-                            except HydrusExceptions.SizeException: continue
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
                             
                             reason_id = self._GetReasonId( reason )
                             
@@ -4582,13 +4597,25 @@ class DB( HydrusDB.HydrusDB ):
                             
                         elif action in ( HC.CONTENT_UPDATE_RESCIND_PEND, HC.CONTENT_UPDATE_RESCIND_PETITION ):
                             
-                            if action == HC.CONTENT_UPDATE_RESCIND_PEND: deletee_status = HC.PENDING
-                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION: deletee_status = HC.PETITIONED
+                            if action == HC.CONTENT_UPDATE_RESCIND_PEND:
+                                
+                                deletee_status = HC.PENDING
+                                
+                            elif action == HC.CONTENT_UPDATE_RESCIND_PETITION:
+                                
+                                deletee_status = HC.PETITIONED
+                                
                             
                             ( old_tag, new_tag ) = row
                             
-                            try: ( old_namespace_id, old_tag_id ) = self._GetNamespaceIdTagId( old_tag )
-                            except HydrusExceptions.SizeException: continue
+                            try:
+                                
+                                ( old_namespace_id, old_tag_id ) = self._GetNamespaceIdTagId( old_tag )
+                                
+                            except HydrusExceptions.SizeException:
+                                
+                                continue
+                                
                             
                             self._c.execute( 'DELETE FROM tag_sibling_petitions WHERE service_id = ? AND old_namespace_id = ? AND old_tag_id = ? AND status = ?;', ( service_id, old_namespace_id, old_tag_id, deletee_status ) )
                             
@@ -6114,6 +6141,44 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
+        if version == 182:
+            
+            hash_ids = { hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM files_info WHERE mime = ?;', ( HC.APPLICATION_FLASH, ) ) }
+            
+            num_done = 0
+            
+            num_to_do = len( hash_ids )
+            
+            for hash_id in hash_ids:
+                
+                num_done += 1
+                
+                if num_done % 10 == 0:
+                    
+                    self._controller.pub( 'splash_set_status_text', 'updating flash thumbnails: ' + HydrusData.ConvertValueRangeToPrettyString( num_done, num_to_do ) )
+                    
+                
+                hash = self._GetHash( hash_id )
+                
+                try:
+                    
+                    file_path = ClientFiles.GetFilePath( hash, HC.APPLICATION_FLASH )
+                    
+                except HydrusExceptions.NotFoundException:
+                    
+                    continue
+                    
+                
+                thumbnail = HydrusFileHandling.GenerateThumbnail( file_path )
+                
+                self._AddThumbnails( [ ( hash, thumbnail ) ] )
+                
+            
+            #
+            
+            self._c.execute( 'DELETE FROM service_info WHERE info_type IN ( ?, ? );', ( HC.SERVICE_INFO_NUM_THUMBNAILS, HC.SERVICE_INFO_NUM_THUMBNAILS_LOCAL ) )
+            
+        
         self._controller.pub( 'splash_set_title_text', 'updating db to v' + str( version + 1 ) )
         
         self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
diff --git a/include/ClientData.py b/include/ClientData.py
index c467c89d..b37d5522 100644
--- a/include/ClientData.py
+++ b/include/ClientData.py
@@ -7,6 +7,7 @@ import collections
 import datetime
 import HydrusConstants as HC
 import HydrusExceptions
+import HydrusPaths
 import HydrusSerialisable
 import HydrusTags
 import threading
@@ -175,11 +176,11 @@ def DeletePath( path ):
     
     if HC.options[ 'delete_to_recycle_bin' ] == True:
         
-        HydrusData.RecyclePath( path )
+        HydrusPaths.RecyclePath( path )
         
     else:
         
-        HydrusData.DeletePath( path )
+        HydrusPaths.DeletePath( path )
         
     
 def GetMediasTagCount( pool, tag_service_key = CC.COMBINED_TAG_SERVICE_KEY, collapse_siblings = False ):
@@ -352,6 +353,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
         self._dictionary[ 'booleans' ][ 'apply_all_parents_to_all_services' ] = False
         self._dictionary[ 'booleans' ][ 'apply_all_siblings_to_all_services' ] = False
         
+        self._dictionary[ 'client_files_locations_ideal_weights' ] = [ ( HydrusPaths.ConvertAbsPathToPortablePath( HC.CLIENT_FILES_DIR ), 1.0 ) ]
+        
     
     def _InitialiseFromSerialisableInfo( self, serialisable_info ):
         
@@ -379,6 +382,20 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
             
         
     
+    def GetClientFilesLocationsToIdealWeights( self ):
+        
+        result = {}
+        
+        for ( portable_path, weight ) in self._dictionary[ 'client_files_locations_ideal_weights' ]:
+            
+            abs_path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path )
+            
+            result[ abs_path ] = weight
+            
+        
+        return result
+        
+    
     def GetDefaultImportTagOptions( self, gallery_identifier = None ):
         
         with self._lock:
@@ -2285,13 +2302,21 @@ class Service( HydrusData.HydrusYAMLBase ):
                 HydrusGlobals.client_controller.pub( 'splash_set_status_text', 'reviewing thumbnails' )
                 job_key.SetVariable( 'popup_text_1', 'reviewing existing thumbnails' )
                 
-                thumbnail_hashes_i_have = ClientFiles.GetAllThumbnailHashes()
-                
                 job_key.SetVariable( 'popup_text_1', 'reviewing service thumbnails' )
                 
                 thumbnail_hashes_i_should_have = HydrusGlobals.client_controller.Read( 'thumbnail_hashes_i_should_have', self._service_key )
                 
-                thumbnail_hashes_i_need = thumbnail_hashes_i_should_have.difference( thumbnail_hashes_i_have )
+                thumbnail_hashes_i_need = set()
+                
+                for hash in thumbnail_hashes_i_should_have:
+                    
+                    path = ClientFiles.GetExpectedThumbnailPath( hash )
+                    
+                    if not os.path.exists( path ):
+                        
+                        thumbnail_hashes_i_need.add( hash )
+                        
+                    
                 
                 if len( thumbnail_hashes_i_need ) > 0:
                     
@@ -2378,48 +2403,6 @@ class Service( HydrusData.HydrusYAMLBase ):
     
     def ToTuple( self ): return ( self._service_key, self._service_type, self._name, self._info )
     
-class ServicesManager( object ):
-    
-    def __init__( self ):
-        
-        self._lock = threading.Lock()
-        self._keys_to_services = {}
-        self._services_sorted = []
-        
-        self.RefreshServices()
-        
-        HydrusGlobals.client_controller.sub( self, 'RefreshServices', 'notify_new_services_data' )
-        
-    
-    def GetService( self, service_key ):
-        
-        with self._lock:
-            
-            try: return self._keys_to_services[ service_key ]
-            except KeyError: raise HydrusExceptions.NotFoundException( 'That service was not found!' )
-            
-        
-    
-    def GetServices( self, types = HC.ALL_SERVICES ):
-        
-        with self._lock: return [ service for service in self._services_sorted if service.GetServiceType() in types ]
-        
-    
-    def RefreshServices( self ):
-        
-        with self._lock:
-            
-            services = HydrusGlobals.client_controller.Read( 'services' )
-            
-            self._keys_to_services = { service.GetServiceKey() : service for service in services }
-            
-            compare_function = lambda a, b: cmp( a.GetName(), b.GetName() )
-            
-            self._services_sorted = list( services )
-            self._services_sorted.sort( cmp = compare_function )
-            
-        
-    
 class Shortcuts( HydrusSerialisable.SerialisableBaseNamed ):
     
     SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_SHORTCUTS
@@ -2579,228 +2562,6 @@ class Shortcuts( HydrusSerialisable.SerialisableBaseNamed ):
     
 HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_SHORTCUTS ] = Shortcuts
 
-class UndoManager( object ):
-    
-    def __init__( self ):
-        
-        self._commands = []
-        self._inverted_commands = []
-        self._current_index = 0
-        
-        self._lock = threading.Lock()
-        
-        HydrusGlobals.client_controller.sub( self, 'Undo', 'undo' )
-        HydrusGlobals.client_controller.sub( self, 'Redo', 'redo' )
-        
-    
-    def _FilterServiceKeysToContentUpdates( self, service_keys_to_content_updates ):
-        
-        filtered_service_keys_to_content_updates = {}
-        
-        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
-            
-            filtered_content_updates = []
-            
-            for content_update in content_updates:
-                
-                ( data_type, action, row ) = content_update.ToTuple()
-                
-                if data_type == HC.CONTENT_TYPE_FILES:
-                    if action in ( HC.CONTENT_UPDATE_ADD, HC.CONTENT_UPDATE_DELETE, HC.CONTENT_UPDATE_UNDELETE, HC.CONTENT_UPDATE_RESCIND_PETITION ): continue
-                elif data_type == HC.CONTENT_TYPE_MAPPINGS:
-                    
-                    if action in ( HC.CONTENT_UPDATE_RESCIND_PETITION, HC.CONTENT_UPDATE_ADVANCED ): continue
-                    
-                else: continue
-                
-                filtered_content_update = HydrusData.ContentUpdate( data_type, action, row )
-                
-                filtered_content_updates.append( filtered_content_update )
-                
-            
-            if len( filtered_content_updates ) > 0:
-                
-                filtered_service_keys_to_content_updates[ service_key ] = filtered_content_updates
-                
-            
-        
-        return filtered_service_keys_to_content_updates
-        
-    
-    def _InvertServiceKeysToContentUpdates( self, service_keys_to_content_updates ):
-        
-        inverted_service_keys_to_content_updates = {}
-        
-        for ( service_key, content_updates ) in service_keys_to_content_updates.items():
-            
-            inverted_content_updates = []
-            
-            for content_update in content_updates:
-                
-                ( data_type, action, row ) = content_update.ToTuple()
-                
-                inverted_row = row
-                
-                if data_type == HC.CONTENT_TYPE_FILES:
-                    
-                    if action == HC.CONTENT_UPDATE_ARCHIVE: inverted_action = HC.CONTENT_UPDATE_INBOX
-                    elif action == HC.CONTENT_UPDATE_INBOX: inverted_action = HC.CONTENT_UPDATE_ARCHIVE
-                    elif action == HC.CONTENT_UPDATE_PEND: inverted_action = HC.CONTENT_UPDATE_RESCIND_PEND
-                    elif action == HC.CONTENT_UPDATE_RESCIND_PEND: inverted_action = HC.CONTENT_UPDATE_PEND
-                    elif action == HC.CONTENT_UPDATE_PETITION:
-                        
-                        inverted_action = HC.CONTENT_UPDATE_RESCIND_PETITION
-                        
-                        ( hashes, reason ) = row
-                        
-                        inverted_row = hashes
-                        
-                    
-                elif data_type == HC.CONTENT_TYPE_MAPPINGS:
-                    
-                    if action == HC.CONTENT_UPDATE_ADD: inverted_action = HC.CONTENT_UPDATE_DELETE
-                    elif action == HC.CONTENT_UPDATE_DELETE: inverted_action = HC.CONTENT_UPDATE_ADD
-                    elif action == HC.CONTENT_UPDATE_PEND: inverted_action = HC.CONTENT_UPDATE_RESCIND_PEND
-                    elif action == HC.CONTENT_UPDATE_RESCIND_PEND: inverted_action = HC.CONTENT_UPDATE_PEND
-                    elif action == HC.CONTENT_UPDATE_PETITION:
-                        
-                        inverted_action = HC.CONTENT_UPDATE_RESCIND_PETITION
-                        
-                        ( tag, hashes, reason ) = row
-                        
-                        inverted_row = ( tag, hashes )
-                        
-                    
-                
-                inverted_content_update = HydrusData.ContentUpdate( data_type, inverted_action, inverted_row )
-                
-                inverted_content_updates.append( inverted_content_update )
-                
-            
-            inverted_service_keys_to_content_updates[ service_key ] = inverted_content_updates
-            
-        
-        return inverted_service_keys_to_content_updates
-        
-    
-    def AddCommand( self, action, *args, **kwargs ):
-        
-        with self._lock:
-            
-            inverted_action = action
-            inverted_args = args
-            inverted_kwargs = kwargs
-            
-            if action == 'content_updates':
-                
-                ( service_keys_to_content_updates, ) = args
-                
-                service_keys_to_content_updates = self._FilterServiceKeysToContentUpdates( service_keys_to_content_updates )
-                
-                if len( service_keys_to_content_updates ) == 0: return
-                
-                inverted_service_keys_to_content_updates = self._InvertServiceKeysToContentUpdates( service_keys_to_content_updates )
-                
-                if len( inverted_service_keys_to_content_updates ) == 0: return
-                
-                inverted_args = ( inverted_service_keys_to_content_updates, )
-                
-            else: return
-            
-            self._commands = self._commands[ : self._current_index ]
-            self._inverted_commands = self._inverted_commands[ : self._current_index ]
-            
-            self._commands.append( ( action, args, kwargs ) )
-            
-            self._inverted_commands.append( ( inverted_action, inverted_args, inverted_kwargs ) )
-            
-            self._current_index += 1
-            
-            HydrusGlobals.client_controller.pub( 'notify_new_undo' )
-            
-        
-    
-    def GetUndoRedoStrings( self ):
-        
-        with self._lock:
-            
-            ( undo_string, redo_string ) = ( None, None )
-            
-            if self._current_index > 0:
-                
-                undo_index = self._current_index - 1
-                
-                ( action, args, kwargs ) = self._commands[ undo_index ]
-                
-                if action == 'content_updates':
-                    
-                    ( service_keys_to_content_updates, ) = args
-                    
-                    undo_string = 'undo ' + ConvertServiceKeysToContentUpdatesToPrettyString( service_keys_to_content_updates )
-                    
-                
-            
-            if len( self._commands ) > 0 and self._current_index < len( self._commands ):
-                
-                redo_index = self._current_index
-                
-                ( action, args, kwargs ) = self._commands[ redo_index ]
-                
-                if action == 'content_updates':
-                    
-                    ( service_keys_to_content_updates, ) = args
-                    
-                    redo_string = 'redo ' + ConvertServiceKeysToContentUpdatesToPrettyString( service_keys_to_content_updates )
-                    
-                
-            
-            return ( undo_string, redo_string )
-            
-        
-    
-    def Undo( self ):
-        
-        action = None
-        
-        with self._lock:
-            
-            if self._current_index > 0:
-                
-                self._current_index -= 1
-                
-                ( action, args, kwargs ) = self._inverted_commands[ self._current_index ]
-                
-        
-        if action is not None:
-            
-            HydrusGlobals.client_controller.WriteSynchronous( action, *args, **kwargs )
-            
-            HydrusGlobals.client_controller.pub( 'notify_new_undo' )
-            
-        
-    
-    def Redo( self ):
-        
-        action = None
-        
-        with self._lock:
-            
-            if len( self._commands ) > 0 and self._current_index < len( self._commands ):
-                
-                ( action, args, kwargs ) = self._commands[ self._current_index ]
-                
-                self._current_index += 1
-                
-            
-        
-        if action is not None:
-            
-            HydrusGlobals.client_controller.WriteSynchronous( action, *args, **kwargs )
-            
-            HydrusGlobals.client_controller.pub( 'notify_new_undo' )
-            
-        
-    
 def GetShortcutFromEvent( event ):
     
     modifier = wx.ACCEL_NORMAL
diff --git a/include/ClientFiles.py b/include/ClientFiles.py
index 7e6e946e..28459807 100644
--- a/include/ClientFiles.py
+++ b/include/ClientFiles.py
@@ -6,6 +6,7 @@ import HydrusData
 import HydrusExceptions
 import HydrusFileHandling
 import HydrusGlobals
+import HydrusPaths
 import HydrusSerialisable
 import itertools
 import os
@@ -109,17 +110,9 @@ def GetAllThumbnailHashes():
     
     thumbnail_hashes = set()
     
-    for path in IterateAllThumbnailPaths():
+    for hash in IterateAllThumbnailHashes():
         
-        ( base, filename ) = os.path.split( path )
-        
-        if not filename.endswith( '_resized' ):
-            
-            try: hash = filename.decode( 'hex' )
-            except TypeError: continue
-            
-            thumbnail_hashes.add( hash )
-            
+        thumbnail_hashes.add( hash )
         
     
     return thumbnail_hashes
@@ -150,7 +143,7 @@ def GetExportPath():
     
     path = os.path.normpath( path ) # converts slashes to backslashes for windows
     
-    path = HydrusData.ConvertPortablePathToAbsPath( path )
+    path = HydrusPaths.ConvertPortablePathToAbsPath( path )
     
     return path
     
@@ -172,9 +165,15 @@ def GetFilePath( hash, mime = None ):
                 
             
         
-    else: path = GetExpectedFilePath( hash, mime )
+    else:
+        
+        path = GetExpectedFilePath( hash, mime )
+        
     
-    if path is None or not os.path.exists( path ): raise HydrusExceptions.NotFoundException( 'File not found!' )
+    if path is None or not os.path.exists( path ):
+        
+        raise HydrusExceptions.NotFoundException( 'File not found!' )
+        
     
     return path
     
@@ -199,7 +198,10 @@ def GetThumbnailPath( hash, full_size = True ):
     
     if not os.path.exists( path ):
         
-        if full_size: raise HydrusExceptions.NotFoundException( 'Thumbnail not found!' )
+        if full_size:
+            
+            raise HydrusExceptions.NotFoundException( 'Thumbnail not found!' )
+            
         else:
             
             full_size_path = GetThumbnailPath( hash, True )
@@ -216,7 +218,10 @@ def GetThumbnailPath( hash, full_size = True ):
                 
                 thumbnail_resized = HydrusFileHandling.GenerateThumbnail( full_size_path, thumbnail_dimensions )
                 
-                with open( path, 'wb' ) as f: f.write( thumbnail_resized )
+                with open( path, 'wb' ) as f:
+                    
+                    f.write( thumbnail_resized )
+                    
                 
             
         
@@ -269,6 +274,21 @@ def IterateAllFilePaths():
             
         
 
+def IterateAllThumbnailHashes():
+    
+    for path in IterateAllThumbnailPaths():
+        
+        ( base, filename ) = os.path.split( path )
+        
+        if not filename.endswith( '_resized' ):
+            
+            try: hash = filename.decode( 'hex' )
+            except TypeError: continue
+            
+            yield hash
+            
+        
+    
 def IterateAllThumbnailPaths():
     
     hex_chars = '0123456789abcdef'
diff --git a/include/ClientGUI.py b/include/ClientGUI.py
index 3ce3564e..58f1cbb8 100755
--- a/include/ClientGUI.py
+++ b/include/ClientGUI.py
@@ -1026,11 +1026,14 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
             menu.AppendMenu( wx.ID_NONE, p( 'Links' ), links )
             
             db_profile_mode_id = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'db_profile_mode' )
+            pubsub_profile_mode_id = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'pubsub_profile_mode' )
             special_debug_mode_id = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'special_debug_mode' )
             
             debug = wx.Menu()
             debug.AppendCheckItem( db_profile_mode_id, p( '&DB Profile Mode' ) )
             debug.Check( db_profile_mode_id, HydrusGlobals.db_profile_mode )
+            debug.AppendCheckItem( pubsub_profile_mode_id, p( '&PubSub Profile Mode' ) )
+            debug.Check( pubsub_profile_mode_id, HydrusGlobals.pubsub_profile_mode )
             debug.AppendCheckItem( special_debug_mode_id, p( '&Special Debug Mode' ) )
             debug.Check( special_debug_mode_id, HydrusGlobals.special_debug_mode )
             debug.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'force_idle' ), p( 'Force Idle Mode' ) )
@@ -1293,7 +1296,7 @@ class FrameGUI( ClientGUICommon.FrameThatResizes ):
             initial_media_results = []
             
         
-        page = ClientGUIPages.Page( self._notebook, management_controller, initial_media_results )
+        page = ClientGUIPages.Page( self._notebook, self._controller, management_controller, initial_media_results )
         
         self._notebook.AddPage( page, page_name, select = True )
         
@@ -1860,6 +1863,10 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
                         
                         job_key.SetVariable( 'popup_text_1', service.GetName() + ' was busy. please try again in a few minutes' )
                         
+                        job_key.DeleteVariable( 'popup_gauge_1' )
+                        
+                        job_key.Cancel()
+                        
                         return
                         
                     
@@ -1921,6 +1928,10 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
                         
                         job_key.SetVariable( 'popup_text_1', service.GetName() + ' was busy. please try again in a few minutes' )
                         
+                        job_key.DeleteVariable( 'popup_gauge_1' )
+                        
+                        job_key.Cancel()
+                        
                         return
                         
                     
@@ -1932,12 +1943,15 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             
         except Exception as e:
             
+            job_key.SetVariable( 'popup_text_1', service.GetName() + ' error' )
+            
+            job_key.DeleteVariable( 'popup_gauge_1' )
+            
             job_key.Cancel()
             
             raise
             
         
-        job_key.DeleteVariable( 'popup_gauge_1' )
         job_key.SetVariable( 'popup_text_1', prefix + 'upload done!' )
         
         HydrusData.Print( job_key.ToString() )
@@ -2139,6 +2153,10 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             elif command == 'pause_subs_sync': self._PauseSync( 'subs' )
             elif command == 'petitions': self._NewPagePetitions( data )
             elif command == 'post_news': self._PostNews( data )
+            elif command == 'pubsub_profile_mode':
+                
+                HydrusGlobals.pubsub_profile_mode = not HydrusGlobals.pubsub_profile_mode
+                
             elif command == 'redo': self._controller.pub( 'redo' )
             elif command == 'refresh':
                 
@@ -2193,9 +2211,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
             menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'tab_menu_close_page' ), 'close page' )
             menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'tab_menu_rename_page' ), 'rename page' )
             
-            self.PopupMenu( menu )
-            
-            wx.CallAfter( menu.Destroy )
+            self._controller.PopupMenu( self, menu )
             
         
     
diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py
index f5f6348d..4bc01d42 100755
--- a/include/ClientGUICanvas.py
+++ b/include/ClientGUICanvas.py
@@ -833,10 +833,13 @@ class Canvas( object ):
         
     
     def _ManageRatings( self ):
-        
-        if self._current_media is not None:
+    
+        if len( HydrusGlobals.client_controller.GetServicesManager().GetServices( HC.RATINGS_SERVICES ) ) > 0:
             
-            with ClientGUIDialogsManage.DialogManageRatings( self, ( self._current_display_media, ) ) as dlg: dlg.ShowModal()
+            if self._current_media is not None:
+                
+                with ClientGUIDialogsManage.DialogManageRatings( self, ( self._current_display_media, ) ) as dlg: dlg.ShowModal()
+                
             
         
     
@@ -1380,7 +1383,7 @@ class CanvasWithDetails( Canvas ):
             
             services_manager = HydrusGlobals.client_controller.GetServicesManager()
             
-            like_services = services_manager.GetServices( ( HC.LOCAL_RATING_LIKE, ) )
+            like_services = services_manager.GetServices( ( HC.LOCAL_RATING_LIKE, ), randomised = False )
             
             like_services.reverse()
             
@@ -1400,7 +1403,7 @@ class CanvasWithDetails( Canvas ):
             if len( like_services ) > 0: current_y += 20
             
             
-            numerical_services = services_manager.GetServices( ( HC.LOCAL_RATING_NUMERICAL, ) )
+            numerical_services = services_manager.GetServices( ( HC.LOCAL_RATING_NUMERICAL, ), randomised = False )
             
             for numerical_service in numerical_services:
                 
@@ -1580,11 +1583,7 @@ class CanvasPanel( Canvas, wx.Window ):
             
             menu.AppendMenu( CC.ID_NULL, 'share', share_menu )
             
-            self.PopupMenu( menu )
-            
-            self._menu_open = False
-            
-            wx.CallAfter( menu.Destroy )
+            HydrusGlobals.client_controller.PopupMenu( self, menu )
             
             event.Skip()
             
@@ -1630,8 +1629,6 @@ class CanvasFullscreenMediaList( ClientMedia.ListeningMediaList, CanvasWithDetai
         
         self._page_key = page_key
         
-        self._menu_open = False
-        
         self._just_started = True
         
         self.Show( True )
@@ -1946,7 +1943,7 @@ class CanvasFullscreenMediaList( ClientMedia.ListeningMediaList, CanvasWithDetai
                 return
                 
             
-            if self._menu_open:
+            if HydrusGlobals.client_controller.MenuIsOpen():
                 
                 self._timer_cursor_hide.Start( 800, wx.TIMER_ONE_SHOT )
                 
@@ -2652,25 +2649,19 @@ class CanvasFullscreenMediaListBrowser( CanvasFullscreenMediaListNavigable ):
         if self.IsFullScreen(): menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'fullscreen_switch' ), 'exit fullscreen' )
         else: menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'fullscreen_switch' ), 'go fullscreen' )
         
-        self._menu_open = True
-        
         if self._timer_slideshow.IsRunning():
             
             self._timer_slideshow.Stop()
             
-            self.PopupMenu( menu )
+            HydrusGlobals.client_controller.PopupMenu( self, menu )
             
             self._timer_slideshow.Start()
             
         else:
             
-            self.PopupMenu( menu )
+            HydrusGlobals.client_controller.PopupMenu( self, menu )
             
         
-        self._menu_open = False
-        
-        wx.CallAfter( menu.Destroy )
-        
         event.Skip()
         
     
@@ -3063,13 +3054,7 @@ class CanvasFullscreenMediaListCustomFilter( CanvasFullscreenMediaListNavigable
         if self.IsFullScreen(): menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'fullscreen_switch' ), 'exit fullscreen' )
         else: menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'fullscreen_switch' ), 'go fullscreen' )
         
-        self._menu_open = True
-        
-        self.PopupMenu( menu )
-        
-        self._menu_open = False
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
         event.Skip()
         
diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py
index 194fdf00..e27f7994 100755
--- a/include/ClientGUICommon.py
+++ b/include/ClientGUICommon.py
@@ -32,6 +32,44 @@ ID_TIMER_DROPDOWN_HIDE = wx.NewId()
 ID_TIMER_AC_LAG = wx.NewId()
 ID_TIMER_POPUP = wx.NewId()
 
+def FlushOutPredicates( parent, predicates ):
+    
+    good_predicates = []
+    
+    for predicate in predicates:
+        
+        predicate = predicate.GetCountlessCopy()
+        
+        ( predicate_type, value, inclusive ) = predicate.GetInfo()
+        
+        if value is None and predicate_type in [ HC.PREDICATE_TYPE_SYSTEM_NUM_TAGS, HC.PREDICATE_TYPE_SYSTEM_LIMIT, HC.PREDICATE_TYPE_SYSTEM_SIZE, HC.PREDICATE_TYPE_SYSTEM_DIMENSIONS, HC.PREDICATE_TYPE_SYSTEM_AGE, HC.PREDICATE_TYPE_SYSTEM_HASH, HC.PREDICATE_TYPE_SYSTEM_DURATION, HC.PREDICATE_TYPE_SYSTEM_NUM_WORDS, HC.PREDICATE_TYPE_SYSTEM_MIME, HC.PREDICATE_TYPE_SYSTEM_RATING, HC.PREDICATE_TYPE_SYSTEM_SIMILAR_TO, HC.PREDICATE_TYPE_SYSTEM_FILE_SERVICE ]:
+            
+            import ClientGUIDialogs
+            
+            with ClientGUIDialogs.DialogInputFileSystemPredicates( parent, predicate_type ) as dlg:
+                
+                if dlg.ShowModal() == wx.ID_OK:
+                    
+                    good_predicates.extend( dlg.GetPredicates() )
+                    
+                else:
+                    
+                    continue
+                    
+                
+            
+        elif predicate_type == HC.PREDICATE_TYPE_SYSTEM_UNTAGGED:
+            
+            good_predicates.append( ClientData.Predicate( HC.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '=', 0 ) ) )
+            
+        else:
+            
+            good_predicates.append( predicate )
+            
+        
+    
+    return good_predicates
+    
 def IsWXAncestor( child, ancestor ):
     
     parent = child
@@ -108,6 +146,7 @@ class AutoCompleteDropdown( wx.Panel ):
         wx.Panel.__init__( self, parent )
         
         self._last_search_text = ''
+        self._next_updatelist_is_probably_fast = False
         
         tlp = self.GetTopLevelParent()
         
@@ -463,7 +502,7 @@ class AutoCompleteDropdown( wx.Panel ):
         
         ( char_limit, long_wait, short_wait ) = HC.options[ 'ac_timings' ]
         
-        if num_chars == 0: self._UpdateList()
+        if num_chars == 0 or self._next_updatelist_is_probably_fast: self._UpdateList()
         elif num_chars < char_limit: self._lag_timer.Start( long_wait, wx.TIMER_ONE_SHOT )
         else: self._lag_timer.Start( short_wait, wx.TIMER_ONE_SHOT )
         
@@ -524,6 +563,8 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
         self._current_namespace = ''
         self._current_matches = []
         
+        self._cached_results = []
+        
         self._file_service_key = file_service_key
         self._tag_service_key = tag_service_key
         
@@ -594,9 +635,7 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
         
         for service in services: menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'change_file_repository', service.GetServiceKey() ), service.GetName() )
         
-        self._file_repo_button.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self._file_repo_button, menu )
         
     
     def EventMenu( self, event ):
@@ -636,9 +675,7 @@ class AutoCompleteDropdownTags( AutoCompleteDropdown ):
         
         for service in services: menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'change_tag_repository', service.GetServiceKey() ), service.GetName() )
         
-        self._tag_repo_button.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self._tag_repo_button, menu )
         
     
 class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
@@ -693,41 +730,7 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
             self._text_ctrl.SetValue( '' )
             
         
-        entry_predicates = []
-        
-        for predicate in predicates:
-            
-            predicate = predicate.GetCountlessCopy()
-            
-            ( predicate_type, value, inclusive ) = predicate.GetInfo()
-            
-            if predicate_type in [ HC.PREDICATE_TYPE_SYSTEM_NUM_TAGS, HC.PREDICATE_TYPE_SYSTEM_LIMIT, HC.PREDICATE_TYPE_SYSTEM_SIZE, HC.PREDICATE_TYPE_SYSTEM_DIMENSIONS, HC.PREDICATE_TYPE_SYSTEM_AGE, HC.PREDICATE_TYPE_SYSTEM_HASH, HC.PREDICATE_TYPE_SYSTEM_DURATION, HC.PREDICATE_TYPE_SYSTEM_NUM_WORDS, HC.PREDICATE_TYPE_SYSTEM_MIME, HC.PREDICATE_TYPE_SYSTEM_RATING, HC.PREDICATE_TYPE_SYSTEM_SIMILAR_TO, HC.PREDICATE_TYPE_SYSTEM_FILE_SERVICE ]:
-                
-                import ClientGUIDialogs
-                
-                with ClientGUIDialogs.DialogInputFileSystemPredicates( self, predicate_type ) as dlg:
-                    
-                    if dlg.ShowModal() == wx.ID_OK:
-                        
-                        entry_predicates.extend( dlg.GetPredicates() )
-                        
-                    else:
-                        
-                        return
-                        
-                    
-                
-            elif predicate_type == HC.PREDICATE_TYPE_SYSTEM_UNTAGGED:
-                
-                entry_predicates.append( ClientData.Predicate( HC.PREDICATE_TYPE_SYSTEM_NUM_TAGS, ( '=', 0 ) ) )
-                
-            else:
-                
-                entry_predicates.append( predicate )
-                
-            
-        
-        HydrusGlobals.client_controller.pub( 'enter_predicates', self._page_key, entry_predicates )
+        HydrusGlobals.client_controller.pub( 'enter_predicates', self._page_key, predicates )
         
     
     def _BroadcastCurrentText( self ):
@@ -786,6 +789,8 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
     
     def _GenerateMatches( self ):
         
+        self._next_updatelist_is_probably_fast = False
+        
         num_autocomplete_chars = HC.options[ 'num_autocomplete_chars' ]
         
         ( inclusive, search_text, entry_predicate ) = self._ParseSearchText()
@@ -808,7 +813,7 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
             
             if ':' in search_text:
                 
-                ( namespace, half_complete_tag ) = search_text.split( ':' )
+                ( namespace, half_complete_tag ) = search_text.split( ':', 1 )
                 
                 if namespace != self._current_namespace:
                     
@@ -853,6 +858,8 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
                         
                         predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, tag = search_text, include_current = self._include_current, include_pending = self._include_pending, add_namespaceless = True )
                         
+                        predicates = ClientSearch.SortPredicates( predicates, collapse_siblings = True )
+                        
                     else:
                         
                         if must_do_a_search or self._cache_text == '' or not search_text.startswith( self._cache_text ):
@@ -861,9 +868,13 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
                             
                             self._cached_results = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, half_complete_tag = search_text, include_current = self._include_current, include_pending = self._include_pending, add_namespaceless = True )
                             
+                            self._cached_results = ClientSearch.SortPredicates( self._cached_results, collapse_siblings = False )
+                            
                         
                         predicates = self._cached_results
                         
+                        self._next_updatelist_is_probably_fast = True
+                        
                     
                 else:
                     
@@ -897,8 +908,10 @@ class AutoCompleteDropdownTagsRead( AutoCompleteDropdownTags ):
                     
                     predicates = [ ClientData.Predicate( HC.PREDICATE_TYPE_TAG, tag, inclusive = inclusive, counts = { HC.CURRENT : current_tags_to_count[ tag ], HC.PENDING : pending_tags_to_count[ tag ] } ) for tag in tags_to_do ]
                     
-                
-                predicates = ClientSearch.SortPredicates( predicates, collapse_siblings = True )
+                    predicates = ClientSearch.SortPredicates( predicates, collapse_siblings = True )
+                    
+                    self._next_updatelist_is_probably_fast = True
+                    
                 
                 matches = ClientSearch.FilterPredicates( search_text, predicates )
                 
@@ -1095,6 +1108,8 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
     
     def _GenerateMatches( self ):
         
+        self._next_updatelist_is_probably_fast = False
+        
         num_autocomplete_chars = HC.options[ 'num_autocomplete_chars' ]
         
         ( search_text, entry_predicate, sibling_predicate ) = self._ParseSearchText()
@@ -1112,7 +1127,7 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
             
             if ':' in search_text:
                 
-                ( namespace, other_half ) = search_text.split( ':' )
+                ( namespace, other_half ) = search_text.split( ':', 1 )
                 
                 if other_half != '' and namespace != self._current_namespace:
                     
@@ -1129,6 +1144,8 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
                 
                 predicates = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, tag = search_text, add_namespaceless = False )
                 
+                predicates = ClientSearch.SortPredicates( predicates, collapse_siblings = False )
+                
             else:
                 
                 if must_do_a_search or self._cache_text == '' or not half_complete_tag.startswith( self._cache_text ):
@@ -1137,11 +1154,13 @@ class AutoCompleteDropdownTagsWrite( AutoCompleteDropdownTags ):
                     
                     self._cached_results = HydrusGlobals.client_controller.Read( 'autocomplete_predicates', file_service_key = self._file_service_key, tag_service_key = self._tag_service_key, half_complete_tag = search_text, add_namespaceless = False )
                     
+                    self._cached_results = ClientSearch.SortPredicates( self._cached_results, collapse_siblings = False )
+                    
                 
                 predicates = self._cached_results
                 
-            
-            predicates = ClientSearch.SortPredicates( predicates, collapse_siblings = False )
+                self._next_updatelist_is_probably_fast = True
+                
             
             matches = ClientSearch.FilterPredicates( half_complete_tag, predicates, service_key = self._tag_service_key, expand_parents = self._expand_parents )
             
@@ -1598,9 +1617,7 @@ class ExportPatternButton( wx.Button ):
         
         menu.Append( self.ID_TAG, 'a particular tag, if the file has it - (...)' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
 class FitResistantStaticText( wx.StaticText ):
@@ -2432,31 +2449,34 @@ class ListBox( wx.ScrolledWindow ):
                 
                 hit_index = None
                 
-                if key_code in ( wx.WXK_HOME, wx.WXK_NUMPAD_HOME ):
+                if len( self._ordered_strings ) > 0:
                     
-                    hit_index = 0
-                    
-                elif key_code in ( wx.WXK_END, wx.WXK_NUMPAD_END ):
-                    
-                    hit_index = len( self._ordered_strings ) - 1
-                    
-                elif self._last_hit_index is not None:
-                    
-                    if key_code in ( wx.WXK_UP, wx.WXK_NUMPAD_UP ):
+                    if key_code in ( wx.WXK_HOME, wx.WXK_NUMPAD_HOME ):
                         
-                        hit_index = self._last_hit_index - 1
+                        hit_index = 0
                         
-                    elif key_code in ( wx.WXK_DOWN, wx.WXK_NUMPAD_DOWN ):
+                    elif key_code in ( wx.WXK_END, wx.WXK_NUMPAD_END ):
                         
-                        hit_index = self._last_hit_index + 1
+                        hit_index = len( self._ordered_strings ) - 1
                         
-                    elif key_code in ( wx.WXK_PAGEUP, wx.WXK_NUMPAD_PAGEUP ):
+                    elif self._last_hit_index is not None:
                         
-                        hit_index = max( 0, self._last_hit_index - self._num_rows_per_page )
-                        
-                    elif key_code in ( wx.WXK_PAGEDOWN, wx.WXK_NUMPAD_PAGEDOWN ):
-                        
-                        hit_index = min( len( self._ordered_strings ) - 1, self._last_hit_index + self._num_rows_per_page )
+                        if key_code in ( wx.WXK_UP, wx.WXK_NUMPAD_UP ):
+                            
+                            hit_index = self._last_hit_index - 1
+                            
+                        elif key_code in ( wx.WXK_DOWN, wx.WXK_NUMPAD_DOWN ):
+                            
+                            hit_index = self._last_hit_index + 1
+                            
+                        elif key_code in ( wx.WXK_PAGEUP, wx.WXK_NUMPAD_PAGEUP ):
+                            
+                            hit_index = max( 0, self._last_hit_index - self._num_rows_per_page )
+                            
+                        elif key_code in ( wx.WXK_PAGEDOWN, wx.WXK_NUMPAD_PAGEDOWN ):
+                            
+                            hit_index = min( len( self._ordered_strings ) - 1, self._last_hit_index + self._num_rows_per_page )
+                            
                         
                     
                 
@@ -2596,6 +2616,8 @@ class ListBoxTags( ListBox ):
                 
             
         
+        predicates = FlushOutPredicates( self, predicates )
+        
         if len( predicates ) > 0:
             
             HydrusGlobals.client_controller.pub( 'new_page_query', CC.LOCAL_FILE_SERVICE_KEY, initial_predicates = predicates )
@@ -2779,9 +2801,7 @@ class ListBoxTags( ListBox ):
                     
                 
             
-            self.PopupMenu( menu )
-            
-            wx.CallAfter( menu.Destroy )
+            HydrusGlobals.client_controller.PopupMenu( self, menu )
             
         
         event.Skip()
@@ -2807,11 +2827,13 @@ class ListBoxTagsAutocompleteDropdown( ListBoxTags ):
     
     def _Activate( self ):
         
-        callable_terms = [ term for term in self._selected_terms if term.GetType() != HC.PREDICATE_TYPE_PARENT ]
+        predicates = [ term for term in self._selected_terms if term.GetType() != HC.PREDICATE_TYPE_PARENT ]
         
-        if len( callable_terms ) > 0:
+        predicates = FlushOutPredicates( self, predicates )
+        
+        if len( predicates ) > 0:
             
-            self._callable( callable_terms )
+            self._callable( predicates )
             
         
     
@@ -2901,19 +2923,22 @@ class ListBoxTagsAutocompleteDropdown( ListBoxTags ):
         
         hit_index = None
         
-        if key_code in ( wx.WXK_END, wx.WXK_NUMPAD_END ):
+        if len( self._ordered_strings ) > 0:
             
-            hit_index = len( self._ordered_strings ) - 1
-            
-        elif self._last_hit_index is not None:
-            
-            if key_code in ( wx.WXK_DOWN, wx.WXK_NUMPAD_DOWN ):
+            if key_code in ( wx.WXK_END, wx.WXK_NUMPAD_END ):
                 
-                hit_index = self._last_hit_index + 1
+                hit_index = len( self._ordered_strings ) - 1
                 
-            elif key_code in ( wx.WXK_PAGEDOWN, wx.WXK_NUMPAD_PAGEDOWN ):
+            elif self._last_hit_index is not None:
                 
-                hit_index = min( len( self._ordered_strings ) - 1, self._last_hit_index + self._num_rows_per_page )
+                if key_code in ( wx.WXK_DOWN, wx.WXK_NUMPAD_DOWN ):
+                    
+                    hit_index = self._last_hit_index + 1
+                    
+                elif key_code in ( wx.WXK_PAGEDOWN, wx.WXK_NUMPAD_PAGEDOWN ):
+                    
+                    hit_index = min( len( self._ordered_strings ) - 1, self._last_hit_index + self._num_rows_per_page )
+                    
                 
             
         
@@ -3015,7 +3040,7 @@ class ListBoxTagsCensorship( ListBoxTags ):
             
             for tag in tags:
                 
-                self._RemoveTag( self._selected_terms )
+                self._RemoveTag( tag )
                 
             
             self._TextsHaveChanged()
@@ -5079,9 +5104,7 @@ class RegexButton( wx.Button ):
         
         menu.AppendMenu( -1, 'favourites', submenu )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def EventMenu( self, event ):
@@ -5462,9 +5485,7 @@ class SeedCacheControl( SaneListCtrl ):
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'set_seed_skipped' ), 'skip' )
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'set_seed_unknown' ), 'try again' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def NotifySeedAdded( self, seed ):
diff --git a/include/ClientGUIDialogsManage.py b/include/ClientGUIDialogsManage.py
index 2f2f20a8..2f607b23 100644
--- a/include/ClientGUIDialogsManage.py
+++ b/include/ClientGUIDialogsManage.py
@@ -3813,7 +3813,7 @@ class DialogManageOptions( ClientGUIDialogs.Dialog ):
             
             if HC.options[ 'export_path' ] is not None:
                 
-                abs_path = HydrusData.ConvertPortablePathToAbsPath( HC.options[ 'export_path' ] )
+                abs_path = HydrusPaths.ConvertPortablePathToAbsPath( HC.options[ 'export_path' ] )
                 
                 if abs_path is not None: self._export_location.SetPath( abs_path )
                 
@@ -4805,8 +4805,8 @@ class DialogManageRatings( ClientGUIDialogs.Dialog ):
         
         #
         
-        like_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, ) )
-        numerical_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_NUMERICAL, ) )
+        like_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, ), randomised = False )
+        numerical_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_NUMERICAL, ), randomised = False )
         
         self._panels = []
         
diff --git a/include/ClientGUIHoverFrames.py b/include/ClientGUIHoverFrames.py
index 559a25d4..1cc82028 100644
--- a/include/ClientGUIHoverFrames.py
+++ b/include/ClientGUIHoverFrames.py
@@ -96,13 +96,18 @@ class FullscreenHoverFrame( wx.Frame ):
                 in_x = my_ideal_x <= mouse_x and mouse_x <= my_ideal_x + my_ideal_width
                 in_y = my_ideal_y <= mouse_y and mouse_y <= my_ideal_y + my_ideal_height
                 
-                no_dialogs_open = True
+                menu_open = HydrusGlobals.client_controller.MenuIsOpen()
+                
+                dialog_open = False
                 
                 tlps = wx.GetTopLevelWindows()
                 
                 for tlp in tlps:
                     
-                    if isinstance( tlp, wx.Dialog ): no_dialogs_open = False
+                    if isinstance( tlp, wx.Dialog ):
+                        
+                        dialog_open = True
+                        
                     
                 
                 mime = self._current_media.GetMime()
@@ -128,8 +133,8 @@ class FullscreenHoverFrame( wx.Frame ):
                     tlp = tlp.GetParent()
                     
                 
-                ready_to_show = in_position and not mouse_is_over_something_important and no_dialogs_open and my_parent_in_focus_tree
-                ready_to_hide = not in_position or not no_dialogs_open or not my_parent_in_focus_tree
+                ready_to_show = in_position and not mouse_is_over_something_important and my_parent_in_focus_tree and not dialog_open and not menu_open
+                ready_to_hide = not menu_open and ( not in_position or dialog_open or not my_parent_in_focus_tree )
                 
                 if ready_to_show:
                     
@@ -137,7 +142,10 @@ class FullscreenHoverFrame( wx.Frame ):
                     
                     self.Show()
                     
-                elif ready_to_hide: self.Hide()
+                elif ready_to_hide:
+                    
+                    self.Hide()
+                    
                 
             
         except wx.PyDeadObjectError:
@@ -493,7 +501,7 @@ class FullscreenHoverFrameRatings( FullscreenHoverFrame ):
         
         like_hbox.AddF( ( 16, 16 ), CC.FLAGS_EXPAND_BOTH_WAYS )
         
-        like_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, ) )
+        like_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_LIKE, ), randomised = False )
         
         for service in like_services:
             
@@ -510,7 +518,7 @@ class FullscreenHoverFrameRatings( FullscreenHoverFrame ):
         vbox.AddF( self._file_repos, CC.FLAGS_EXPAND_BOTH_WAYS )
         vbox.AddF( like_hbox, CC.FLAGS_EXPAND_SIZER_BOTH_WAYS )
         
-        numerical_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_NUMERICAL, ) )
+        numerical_services = HydrusGlobals.client_controller.GetServicesManager().GetServices( ( HC.LOCAL_RATING_NUMERICAL, ), randomised = False )
         
         for service in numerical_services:
             
diff --git a/include/ClientGUIMedia.py b/include/ClientGUIMedia.py
index 13f50ff2..f83411de 100755
--- a/include/ClientGUIMedia.py
+++ b/include/ClientGUIMedia.py
@@ -1879,7 +1879,10 @@ class MediaPanelThumbnails( MediaPanel ):
             
             if len( self._sorted_media ) > 0:
                 
-                if menu.GetMenuItemCount() > 0: menu.AppendSeparator()
+                if menu.GetMenuItemCount() > 0:
+                    
+                    menu.AppendSeparator()
+                    
                 
                 select_menu = wx.Menu()
                 
@@ -2201,10 +2204,10 @@ class MediaPanelThumbnails( MediaPanel ):
                 
                 copy_menu = wx.Menu()
                 
-                copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_files' ), copy_phrase )
-                
                 if selection_has_local_file_service:
                     
+                    copy_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_files' ), copy_phrase )
+                    
                     copy_hash_menu = wx.Menu()
                     
                     copy_hash_menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'copy_hash', 'sha256' ) , 'sha256 (hydrus default)' )
@@ -2309,9 +2312,7 @@ class MediaPanelThumbnails( MediaPanel ):
                 
             
         
-        if menu.GetMenuItemCount() > 0: self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
         event.Skip()
         
diff --git a/include/ClientGUIMessages.py b/include/ClientGUIMessages.py
index 7a98fda9..f393a10d 100755
--- a/include/ClientGUIMessages.py
+++ b/include/ClientGUIMessages.py
@@ -268,9 +268,7 @@ class ConversationsListCtrl( wx.ListCtrl, ListCtrlAutoWidthMixin, ColumnSorterMi
         menu.AppendSeparator()
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'delete' ), 'delete' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def GetListCtrl( self ): return self
@@ -641,9 +639,7 @@ class DestinationPanel( wx.Panel ):
         
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'read' ), 'read' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def EventRetryMenu( self, event ):
@@ -652,9 +648,7 @@ class DestinationPanel( wx.Panel ):
         
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'retry' ), 'retry' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def EventUnreadMenu( self, event ):
@@ -663,9 +657,7 @@ class DestinationPanel( wx.Panel ):
         
         menu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'unread' ), 'unread' )
         
-        self.PopupMenu( menu )
-        
-        wx.CallAfter( menu.Destroy )
+        HydrusGlobals.client_controller.PopupMenu( self, menu )
         
     
     def SetStatus( self, status ):
diff --git a/include/ClientGUIPages.py b/include/ClientGUIPages.py
index d04c56db..55b79086 100755
--- a/include/ClientGUIPages.py
+++ b/include/ClientGUIPages.py
@@ -19,12 +19,14 @@ import HydrusGlobals
 
 class Page( wx.SplitterWindow ):
     
-    def __init__( self, parent, management_controller, initial_media_results ):
+    def __init__( self, parent, controller, management_controller, initial_media_results ):
         
         wx.SplitterWindow.__init__( self, parent )
         
         self._page_key = HydrusData.GenerateKey()
         
+        self._controller = controller
+        
         self._management_controller = management_controller
         
         self._management_controller.SetKey( 'page', self._page_key )
@@ -60,8 +62,8 @@ class Page( wx.SplitterWindow ):
             wx.CallAfter( self._search_preview_split.Unsplit, self._preview_panel )
             
         
-        HydrusGlobals.client_controller.sub( self, 'SetPrettyStatus', 'new_page_status' )
-        HydrusGlobals.client_controller.sub( self, 'SwapMediaPanel', 'swap_media_panel' )
+        self._controller.sub( self, 'SetPrettyStatus', 'new_page_status' )
+        self._controller.sub( self, 'SwapMediaPanel', 'swap_media_panel' )
         
     
     def CleanBeforeDestroy( self ): self._management_panel.CleanBeforeDestroy()
@@ -70,14 +72,14 @@ class Page( wx.SplitterWindow ):
         
         self._search_preview_split.Unsplit( self._preview_panel )
         
-        HydrusGlobals.client_controller.pub( 'set_focus', self._page_key, None )
+        self._controller.pub( 'set_focus', self._page_key, None )
         
     
     def EventUnsplit( self, event ):
         
         self.Unsplit( self._search_preview_split )
         
-        HydrusGlobals.client_controller.pub( 'set_focus', self._page_key, None )
+        self._controller.pub( 'set_focus', self._page_key, None )
         
     
     def GetManagementController( self ):
@@ -126,16 +128,25 @@ class Page( wx.SplitterWindow ):
         return ( x, y )
         
     
-    def PageHidden( self ): HydrusGlobals.client_controller.pub( 'page_hidden', self._page_key )
+    def PageHidden( self ):
+        
+        self._controller.pub( 'page_hidden', self._page_key )
+        
     
-    def PageShown( self ): HydrusGlobals.client_controller.pub( 'page_shown', self._page_key )
+    def PageShown( self ):
+        
+        self._controller.pub( 'page_shown', self._page_key )
+        
     
     def PrepareToHide( self ):
         
-        HydrusGlobals.client_controller.pub( 'set_focus', self._page_key, None )
+        self._controller.pub( 'set_focus', self._page_key, None )
         
     
-    def RefreshQuery( self ): HydrusGlobals.client_controller.pub( 'refresh_query', self._page_key )
+    def RefreshQuery( self ):
+        
+        self._controller.pub( 'refresh_query', self._page_key )
+        
     
     def ShowHideSplit( self ):
         
@@ -143,7 +154,7 @@ class Page( wx.SplitterWindow ):
             
             self.Unsplit( self._search_preview_split )
             
-            HydrusGlobals.client_controller.pub( 'set_focus', self._page_key, None )
+            self._controller.pub( 'set_focus', self._page_key, None )
             
         else:
             
@@ -161,13 +172,19 @@ class Page( wx.SplitterWindow ):
             
             self._pretty_status = status
             
-            HydrusGlobals.client_controller.pub( 'refresh_status' )
+            self._controller.pub( 'refresh_status' )
             
         
     
-    def SetSearchFocus( self ): HydrusGlobals.client_controller.pub( 'set_search_focus', self._page_key )
+    def SetSearchFocus( self ):
+        
+        self._controller.pub( 'set_search_focus', self._page_key )
+        
     
-    def SetSynchronisedWait( self ): HydrusGlobals.client_controller.pub( 'synchronised_wait_switch', self._page_key )
+    def SetSynchronisedWait( self ):
+        
+        self._controller.pub( 'synchronised_wait_switch', self._page_key )
+        
     
     def SwapMediaPanel( self, page_key, new_panel ):
         
diff --git a/include/ClientRendering.py b/include/ClientRendering.py
index 19836ee5..ec80db90 100644
--- a/include/ClientRendering.py
+++ b/include/ClientRendering.py
@@ -187,14 +187,7 @@ class RasterContainerVideo( RasterContainer ):
             
         else:
             
-            try:
-                
-                self._frame_duration = ClientVideoHandling.GetVideoFrameDuration( self._path )
-                
-            except HydrusExceptions.CantRenderWithCVException:
-                
-                self._frame_duration = float( duration ) / num_frames
-                
+            self._frame_duration = float( duration ) / num_frames
             
             self._renderer = HydrusVideoHandling.VideoRendererFFMPEG( path, mime, duration, num_frames, target_resolution )
             
diff --git a/include/ClientVideoHandling.py b/include/ClientVideoHandling.py
index c2ec181d..09f1398f 100644
--- a/include/ClientVideoHandling.py
+++ b/include/ClientVideoHandling.py
@@ -48,7 +48,7 @@ def GetVideoFrameDuration( path ):
     
     fps = cv_video.get( CAP_PROP_FPS )
     
-    if fps == 0: raise HydrusExceptions.CantRenderWithCVException()
+    if fps in ( 0, 1000 ): raise HydrusExceptions.CantRenderWithCVException()
     
     return 1000.0 / fps
     
diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py
index 47bed022..5609f737 100755
--- a/include/HydrusConstants.py
+++ b/include/HydrusConstants.py
@@ -51,7 +51,7 @@ options = {}
 # Misc
 
 NETWORK_VERSION = 17
-SOFTWARE_VERSION = 182
+SOFTWARE_VERSION = 183
 
 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
 
@@ -291,7 +291,7 @@ NOISY_MIMES = tuple( [ APPLICATION_FLASH ] + list( AUDIO ) + list( VIDEO ) )
 
 ARCHIVES = ( APPLICATION_ZIP, APPLICATION_HYDRUS_ENCRYPTED_ZIP )
 
-MIMES_WITH_THUMBNAILS = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_WEBM, VIDEO_FLV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM )
+MIMES_WITH_THUMBNAILS = ( APPLICATION_FLASH, IMAGE_JPEG, IMAGE_PNG, IMAGE_GIF, IMAGE_BMP, VIDEO_FLV, VIDEO_MP4, VIDEO_WMV, VIDEO_MKV, VIDEO_WEBM )
 
 # mp3 header is complicated
 
diff --git a/include/HydrusDB.py b/include/HydrusDB.py
index e8295f24..9b212dcc 100644
--- a/include/HydrusDB.py
+++ b/include/HydrusDB.py
@@ -1,9 +1,11 @@
 import cProfile
+import cStringIO
 import HydrusConstants as HC
 import HydrusData
 import HydrusExceptions
 import HydrusGlobals
 import os
+import pstats
 import Queue
 import sqlite3
 import sys
@@ -266,11 +268,7 @@ class HydrusDB( object ):
                         
                         HydrusData.ShowText( 'Profiling ' + job.GetType() + ' ' + job.GetAction() )
                         
-                        profile = cProfile.Profile()
-                        
-                        profile.runctx( 'self._ProcessJob( job )', globals(), locals() )
-                        
-                        profile.print_stats( sort = 'tottime' )
+                        HydrusData.Profile( 'self._ProcessJob( job )', globals(), locals() )
                         
                     else:
                         
diff --git a/include/HydrusData.py b/include/HydrusData.py
index 220ad767..6902c0fe 100644
--- a/include/HydrusData.py
+++ b/include/HydrusData.py
@@ -1,13 +1,15 @@
 import bs4
 import collections
+import cProfile
+import cStringIO
 import HydrusConstants as HC
 import HydrusExceptions
 import HydrusGlobals
 import HydrusSerialisable
 import locale
 import os
+import pstats
 import psutil
-import send2trash
 import shutil
 import sqlite3
 import subprocess
@@ -251,16 +253,6 @@ def ConvertPixelsToInt( unit ):
     elif unit == 'kilopixels': return 1000
     elif unit == 'megapixels': return 1000000
     
-def ConvertPortablePathToAbsPath( portable_path ):
-    
-    if portable_path is None: return None
-    
-    if os.path.isabs( portable_path ): abs_path = portable_path
-    else: abs_path = os.path.normpath( os.path.join( HC.BASE_DIR, portable_path ) )
-    
-    if os.path.exists( abs_path ): return abs_path
-    else: return None
-    
 def ConvertPrettyStringsToUglyNamespaces( pretty_strings ):
     
     result = { s for s in pretty_strings if s != 'no namespace' }
@@ -570,20 +562,6 @@ def DebugPrint( debug_info ):
     sys.stdout.flush()
     sys.stderr.flush()
     
-def DeletePath( path ):
-    
-    if os.path.exists( path ):
-        
-        if os.path.isdir( path ):
-            
-            shutil.rmtree( path )
-            
-        else:
-            
-            os.remove( path )
-            
-        
-    
 def DeserialisePrettyTags( text ):
     
     text = text.replace( '\r', '' )
@@ -778,6 +756,42 @@ def Print( text ):
     
     print( ToByteString( text ) )
     
+def Profile( code, g, l ):
+    
+    profile = cProfile.Profile()
+    
+    profile.runctx( code, g, l )
+    
+    output = cStringIO.StringIO()
+    
+    stats = pstats.Stats( profile, stream = output )
+    
+    stats.strip_dirs()
+    
+    stats.sort_stats( 'tottime' )
+    
+    output.seek( 0 )
+    
+    output.write( 'Stats' )
+    output.write( os.linesep )
+    
+    stats.print_stats()
+    
+    output.seek( 0 )
+    
+    Print( output.read() )
+    
+    output.seek( 0 )
+    
+    output.write( 'Callers' )
+    output.write( os.linesep )
+    
+    stats.print_callers()
+    
+    output.seek( 0 )
+    
+    Print( output.read() )
+    
 def RecordRunningStart( instance ):
     
     path = os.path.join( HC.BASE_DIR, instance + '_running' )
@@ -802,47 +816,6 @@ def RecordRunningStart( instance ):
         f.write( ToByteString( record_string ) )
         
     
-def RecyclePath( path ):
-    
-    original_path = path
-    
-    if HC.PLATFORM_LINUX:
-        
-        # send2trash for Linux tries to do some Python3 str() stuff in prepping non-str paths for recycling
-        
-        if not isinstance( path, str ):
-            
-            try:
-                
-                path = path.encode( sys.getfilesystemencoding() )
-                
-            except:
-                
-                Print( 'Trying to prepare a file for recycling created this error:' )
-                traceback.print_exc()
-                
-                return
-                
-            
-        
-    
-    if os.path.exists( path ):
-        
-        try:
-            
-            send2trash.send2trash( path )
-            
-        except:
-            
-            Print( 'Trying to recycle a file created this error:' )
-            traceback.print_exc()
-            
-            Print( 'It has been fully deleted instead.' )
-            
-            DeletePath( original_path )
-            
-        
-    
 def ShowExceptionDefault( e ):
     
     if isinstance( e, HydrusExceptions.ShutdownException ):
diff --git a/include/HydrusFileHandling.py b/include/HydrusFileHandling.py
index 1208131d..7c104ad0 100644
--- a/include/HydrusFileHandling.py
+++ b/include/HydrusFileHandling.py
@@ -42,35 +42,63 @@ header_and_mime = [
     ( 0, '\x30\x26\xB2\x75\x8E\x66\xCF\x11\xA6\xD9\x00\xAA\x00\x62\xCE\x6C', HC.UNDETERMINED_WM )
     ]
 
+def SaveThumbnailToStream( pil_image, dimensions, f ):
+    
+    HydrusImageHandling.EfficientlyThumbnailPILImage( pil_image, dimensions )
+    
+    if pil_image.mode == 'P' and pil_image.info.has_key( 'transparency' ):
+        
+        pil_image.save( f, 'PNG', transparency = pil_image.info[ 'transparency' ] )
+        
+    elif pil_image.mode == 'RGBA':
+        
+        pil_image.save( f, 'PNG' )
+        
+    else:
+        
+        pil_image = pil_image.convert( 'RGB' )
+        
+        pil_image.save( f, 'JPEG', quality = 92 )
+        
+    
 def GenerateThumbnail( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS ):
     
     mime = GetMime( path )
     
+    f = cStringIO.StringIO()
+    
     if mime in HC.IMAGES:
         
         pil_image = HydrusImageHandling.GeneratePILImage( path )
         
-        HydrusImageHandling.EfficientlyThumbnailPILImage( pil_image, dimensions )
+        SaveThumbnailToStream( pil_image, dimensions, f )
         
-        f = cStringIO.StringIO()
+    elif mime == HC.APPLICATION_FLASH:
         
-        if pil_image.mode == 'P' and pil_image.info.has_key( 'transparency' ):
-            
-            pil_image.save( f, 'PNG', transparency = pil_image.info[ 'transparency' ] )
-            
-        elif pil_image.mode == 'RGBA': pil_image.save( f, 'PNG' )
-        else:
-            
-            pil_image = pil_image.convert( 'RGB' )
-            
-            pil_image.save( f, 'JPEG', quality=92 )
-            
+        ( os_file_handle, temp_path ) = HydrusPaths.GetTempPath()
         
-        f.seek( 0 )
-        
-        thumbnail = f.read()
-        
-        f.close()
+        try:
+            
+            HydrusFlashHandling.RenderPageToFile( path, temp_path, 1 )
+            
+            pil_image = HydrusImageHandling.GeneratePILImage( temp_path )
+            
+            SaveThumbnailToStream( pil_image, dimensions, f )
+            
+        except:
+            
+            flash_default_path = os.path.join( HC.STATIC_DIR, 'flash.png' )
+            
+            pil_image = HydrusImageHandling.GeneratePILImage( flash_default_path )
+            
+            SaveThumbnailToStream( pil_image, dimensions, f )
+            
+        finally:
+            
+            del pil_image
+            
+            HydrusPaths.CleanUpTempPath( os_file_handle, temp_path )
+            
         
     else:
         
@@ -84,17 +112,15 @@ def GenerateThumbnail( path, dimensions = HC.UNSCALED_THUMBNAIL_DIMENSIONS ):
         
         pil_image = HydrusImageHandling.GeneratePILImageFromNumpyImage( numpy_image )
         
-        f = cStringIO.StringIO()
-        
-        pil_image.save( f, 'JPEG', quality=92 )
-        
-        f.seek( 0 )
-        
-        thumbnail = f.read()
-        
-        f.close()
+        SaveThumbnailToStream( pil_image, dimensions, f )
         
     
+    f.seek( 0 )
+    
+    thumbnail = f.read()
+    
+    f.close()
+    
     return thumbnail
     
 def GetExtraHashesFromPath( path ):
diff --git a/include/HydrusFlashHandling.py b/include/HydrusFlashHandling.py
index f0895cfd..99d09999 100755
--- a/include/HydrusFlashHandling.py
+++ b/include/HydrusFlashHandling.py
@@ -1,7 +1,22 @@
 import hexagonitswfheader
 import HydrusConstants as HC
+import HydrusData
+import os
+import subprocess
 import traceback
 
+if HC.PLATFORM_LINUX:
+    
+    SWFRENDER_PATH = os.path.join( HC.BIN_DIR, 'swfrender_linux' )
+    
+elif HC.PLATFORM_OSX:
+    
+    SWFRENDER_PATH = os.path.join( HC.BIN_DIR, 'swfrender_osx' )
+    
+elif HC.PLATFORM_WINDOWS:
+    
+    SWFRENDER_PATH = os.path.join( HC.BIN_DIR, 'swfrender_win32.exe' )
+    
 # to all out there who write libraries:
 # hexagonit.swfheader is a perfect library. it is how you are supposed to do it.
 def GetFlashProperties( path ):
@@ -20,4 +35,14 @@ def GetFlashProperties( path ):
         
         return ( ( width, height ), duration, num_frames )
         
+    
+def RenderPageToFile( path, temp_path, page_index ):
+    
+    cmd = [ SWFRENDER_PATH, path, '-o', temp_path, '-p', str( page_index ) ]
+    
+    p = subprocess.Popen( cmd, startupinfo = HydrusData.GetSubprocessStartupInfo() )
+    
+    p.wait()
+    
+    p.communicate()
     
\ No newline at end of file
diff --git a/include/HydrusGlobals.py b/include/HydrusGlobals.py
index b1f9c135..0b565ace 100644
--- a/include/HydrusGlobals.py
+++ b/include/HydrusGlobals.py
@@ -11,5 +11,6 @@ is_first_start = False
 is_db_updated = False
 
 db_profile_mode = False
+pubsub_profile_mode = False
 special_debug_mode = False
 server_busy = False
diff --git a/include/HydrusImageHandling.py b/include/HydrusImageHandling.py
index d4eca53a..9eb62c4e 100755
--- a/include/HydrusImageHandling.py
+++ b/include/HydrusImageHandling.py
@@ -65,7 +65,10 @@ def EfficientlyThumbnailPILImage( pil_image, ( target_x, target_y ) ):
     #    if im_x > 2 * target_x or im_y > 2 * target_y: pil_image.thumbnail( ( 2 * target_x, 2 * target_y ), PILImage.NEAREST )
     #    
     
-    pil_image.thumbnail( ( target_x, target_y ), PILImage.ANTIALIAS )
+    if im_x > target_x or im_y > target_y:
+        
+        pil_image.thumbnail( ( target_x, target_y ), PILImage.ANTIALIAS )
+        
     
 def GeneratePILImage( path ):
     
diff --git a/include/HydrusPaths.py b/include/HydrusPaths.py
index 2227cdfb..50c6ed40 100644
--- a/include/HydrusPaths.py
+++ b/include/HydrusPaths.py
@@ -2,9 +2,13 @@ import gc
 import HydrusConstants as HC
 import HydrusData
 import os
+import send2trash
+import shutil
 import subprocess
+import sys
 import tempfile
 import threading
+import traceback
 
 def CleanUpTempPath( os_file_handle, temp_path ):
     
@@ -53,10 +57,34 @@ def ConvertAbsPathToPortablePath( abs_path ):
     try: return os.path.relpath( abs_path, HC.BASE_DIR )
     except: return abs_path
     
+def ConvertPortablePathToAbsPath( portable_path ):
+    
+    if portable_path is None: return None
+    
+    if os.path.isabs( portable_path ): abs_path = portable_path
+    else: abs_path = os.path.normpath( os.path.join( HC.BASE_DIR, portable_path ) )
+    
+    if os.path.exists( abs_path ): return abs_path
+    else: return None
+    
 def CopyFileLikeToFileLike( f_source, f_dest ):
     
     for block in ReadFileLikeAsBlocks( f_source ): f_dest.write( block )
     
+def DeletePath( path ):
+    
+    if os.path.exists( path ):
+        
+        if os.path.isdir( path ):
+            
+            shutil.rmtree( path )
+            
+        else:
+            
+            os.remove( path )
+            
+        
+    
 def GetTempFile(): return tempfile.TemporaryFile()
 def GetTempFileQuick(): return tempfile.SpooledTemporaryFile( max_size = 1024 * 1024 * 4 )
 def GetTempPath(): return tempfile.mkstemp( prefix = 'hydrus' )
@@ -129,4 +157,45 @@ def ReadFileLikeAsBlocks( f ):
         
         next_block = f.read( HC.READ_BLOCK_SIZE )
         
+    
+def RecyclePath( path ):
+    
+    original_path = path
+    
+    if HC.PLATFORM_LINUX:
+        
+        # send2trash for Linux tries to do some Python3 str() stuff in prepping non-str paths for recycling
+        
+        if not isinstance( path, str ):
+            
+            try:
+                
+                path = path.encode( sys.getfilesystemencoding() )
+                
+            except:
+                
+                HydrusData.Print( 'Trying to prepare a file for recycling created this error:' )
+                traceback.print_exc()
+                
+                return
+                
+            
+        
+    
+    if os.path.exists( path ):
+        
+        try:
+            
+            send2trash.send2trash( path )
+            
+        except:
+            
+            HydrusData.Print( 'Trying to recycle a file created this error:' )
+            traceback.print_exc()
+            
+            HydrusData.Print( 'It has been fully deleted instead.' )
+            
+            DeletePath( original_path )
+            
+        
     
\ No newline at end of file
diff --git a/include/HydrusPubSub.py b/include/HydrusPubSub.py
index 52d988d3..c51bb5d2 100755
--- a/include/HydrusPubSub.py
+++ b/include/HydrusPubSub.py
@@ -1,4 +1,5 @@
 import HydrusConstants as HC
+import HydrusData
 import Queue
 import threading
 import traceback
@@ -103,7 +104,25 @@ class HydrusPubSub( object ):
         
         for callable in callables:
             
-            callable( *args, **kwargs )
+            if HydrusGlobals.pubsub_profile_mode:
+                
+                text = 'Profiling ' + topic + ': ' + repr( callable )
+                
+                if topic == 'message':
+                    
+                    HydrusData.Print( text )
+                    
+                else:
+                    
+                    HydrusData.ShowText( text )
+                    
+                
+                HydrusData.Profile( 'callable( *args, **kwargs )', globals(), locals() )
+                
+            else:
+                
+                callable( *args, **kwargs )
+                
             
         
     
diff --git a/include/HydrusServerResources.py b/include/HydrusServerResources.py
index ff9c0d2f..cefb5fba 100644
--- a/include/HydrusServerResources.py
+++ b/include/HydrusServerResources.py
@@ -77,7 +77,7 @@ def ParseFileArguments( path ):
     if num_frames is not None: args[ 'num_frames' ] = num_frames
     if num_words is not None: args[ 'num_words' ] = num_words
     
-    if mime in HC.IMAGES:
+    if mime in HC.MIMES_WITH_THUMBNAILS:
         
         try: thumbnail = HydrusFileHandling.GenerateThumbnail( path )
         except: raise HydrusExceptions.ForbiddenException( 'Could not generate thumbnail from that file.' )
@@ -438,6 +438,11 @@ class HydrusResourceCommand( Resource ):
             
         
     
+    def _errbackDisconnected( self, failure, request_deferred ):
+        
+        request_deferred.cancel()
+        
+    
     def _errbackHandleEmergencyError( self, failure, request ):
         
         try: self._CleanUpTempFile( request )
@@ -449,7 +454,8 @@ class HydrusResourceCommand( Resource ):
         try: request.write( failure.getTraceback() )
         except: pass
         
-        request.finish()
+        try: request.finish()
+        except: pass
         
     
     def _errbackHandleProcessingError( self, failure, request ):
@@ -547,6 +553,8 @@ class HydrusResourceCommand( Resource ):
         
         reactor.callLater( 0, d.callback, request )
         
+        request.notifyFinish().addErrback( self._errbackDisconnected, d )
+        
         return NOT_DONE_YET
         
     
@@ -570,6 +578,8 @@ class HydrusResourceCommand( Resource ):
         
         reactor.callLater( 0, d.callback, request )
         
+        request.notifyFinish().addErrback( self._errbackDisconnected, d )
+        
         return NOT_DONE_YET
         
     
diff --git a/include/HydrusVideoHandling.py b/include/HydrusVideoHandling.py
index 398ad9e6..68c4f647 100755
--- a/include/HydrusVideoHandling.py
+++ b/include/HydrusVideoHandling.py
@@ -275,6 +275,7 @@ class VideoRendererFFMPEG( object ):
         else: self.depth = 3
         
         ( x, y ) = self._target_resolution
+        
         bufsize = self.depth * x * y
         
         self.process = None
diff --git a/include/ServerDB.py b/include/ServerDB.py
index 3814235b..099112b3 100755
--- a/include/ServerDB.py
+++ b/include/ServerDB.py
@@ -4,6 +4,7 @@ import httplib
 import HydrusConstants as HC
 import HydrusDB
 import HydrusExceptions
+import HydrusFileHandling
 import HydrusNATPunch
 import HydrusPaths
 import HydrusSerialisable
@@ -731,14 +732,14 @@ class DB( HydrusDB.HydrusDB ):
                 
                 path = ServerFiles.GetPath( 'file', hash )
                 
-                HydrusData.RecyclePath( path )
+                HydrusPaths.RecyclePath( path )
                 
             
             for hash in thumbnails_hashes & deletee_hashes:
                 
                 path = ServerFiles.GetPath( 'thumbnail', hash )
                 
-                HydrusData.RecyclePath( path )
+                HydrusPaths.RecyclePath( path )
                 
             
             self._c.execute( 'DELETE FROM files_info WHERE hash_id IN ' + HydrusData.SplayListForDB( deletees ) + ';' )
@@ -1761,7 +1762,7 @@ class DB( HydrusDB.HydrusDB ):
         backup_path = os.path.join( HC.DB_DIR, 'server_backup' )
         
         HydrusData.Print( 'backing up: deleting old backup' )
-        HydrusData.RecyclePath( backup_path )
+        HydrusPaths.RecyclePath( backup_path )
         
         os.mkdir( backup_path )
         
@@ -1938,7 +1939,7 @@ class DB( HydrusDB.HydrusDB ):
                 
                 if os.path.exists( update_dir ):
                     
-                    HydrusData.DeletePath( update_dir )
+                    HydrusPaths.DeletePath( update_dir )
                     
                 
                 self.pub_after_commit( 'action_service', service_key, 'stop' )
@@ -2387,7 +2388,7 @@ class DB( HydrusDB.HydrusDB ):
                 
                 path = os.path.join( HC.SERVER_UPDATES_DIR, filename )
                 
-                HydrusData.RecyclePath( path )
+                HydrusPaths.RecyclePath( path )
                 
             
             for ( service_id, end ) in first_ends:
@@ -2486,6 +2487,39 @@ class DB( HydrusDB.HydrusDB ):
                 
             
         
+        if version == 182:
+            
+            HydrusData.Print( 'generating swf thumbnails' )
+            
+            mimes = { HC.APPLICATION_FLASH }
+            mimes.update( HC.VIDEO )
+            
+            hash_ids = { hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM files_info WHERE mime IN ' + HydrusData.SplayListForDB( mimes ) + ';' ) }
+            
+            for hash_id in hash_ids:
+                
+                hash = self._GetHash( hash_id )
+                
+                try:
+                    
+                    file_path = ServerFiles.GetPath( 'file', hash )
+                    
+                except HydrusExceptions.NotFoundException:
+                    
+                    continue
+                    
+                
+                thumbnail = HydrusFileHandling.GenerateThumbnail( file_path )
+                
+                thumbnail_path = ServerFiles.GetExpectedPath( 'thumbnail', hash )
+                
+                with open( thumbnail_path, 'wb' ) as f:
+                    
+                    f.write( thumbnail )
+                    
+                
+            
+        
         HydrusData.Print( 'The server has updated to version ' + str( version + 1 ) )
         
         self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
diff --git a/include/TestClientConstants.py b/include/TestClientConstants.py
index b90c0295..bcb93c53 100644
--- a/include/TestClientConstants.py
+++ b/include/TestClientConstants.py
@@ -2,6 +2,7 @@ import ClientConstants as CC
 import ClientData
 import ClientGUIManagement
 import ClientGUIDialogsManage
+import ClientCaches
 import collections
 import HydrusConstants as HC
 import os
@@ -79,7 +80,7 @@ class TestManagers( unittest.TestCase ):
         
         HydrusGlobals.test_controller.SetRead( 'services', services )
         
-        services_manager = ClientData.ServicesManager()
+        services_manager = ClientCaches.ServicesManager( HydrusGlobals.client_controller )
         
         #
         
@@ -123,7 +124,7 @@ class TestManagers( unittest.TestCase ):
         command_1_inverted = { CC.LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_INBOX, { hash_1 } ) ] }
         command_2_inverted = { CC.LOCAL_FILE_SERVICE_KEY : [ HydrusData.ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, { hash_2 } ) ] }
         
-        undo_manager = ClientData.UndoManager()
+        undo_manager = ClientCaches.UndoManager( HydrusGlobals.client_controller )
         
         #
         
diff --git a/include/TestHydrusTags.py b/include/TestHydrusTags.py
index e044f822..216896ac 100644
--- a/include/TestHydrusTags.py
+++ b/include/TestHydrusTags.py
@@ -580,7 +580,7 @@ class TestTagParents( unittest.TestCase ):
         
         HydrusGlobals.test_controller.SetRead( 'tag_parents', tag_parents )
         
-        self._tag_parents_manager = ClientCaches.TagParentsManager()
+        self._tag_parents_manager = ClientCaches.TagParentsManager( HydrusGlobals.client_controller )
         
     
     def test_expand_predicates( self ):
@@ -725,7 +725,7 @@ class TestTagSiblings( unittest.TestCase ):
         
         HydrusGlobals.test_controller.SetRead( 'tag_siblings', tag_siblings )
         
-        self._tag_siblings_manager = ClientCaches.TagSiblingsManager()
+        self._tag_siblings_manager = ClientCaches.TagSiblingsManager( HydrusGlobals.client_controller )
         
     
     def test_autocomplete( self ):
diff --git a/server.py b/server.py
index ca79805d..c25ef431 100644
--- a/server.py
+++ b/server.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+
 # This program is free software. It comes without any warranty, to
 # the extent permitted by applicable law. You can redistribute it
 # and/or modify it under the terms of the Do What The Fuck You Want
diff --git a/test.py b/test.py
index 3735b6a1..8129972f 100644
--- a/test.py
+++ b/test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+
 import locale
 
 try: locale.setlocale( locale.LC_ALL, '' )
@@ -87,17 +89,17 @@ class Controller( object ):
         
         self._managers = {}
         
-        self._services_manager = ClientData.ServicesManager()
+        self._services_manager = ClientCaches.ServicesManager( self )
         
-        self._managers[ 'hydrus_sessions' ] = ClientCaches.HydrusSessionManagerClient()
-        self._managers[ 'tag_censorship' ] = ClientCaches.TagCensorshipManager()
-        self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager()
-        self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager()
-        self._managers[ 'undo' ] = ClientData.UndoManager()
+        self._managers[ 'hydrus_sessions' ] = ClientCaches.HydrusSessionManager( self )
+        self._managers[ 'tag_censorship' ] = ClientCaches.TagCensorshipManager( self )
+        self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
+        self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
+        self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
         self._managers[ 'web_sessions' ] = TestConstants.FakeWebSessionManager()
         self._managers[ 'restricted_services_sessions' ] = HydrusSessions.HydrusSessionManagerServer()
         self._managers[ 'messaging_sessions' ] = HydrusSessions.HydrusMessagingSessionManagerServer()
-        self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache()
+        self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache( self )
         
         self._cookies = {}