diff --git a/help/changelog.html b/help/changelog.html
index c2d960ea..f2525ce6 100755
--- a/help/changelog.html
+++ b/help/changelog.html
@@ -8,6 +8,38 @@
changelog
+ version 278
+ - fixed the tumblr raw url converter to now point to the data.tumblr.com domain
+ - added a hardcoded ssl verify exception for data.tumblr.com, which has an incorrectly defined ssl cert (at least for public-facing interactions), wew
+ - all existing db urls and file import cache urls for media.tumblr.com will be updated to data.tumblr.com on update! (everything _should_ just magically work again)
+ - fixed apng import, which the decompressionbomb detection code was not handling correctly
+ - collapsed the different instatiations of the 'file import status' button down to one class
+ - the file import status button now has a right-click menu that supports 'retry failures' and 'delete processed', if applicable
+ - misc import status cache cleanup and refactoring
+ - you can now edit or completely turn off the [404] and [DEAD] thread watcher page name prefixes under options->downloading
+ - thread watchers should more reliably keep 404 status
+ - 'open selection in a new page' now preserves file order!
+ - 'view this file's duplicates' now sorts the files!
+ - options->gui now has an option to change how often 'last session' is saved
+ - 'last session' will no longer autosave to the database if there are no changes
+ - tags exported to neighbouring .txt files are now correctly sibling-collapsed
+ - tags imported or exported via neighbouring .txt files are now correctly tag censored
+ - the manage tags dialog will now protest with a yes/no dialog if you attempt to cancel it with uncommitted changes
+ - the manage parents and siblings dialogs will now protest with a yes/no on an ok event if there are 'uncommitted' pairs in the lower boxes (e.g. if you forgot to click the 'add' button)
+ - fixed an issue that would sometimes stop old sessions from loading properly
+ - the duplicates page now does its maintenance jobs in modal popups!
+ - attempting to apply a duplicate status to more than 100 pairs now throws up a warning yes/no dialog
+ - the manage urls dialog now has copy/paste buttons
+ - added a (somewhat debug) option to disable the mouse hide&anchor behaviour on slow Windows canvas drags to options->media
+ - added a 'regen all phashes' command to the database regen menu
+ - the disk cache options in help now have a help button to explain good values for ssd vs hdd
+ - the edit bandwidth rules dialog now uses the new listctrl
+ - merged the old and new login managers
+ - misc login work
+ - misc refactoring
+ - misc cleanup
+
version 277
- expanded the domain manager into a legit serialisable object that holds data and saves itself on changes. to begin with, it supports custom http headers for particular on network contexts
diff --git a/include/ClientCaches.py b/include/ClientCaches.py
index 09c9eb1e..4d762041 100644
--- a/include/ClientCaches.py
+++ b/include/ClientCaches.py
@@ -3174,360 +3174,3 @@ class UndoManager( object ):
-class WebSessionManagerClient( object ):
-
- SESSION_TIMEOUT = 60 * 60
-
- def __init__( self, controller ):
-
- self._controller = controller
-
- self._error_names = set()
-
- self._network_contexts_to_session_timeouts = {}
-
- self._lock = threading.Lock()
-
-
- def _GetCookiesDict( self, network_context ):
-
- session = self._GetSession( network_context )
-
- cookies = session.cookies
-
- cookies.clear_expired_cookies()
-
- domains = cookies.list_domains()
-
- for domain in domains:
-
- if domain.endswith( network_context.context_data ):
-
- return cookies.get_dict( domain )
-
-
-
- return {}
-
-
- def _GetSession( self, network_context ):
-
- session = self._controller.network_engine.session_manager.GetSession( network_context )
-
- if network_context not in self._network_contexts_to_session_timeouts:
-
- self._network_contexts_to_session_timeouts[ network_context ] = 0
-
-
- if HydrusData.TimeHasPassed( self._network_contexts_to_session_timeouts[ network_context ] ):
-
- session.cookies.clear_session_cookies()
-
-
- self._network_contexts_to_session_timeouts[ network_context ] = HydrusData.GetNow() + self.SESSION_TIMEOUT
-
- return session
-
-
- def _IsLoggedIn( self, network_context, required_cookies ):
-
- cookie_dict = self._GetCookiesDict( network_context )
-
- for name in required_cookies:
-
- if name not in cookie_dict:
-
- return False
-
-
-
- return True
-
-
- def EnsureHydrusSessionIsOK( self, service_key ):
-
- with self._lock:
-
- if not self._controller.services_manager.ServiceExists( service_key ):
-
- raise HydrusExceptions.DataMissing( 'Service does not exist!' )
-
-
- name = self._controller.services_manager.GetService( service_key ).GetName()
-
- if service_key in self._error_names:
-
- raise Exception( 'Could not establish a hydrus network session for ' + name + '! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
-
-
- network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, service_key )
-
- required_cookies = [ 'session_key' ]
-
- if self._IsLoggedIn( network_context, required_cookies ):
-
- return
-
-
- try:
-
- self.SetupHydrusSession( service_key )
-
- if not self._IsLoggedIn( network_context, required_cookies ):
-
- return
-
-
- HydrusData.Print( 'Successfully logged into ' + name + '.' )
-
- except:
-
- self._error_names.add( service_key )
-
- raise
-
-
-
-
- def EnsureLoggedIn( self, name ):
-
- with self._lock:
-
- if name in self._error_names:
-
- raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
-
-
- if name == 'hentai foundry':
-
- network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
-
- required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
-
- elif name == 'pixiv':
-
- network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
-
- required_cookies = [ 'PHPSESSID' ]
-
-
- if self._IsLoggedIn( network_context, required_cookies ):
-
- return
-
-
- try:
-
- if name == 'hentai foundry':
-
- self.LoginHF( network_context )
-
- elif name == 'pixiv':
-
- result = self._controller.Read( 'serialisable_simple', 'pixiv_account' )
-
- if result is None:
-
- raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
-
-
- ( pixiv_id, password ) = result
-
- self.LoginPixiv( network_context, pixiv_id, password )
-
-
- if not self._IsLoggedIn( network_context, required_cookies ):
-
- raise Exception( name + ' login did not work correctly!' )
-
-
- HydrusData.Print( 'Successfully logged into ' + name + '.' )
-
- except:
-
- self._error_names.add( name )
-
- raise
-
-
-
-
- def LoginHF( self, network_context ):
-
- session = self._GetSession( network_context )
-
- response = session.get( 'https://www.hentai-foundry.com/' )
-
- time.sleep( 1 )
-
- response = session.get( 'https://www.hentai-foundry.com/?enterAgree=1' )
-
- time.sleep( 1 )
-
- cookie_dict = self._GetCookiesDict( network_context )
-
- raw_csrf = cookie_dict[ 'YII_CSRF_TOKEN' ] # 19b05b536885ec60b8b37650a32f8deb11c08cd1s%3A40%3A%222917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32%22%3B
-
- processed_csrf = urllib.unquote( raw_csrf ) # 19b05b536885ec60b8b37650a32f8deb11c08cd1s:40:"2917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32";
-
- csrf_token = processed_csrf.split( '"' )[1] # the 2917... bit
-
- hentai_foundry_form_info = ClientDefaults.GetDefaultHentaiFoundryInfo()
-
- hentai_foundry_form_info[ 'YII_CSRF_TOKEN' ] = csrf_token
-
- response = session.post( 'http://www.hentai-foundry.com/site/filters', data = hentai_foundry_form_info )
-
- time.sleep( 1 )
-
-
- # This updated login form is cobbled together from the example in PixivUtil2
- # it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
- # Pixiv 400s if cookies and referrers aren't passed correctly
- # I am leaving this as a mess with the hope the eventual login engine will replace it
- def LoginPixiv( self, network_context, pixiv_id, password ):
-
- session = self._GetSession( network_context )
-
- response = session.get( 'https://accounts.pixiv.net/login' )
-
- soup = ClientDownloading.GetSoup( response.content )
-
- # some whocking 20kb bit of json tucked inside a hidden form input wew lad
- i = soup.find( 'input', id = 'init-config' )
-
- raw_json = i['value']
-
- j = json.loads( raw_json )
-
- if 'pixivAccount.postKey' not in j:
-
- raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
-
-
- post_key = j[ 'pixivAccount.postKey' ]
-
- form_fields = {}
-
- form_fields[ 'pixiv_id' ] = pixiv_id
- form_fields[ 'password' ] = password
- form_fields[ 'captcha' ] = ''
- form_fields[ 'g_recaptcha_response' ] = ''
- form_fields[ 'return_to' ] = 'https://www.pixiv.net'
- form_fields[ 'lang' ] = 'en'
- form_fields[ 'post_key' ] = post_key
- form_fields[ 'source' ] = 'pc'
-
- headers = {}
-
- headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
- headers[ 'origin' ] = "https://accounts.pixiv.net"
-
- session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
-
- time.sleep( 1 )
-
-
- def SetupHydrusSession( self, service_key ):
-
- # nah, replace this with a proper login script
-
- service = self._controller.services_manager.GetService( service_key )
-
- if not service.HasAccessKey():
-
- raise HydrusExceptions.DataMissing( 'No access key for this service, so cannot set up session!' )
-
-
- access_key = service.GetAccessKey()
-
- url = 'blah'
-
- network_job = ClientNetworking.NetworkJobHydrus( service_key, 'GET', url )
-
- network_job.SetForLogin( True )
-
- network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.encode( 'hex' ) )
-
- self._controller.network_engine.AddJob( network_job )
-
- network_job.WaitUntilDone()
-
-
- def TestPixiv( self, pixiv_id, password ):
-
- # this is just an ugly copy, but fuck it for the minute
- # we'll figure out a proper testing engine later with the login engine and tie the manage gui into it as well
-
- session = requests.Session()
-
- response = session.get( 'https://accounts.pixiv.net/login' )
-
- soup = ClientDownloading.GetSoup( response.content )
-
- # some whocking 20kb bit of json tucked inside a hidden form input wew lad
- i = soup.find( 'input', id = 'init-config' )
-
- raw_json = i['value']
-
- j = json.loads( raw_json )
-
- if 'pixivAccount.postKey' not in j:
-
- return ( False, 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
-
-
- post_key = j[ 'pixivAccount.postKey' ]
-
- form_fields = {}
-
- form_fields[ 'pixiv_id' ] = pixiv_id
- form_fields[ 'password' ] = password
- form_fields[ 'captcha' ] = ''
- form_fields[ 'g_recaptcha_response' ] = ''
- form_fields[ 'return_to' ] = 'https://www.pixiv.net'
- form_fields[ 'lang' ] = 'en'
- form_fields[ 'post_key' ] = post_key
- form_fields[ 'source' ] = 'pc'
-
- headers = {}
-
- headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
- headers[ 'origin' ] = "https://accounts.pixiv.net"
-
- r = session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
-
- if not r.ok:
-
- HydrusData.ShowText( r.content )
-
- return ( False, 'Login request failed! Info printed to log.' )
-
-
- cookies = session.cookies
-
- cookies.clear_expired_cookies()
-
- domains = cookies.list_domains()
-
- for domain in domains:
-
- if domain.endswith( 'pixiv.net' ):
-
- d = cookies.get_dict( domain )
-
- if 'PHPSESSID' not in d:
-
- HydrusData.ShowText( r.content )
-
- return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
-
-
- return ( True, '' )
-
-
-
- HydrusData.ShowText( r.content )
-
- return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
-
diff --git a/include/ClientController.py b/include/ClientController.py
index a85791cd..9f86faf6 100755
--- a/include/ClientController.py
+++ b/include/ClientController.py
@@ -5,6 +5,7 @@ import ClientDefaults
import ClientGUIMenus
import ClientNetworking
import ClientNetworkingDomain
+import ClientNetworkingLogin
import ClientThreading
import hashlib
import HydrusConstants as HC
@@ -51,10 +52,10 @@ class Controller( HydrusController.HydrusController ):
HG.client_controller = self
# just to set up some defaults, in case some db update expects something for an odd yaml-loading reason
- self._options = ClientDefaults.GetClientDefaultOptions()
- self._new_options = ClientData.ClientOptions( self.db_dir )
+ self.options = ClientDefaults.GetClientDefaultOptions()
+ self.new_options = ClientData.ClientOptions( self.db_dir )
- HC.options = self._options
+ HC.options = self.options
self._last_mouse_position = None
self._menu_open = False
@@ -324,9 +325,9 @@ class Controller( HydrusController.HydrusController ):
return False
- idle_normal = self._options[ 'idle_normal' ]
- idle_period = self._options[ 'idle_period' ]
- idle_mouse_period = self._options[ 'idle_mouse_period' ]
+ idle_normal = self.options[ 'idle_normal' ]
+ idle_period = self.options[ 'idle_period' ]
+ idle_mouse_period = self.options[ 'idle_mouse_period' ]
if idle_normal:
@@ -389,11 +390,11 @@ class Controller( HydrusController.HydrusController ):
def DoIdleShutdownWork( self ):
- stop_time = HydrusData.GetNow() + ( self._options[ 'idle_shutdown_max_minutes' ] * 60 )
+ stop_time = HydrusData.GetNow() + ( self.options[ 'idle_shutdown_max_minutes' ] * 60 )
self.MaintainDB( stop_time = stop_time )
- if not self._options[ 'pause_repo_sync' ]:
+ if not self.options[ 'pause_repo_sync' ]:
services = self.services_manager.GetServices( HC.REPOSITORIES )
@@ -422,11 +423,11 @@ class Controller( HydrusController.HydrusController ):
self._CreateSplash()
- idle_shutdown_action = self._options[ 'idle_shutdown' ]
+ idle_shutdown_action = self.options[ 'idle_shutdown' ]
if idle_shutdown_action in ( CC.IDLE_ON_SHUTDOWN, CC.IDLE_ON_SHUTDOWN_ASK_FIRST ):
- idle_shutdown_max_minutes = self._options[ 'idle_shutdown_max_minutes' ]
+ idle_shutdown_max_minutes = self.options[ 'idle_shutdown_max_minutes' ]
time_to_stop = HydrusData.GetNow() + ( idle_shutdown_max_minutes * 60 )
@@ -497,12 +498,12 @@ class Controller( HydrusController.HydrusController ):
def GetOptions( self ):
- return self._options
+ return self.options
def GetNewOptions( self ):
- return self._new_options
+ return self.new_options
def GoodTimeToDoForegroundWork( self ):
@@ -555,12 +556,12 @@ class Controller( HydrusController.HydrusController ):
self.services_manager = ClientCaches.ServicesManager( self )
- self._options = self.Read( 'options' )
- self._new_options = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
+ self.options = self.Read( 'options' )
+ self.new_options = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
- HC.options = self._options
+ HC.options = self.options
- if self._new_options.GetBoolean( 'use_system_ffmpeg' ):
+ if self.new_options.GetBoolean( 'use_system_ffmpeg' ):
if HydrusVideoHandling.FFMPEG_PATH.startswith( HC.BIN_DIR ):
@@ -600,14 +601,14 @@ class Controller( HydrusController.HydrusController ):
if domain_manager is None:
- domain_manager = ClientNetworking.NetworkSessionManager()
+ domain_manager = ClientNetworkingDomain.NetworkDomainManager()
domain_manager._dirty = True
wx.MessageBox( 'Your domain manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
self.network_engine = ClientNetworking.NetworkEngine( self, bandwidth_manager, session_manager, domain_manager, login_manager )
@@ -624,7 +625,6 @@ class Controller( HydrusController.HydrusController ):
self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
- self._managers[ 'web_sessions' ] = ClientCaches.WebSessionManagerClient( self )
def wx_code():
@@ -642,7 +642,7 @@ class Controller( HydrusController.HydrusController ):
def InitView( self ):
- if self._options[ 'password' ] is not None:
+ if self.options[ 'password' ] is not None:
self.pub( 'splash_set_status_text', 'waiting for password' )
@@ -657,7 +657,7 @@ class Controller( HydrusController.HydrusController ):
# this can produce unicode with cyrillic or w/e keyboards, which hashlib can't handle
password = HydrusData.ToByteString( dlg.GetValue() )
- if hashlib.sha256( password ).digest() == self._options[ 'password' ]: break
+ if hashlib.sha256( password ).digest() == self.options[ 'password' ]: break
else:
@@ -743,7 +743,7 @@ class Controller( HydrusController.HydrusController ):
def MaintainDB( self, stop_time = None ):
- if self._new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ):
+ if self.new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ):
phashes_stop_time = stop_time
@@ -763,7 +763,7 @@ class Controller( HydrusController.HydrusController ):
self.WriteInterruptable( 'maintain_similar_files_tree', stop_time = tree_stop_time, abandon_if_other_work_to_do = True )
- search_distance = self._new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
+ search_distance = self.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
search_stop_time = stop_time
@@ -815,7 +815,7 @@ class Controller( HydrusController.HydrusController ):
self._timestamps[ 'last_page_change' ] = HydrusData.GetNow()
- disk_cache_maintenance_mb = self._new_options.GetNoneableInteger( 'disk_cache_maintenance_mb' )
+ disk_cache_maintenance_mb = self.new_options.GetNoneableInteger( 'disk_cache_maintenance_mb' )
if disk_cache_maintenance_mb is not None:
@@ -1141,7 +1141,7 @@ class Controller( HydrusController.HydrusController ):
return False
- max_cpu = self._options[ 'idle_cpu_max' ]
+ max_cpu = self.options[ 'idle_cpu_max' ]
if max_cpu is None:
diff --git a/include/ClientDB.py b/include/ClientDB.py
index 98ce5869..775d54ec 100755
--- a/include/ClientDB.py
+++ b/include/ClientDB.py
@@ -1499,7 +1499,7 @@ class DB( HydrusDB.HydrusDB ):
path = client_files_manager.GetFilePath( hash, mime )
- if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG ):
+ if mime in HC.MIMES_WE_CAN_PHASH:
try:
@@ -9830,6 +9830,31 @@ class DB( HydrusDB.HydrusDB ):
self.pub_initial_message( message )
+ if version == 277:
+
+ self._controller.pub( 'splash_set_status_text', 'updating tumblr urls' )
+
+ urls = self._c.execute( 'SELECT hash_id, url FROM urls;' ).fetchall()
+
+ # don't catch the 68.media.whatever, as these may be valid, not raw urls
+ magic_phrase = '//media.tumblr.com'
+ replacement = '//data.tumblr.com'
+
+ updates = []
+
+ for ( hash_id, url ) in urls:
+
+ if magic_phrase in url:
+
+ fixed_url = url.replace( magic_phrase, replacement )
+
+ updates.append( ( fixed_url, hash_id ) )
+
+
+
+ self._c.executemany( 'UPDATE OR IGNORE urls SET url = ? WHERE hash_id = ?;', updates )
+
+
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
@@ -10426,6 +10451,7 @@ class DB( HydrusDB.HydrusDB ):
elif action == 'repair_client_files': result = self._RepairClientFiles( *args, **kwargs )
elif action == 'reset_repository': result = self._ResetRepository( *args, **kwargs )
elif action == 'save_options': result = self._SaveOptions( *args, **kwargs )
+ elif action == 'schedule_full_phash_regen': result = self._CacheSimilarFilesSchedulePHashRegeneration()
elif action == 'serialisable_simple': result = self._SetJSONSimple( *args, **kwargs )
elif action == 'serialisable': result = self._SetJSONDump( *args, **kwargs )
elif action == 'serialisables_overwrite': result = self._OverwriteJSONDumps( *args, **kwargs )
diff --git a/include/ClientData.py b/include/ClientData.py
index e0a97d33..16905151 100644
--- a/include/ClientData.py
+++ b/include/ClientData.py
@@ -853,6 +853,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'booleans' ][ 'reverse_page_shift_drag_behaviour' ] = False
+ self._dictionary[ 'booleans' ][ 'anchor_and_hide_canvas_drags' ] = HC.PLATFORM_WINDOWS
+
#
self._dictionary[ 'colours' ] = HydrusSerialisable.SerialisableDictionary()
@@ -919,6 +921,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'integers' ][ 'thumbnail_visibility_scroll_percent' ] = 75
+ self._dictionary[ 'integers' ][ 'last_session_save_period_minutes' ] = 5
+
#
self._dictionary[ 'keys' ] = {}
@@ -951,6 +955,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'noneable_strings' ][ 'favourite_file_lookup_script' ] = 'gelbooru md5'
self._dictionary[ 'noneable_strings' ][ 'suggested_tags_layout' ] = 'notebook'
self._dictionary[ 'noneable_strings' ][ 'backup_path' ] = None
+ self._dictionary[ 'noneable_strings' ][ 'thread_watcher_not_found_page_string' ] = '[404]'
+ self._dictionary[ 'noneable_strings' ][ 'thread_watcher_dead_page_string' ] = '[DEAD]'
self._dictionary[ 'strings' ] = {}
diff --git a/include/ClientDefaults.py b/include/ClientDefaults.py
index 9b93ebe3..b7518308 100644
--- a/include/ClientDefaults.py
+++ b/include/ClientDefaults.py
@@ -146,7 +146,7 @@ def GetClientDefaultOptions():
options[ 'idle_shutdown_max_minutes' ] = 5
options[ 'maintenance_delete_orphans_period' ] = 86400 * 3
options[ 'trash_max_age' ] = 72
- options[ 'trash_max_size' ] = 512
+ options[ 'trash_max_size' ] = 2048
options[ 'remove_trashed_files' ] = False
options[ 'remove_filtered_files' ] = False
options[ 'external_host' ] = None
diff --git a/include/ClientDownloading.py b/include/ClientDownloading.py
index a48bba47..480c2504 100644
--- a/include/ClientDownloading.py
+++ b/include/ClientDownloading.py
@@ -1239,9 +1239,7 @@ class GalleryHentaiFoundry( Gallery ):
def _EnsureLoggedIn( self ):
- manager = HG.client_controller.GetManager( 'web_sessions' )
-
- manager.EnsureLoggedIn( 'hentai foundry' )
+ HG.client_controller.network_engine.login_manager.EnsureLoggedIn( 'hentai foundry' )
def _GetFileURLAndTags( self, url ):
@@ -1564,9 +1562,7 @@ class GalleryPixiv( Gallery ):
def _EnsureLoggedIn( self ):
- manager = HG.client_controller.GetManager( 'web_sessions' )
-
- manager.EnsureLoggedIn( 'pixiv' )
+ HG.client_controller.network_engine.login_manager.EnsureLoggedIn( 'pixiv' )
def _ParseGalleryPage( self, html, url_base ):
@@ -1753,6 +1749,8 @@ class GalleryTumblr( Gallery ):
# I am not sure if it is always 68, but let's not assume
+ # Indeed, this is apparently now 78, wew!
+
( scheme, rest ) = long_url.split( '://', 1 )
if rest.startswith( 'media.tumblr.com' ):
@@ -1767,6 +1765,11 @@ class GalleryTumblr( Gallery ):
return shorter_url
+ def MediaToDataSubdomain( url ):
+
+ return url.replace( 'media', 'data', 1 )
+
+
definitely_no_more_pages = False
processed_raw_json = data.split( 'var tumblr_api_read = ' )[1][:-2] # -1 takes a js ';' off the end
@@ -1823,6 +1826,8 @@ class GalleryTumblr( Gallery ):
url = Remove68Subdomain( url )
+ url = MediaToDataSubdomain( url )
+
url = ClientData.ConvertHTTPToHTTPS( url )
diff --git a/include/ClientGUI.py b/include/ClientGUI.py
index 5912e17e..9ebb10f2 100755
--- a/include/ClientGUI.py
+++ b/include/ClientGUI.py
@@ -1154,7 +1154,8 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
submenu = wx.Menu()
ClientGUIMenus.AppendMenuItem( self, submenu, 'autocomplete cache', 'Delete and recreate the tag autocomplete cache, fixing any miscounts.', self._RegenerateACCache )
- ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search data', 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesData )
+ ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search metadata', 'Delete and recreate the similar files search phashes.', self._RegenerateSimilarFilesPhashes )
+ ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search tree', 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesTree )
ClientGUIMenus.AppendMenuItem( self, submenu, 'all thumbnails', 'Delete all thumbnails and regenerate them from their original files.', self._RegenerateThumbnails )
ClientGUIMenus.AppendMenu( menu, submenu, 'regenerate' )
@@ -1656,7 +1657,9 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
self._notebook.LoadGUISession( default_gui_session )
- wx.CallLater( 5 * 60 * 1000, self.SaveLastSession )
+ last_session_save_period_minutes = self._controller.new_options.GetInteger( 'last_session_save_period_minutes' )
+
+ wx.CallLater( last_session_save_period_minutes * 60 * 1000, self.SaveLastSession )
def _ManageAccountTypes( self, service_key ):
@@ -2101,7 +2104,26 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
- def _RegenerateSimilarFilesData( self ):
+ def _RegenerateSimilarFilesPhashes( self ):
+
+ message = 'This will schedule all similar files \'phash\' metadata to be regenerated. This is a very expensive operation that will occur in future idle time.'
+ message += os.linesep * 2
+ message += 'This ultimately requires a full read for all valid files. It is a large investment of CPU and HDD time.'
+ message += os.linesep * 2
+ message += 'Do not run this unless you know your phashes need to be regenerated.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message, yes_label = 'do it', no_label = 'forget it' ) as dlg:
+
+ result = dlg.ShowModal()
+
+ if result == wx.ID_YES:
+
+ self._controller.Write( 'schedule_full_phash_regen' )
+
+
+
+
+ def _RegenerateSimilarFilesTree( self ):
message = 'This will delete and then recreate the similar files search tree. This is useful if it has somehow become unbalanced and similar files searches are running slow.'
message += os.linesep * 2
@@ -3125,7 +3147,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._notebook.NewPage( management_controller, on_deepest_notebook = True )
- def NewPageQuery( self, service_key, initial_hashes = None, initial_predicates = None, page_name = None ):
+ def NewPageQuery( self, service_key, initial_hashes = None, initial_predicates = None, page_name = None, do_sort = False ):
if initial_hashes is None:
@@ -3137,7 +3159,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
initial_predicates = []
- self._notebook.NewPageQuery( service_key, initial_hashes = initial_hashes, initial_predicates = initial_predicates, page_name = page_name, on_deepest_notebook = True )
+ self._notebook.NewPageQuery( service_key, initial_hashes = initial_hashes, initial_predicates = initial_predicates, page_name = page_name, on_deepest_notebook = True, do_sort = do_sort )
def NotifyClosedPage( self, page ):
@@ -3312,7 +3334,9 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._notebook.SaveGUISession( 'last session' )
- wx.CallLater( 5 * 60 * 1000, self.SaveLastSession )
+ last_session_save_period_minutes = self._controller.new_options.GetInteger( 'last_session_save_period_minutes' )
+
+ wx.CallLater( last_session_save_period_minutes * 60 * 1000, self.SaveLastSession )
def SetMediaFocus( self ): self._SetMediaFocus()
diff --git a/include/ClientGUICanvas.py b/include/ClientGUICanvas.py
index 09354185..012b9d9d 100755
--- a/include/ClientGUICanvas.py
+++ b/include/ClientGUICanvas.py
@@ -2945,7 +2945,9 @@ class CanvasWithHovers( CanvasWithDetails ):
self._current_drag_is_touch = True
- if HC.PLATFORM_WINDOWS and not self._current_drag_is_touch:
+ anchor_and_hide_canvas_drags = HG.client_controller.new_options.GetBoolean( 'anchor_and_hide_canvas_drags' )
+
+ if HC.PLATFORM_WINDOWS and anchor_and_hide_canvas_drags and not self._current_drag_is_touch:
# touch events obviously don't mix with warping well. the touch just warps it back and again and we get a massive delta!
diff --git a/include/ClientGUICommon.py b/include/ClientGUICommon.py
index 86140241..e44df809 100755
--- a/include/ClientGUICommon.py
+++ b/include/ClientGUICommon.py
@@ -733,6 +733,7 @@ class ChoiceSort( wx.Panel ):
self._sort_asc_choice.Bind( wx.EVT_CHOICE, self.EventSortAscChoice )
HG.client_controller.sub( self, 'ACollectHappened', 'collect_media' )
+ HG.client_controller.sub( self, 'BroadcastSort', 'do_page_sort' )
if self._management_controller is not None and self._management_controller.HasVariable( 'media_sort' ):
@@ -815,7 +816,12 @@ class ChoiceSort( wx.Panel ):
- def BroadcastSort( self ):
+ def BroadcastSort( self, page_key = None ):
+
+ if page_key is not None and page_key != self._management_controller.GetKey( 'page' ):
+
+ return
+
self._BroadcastSort()
@@ -1635,11 +1641,20 @@ class NoneableSpinCtrl( wx.Panel ):
def GetValue( self ):
- if self._checkbox.GetValue(): return None
+ if self._checkbox.GetValue():
+
+ return None
+
else:
- if self._num_dimensions == 2: return ( self._one.GetValue() * self._multiplier, self._two.GetValue() * self._multiplier )
- else: return self._one.GetValue() * self._multiplier
+ if self._num_dimensions == 2:
+
+ return ( self._one.GetValue() * self._multiplier, self._two.GetValue() * self._multiplier )
+
+ else:
+
+ return self._one.GetValue() * self._multiplier
+
@@ -1681,6 +1696,90 @@ class NoneableSpinCtrl( wx.Panel ):
+class NoneableTextCtrl( wx.Panel ):
+
+ def __init__( self, parent, message = '', none_phrase = 'no limit' ):
+
+ wx.Panel.__init__( self, parent )
+
+ self._checkbox = wx.CheckBox( self )
+ self._checkbox.Bind( wx.EVT_CHECKBOX, self.EventCheckBox )
+ self._checkbox.SetLabelText( none_phrase )
+
+ self._text = wx.TextCtrl( self )
+
+ hbox = wx.BoxSizer( wx.HORIZONTAL )
+
+ if len( message ) > 0:
+
+ hbox.AddF( BetterStaticText( self, message + ': ' ), CC.FLAGS_VCENTER )
+
+
+ hbox.AddF( self._text, CC.FLAGS_VCENTER )
+ hbox.AddF( self._checkbox, CC.FLAGS_VCENTER )
+
+ self.SetSizer( hbox )
+
+
+ def Bind( self, event_type, callback ):
+
+ self._checkbox.Bind( wx.EVT_CHECKBOX, callback )
+
+ self._text.Bind( wx.EVT_TEXT, callback )
+
+
+ def EventCheckBox( self, event ):
+
+ if self._checkbox.GetValue():
+
+ self._text.Disable()
+
+ else:
+
+ self._text.Enable()
+
+
+
+ def GetValue( self ):
+
+ if self._checkbox.GetValue():
+
+ return None
+
+ else:
+
+ return self._text.GetValue()
+
+
+
+ def SetToolTipString( self, text ):
+
+ wx.Panel.SetToolTipString( self, text )
+
+ for c in self.GetChildren():
+
+ c.SetToolTipString( text )
+
+
+
+ def SetValue( self, value ):
+
+ if value is None:
+
+ self._checkbox.SetValue( True )
+
+ self._text.Disable()
+
+ else:
+
+ self._checkbox.SetValue( False )
+
+ self._text.Enable()
+
+ self._text.SetValue( value )
+
+
+
class OnOffButton( wx.Button ):
def __init__( self, parent, page_key, topic, on_label, off_label = None, start_on = True ):
diff --git a/include/ClientGUIControls.py b/include/ClientGUIControls.py
index 6be06c43..280333a8 100644
--- a/include/ClientGUIControls.py
+++ b/include/ClientGUIControls.py
@@ -20,11 +20,9 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
ClientGUICommon.StaticBox.__init__( self, parent, 'bandwidth rules' )
- columns = [ ( 'type', -1 ), ( 'time delta', 120 ), ( 'max allowed', 80 ) ]
+ listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
- listctrl_panel = ClientGUIListCtrl.SaneListCtrlPanel( self )
-
- self._listctrl = ClientGUIListCtrl.SaneListCtrl( listctrl_panel, 100, columns, delete_key_callback = self._Delete, activation_callback = self._Edit )
+ self._listctrl = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, 'bandwidth_rules', 8, 10, [ ( 'type', -1 ), ( 'time delta', 16 ), ( 'max allowed', 14 ) ], self._ConvertRuleToListctrlTuples, delete_key_callback = self._Delete, activation_callback = self._Edit )
listctrl_panel.SetListCtrl( self._listctrl )
@@ -34,18 +32,13 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
#
- for rule in bandwidth_rules.GetRules():
-
- sort_tuple = rule
-
- display_tuple = self._GetDisplayTuple( sort_tuple )
-
- self._listctrl.Append( display_tuple, sort_tuple )
-
+ self._listctrl.AddDatas( bandwidth_rules.GetRules() )
+
+ self._listctrl.Sort( 0 )
#
- self.AddF( listctrl_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
+ self.AddF( listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
def _Add( self ):
@@ -62,16 +55,14 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
new_rule = panel.GetValue()
- sort_tuple = new_rule
+ self._listctrl.AddDatas( ( new_rule, ) )
- display_tuple = self._GetDisplayTuple( sort_tuple )
-
- self._listctrl.Append( display_tuple, sort_tuple )
+ self._listctrl.Sort()
- def _GetDisplayTuple( self, rule ):
+ def _ConvertRuleToListctrlTuples( self, rule ):
( bandwidth_type, time_delta, max_allowed ) = rule
@@ -88,7 +79,10 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
pretty_max_allowed = HydrusData.ConvertIntToPrettyString( max_allowed )
- return ( pretty_bandwidth_type, pretty_time_delta, pretty_max_allowed )
+ sort_tuple = ( pretty_bandwidth_type, time_delta, max_allowed )
+ display_tuple = ( pretty_bandwidth_type, pretty_time_delta, pretty_max_allowed )
+
+ return ( display_tuple, sort_tuple )
def _Delete( self ):
@@ -97,18 +91,16 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
if dlg.ShowModal() == wx.ID_YES:
- self._listctrl.RemoveAllSelected()
+ self._listctrl.DeleteSelected()
def _Edit( self ):
- all_selected = self._listctrl.GetAllSelected()
+ selected_rules = self._listctrl.GetData( only_selected = True )
- for index in all_selected:
-
- rule = self._listctrl.GetClientData( index )
+ for rule in selected_rules:
with ClientGUITopLevelWindows.DialogEdit( self, 'edit rule' ) as dlg:
@@ -120,11 +112,9 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
edited_rule = panel.GetValue()
- sort_tuple = edited_rule
+ self._listctrl.DeleteDatas( ( rule, ) )
- display_tuple = self._GetDisplayTuple( sort_tuple )
-
- self._listctrl.UpdateRow( index, display_tuple, sort_tuple )
+ self._listctrl.AddDatas( ( edited_rule, ) )
else:
@@ -133,12 +123,16 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
+ self._listctrl.Sort()
+
def GetValue( self ):
bandwidth_rules = HydrusNetworking.BandwidthRules()
- for ( bandwidth_type, time_delta, max_allowed ) in self._listctrl.GetClientData():
+ for rule in self._listctrl.GetData():
+
+ ( bandwidth_type, time_delta, max_allowed ) = rule
bandwidth_rules.AddRule( bandwidth_type, time_delta, max_allowed )
diff --git a/include/ClientGUIDialogs.py b/include/ClientGUIDialogs.py
index cf48f7f1..0483f330 100755
--- a/include/ClientGUIDialogs.py
+++ b/include/ClientGUIDialogs.py
@@ -2525,6 +2525,10 @@ class DialogPathsToTags( Dialog ):
txt_tags = siblings_manager.CollapseTags( self._service_key, txt_tags )
+ tag_censorship_manager = HG.client_controller.GetManager( 'tag_censorship' )
+
+ txt_tags = tag_censorship_manager.FilterTags( self._service_key, txt_tags )
+
tags.extend( txt_tags )
except:
@@ -3337,9 +3341,19 @@ class DialogSetupExport( Dialog ):
tags = set()
+ siblings_manager = HG.controller.GetManager( 'tag_siblings' )
+
+ tag_censorship_manager = HG.client_controller.GetManager( 'tag_censorship' )
+
for service_key in self._neighbouring_txt_tag_service_keys:
- tags.update( tags_manager.GetCurrent( service_key ) )
+ current_tags = tags_manager.GetCurrent( service_key )
+
+ current_tags = siblings_manager.CollapseTags( service_key, current_tags )
+
+ current_tags = tag_censorship_manager.FilterTags( service_key, current_tags )
+
+ tags.update( current_tags )
tags = list( tags )
diff --git a/include/ClientGUIDialogsManage.py b/include/ClientGUIDialogsManage.py
index 06c5bdff..8859fd1b 100644
--- a/include/ClientGUIDialogsManage.py
+++ b/include/ClientGUIDialogsManage.py
@@ -2363,8 +2363,7 @@ class DialogManageImportFoldersEdit( ClientGUIDialogs.Dialog ):
self._paused = wx.CheckBox( self._folder_box )
- self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._folder_box, CC.GlobalBMPs.seed_cache, self.ShowSeedCache )
- self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
+ self._seed_cache_button = ClientGUISeedCache.SeedCacheButton( self, HG.client_controller, self._import_folder.GetSeedCache, seed_cache_set_callable = self._import_folder.SetSeedCache )
#
@@ -2756,25 +2755,6 @@ class DialogManageImportFoldersEdit( ClientGUIDialogs.Dialog ):
return self._import_folder
- def ShowSeedCache( self ):
-
- seed_cache = self._import_folder.GetSeedCache()
-
- dupe_seed_cache = seed_cache.Duplicate()
-
- with ClientGUITopLevelWindows.DialogEdit( self, 'file import status' ) as dlg:
-
- panel = ClientGUISeedCache.EditSeedCachePanel( dlg, HG.client_controller, dupe_seed_cache )
-
- dlg.SetPanel( panel )
-
- if dlg.ShowModal() == wx.ID_OK:
-
- self._import_folder.SetSeedCache( dupe_seed_cache )
-
-
-
-
class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
def __init__( self, parent ):
@@ -2872,7 +2852,7 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
try:
- manager = HG.client_controller.GetManager( 'web_sessions' )
+ manager = HG.client_controller.network_engine.login_manager
( result, message ) = manager.TestPixiv( pixiv_id, password )
@@ -3611,6 +3591,19 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
def EventOK( self, event ):
+ if self._tag_repositories.GetCurrentPage().HasUncommittedPair():
+
+ message = 'Are you sure you want to OK? You have an uncommitted pair.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
+
+ if dlg.ShowModal() != wx.ID_YES:
+
+ return
+
+
+
+
service_keys_to_content_updates = {}
try:
@@ -3624,7 +3617,10 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
HG.client_controller.Write( 'content_updates', service_keys_to_content_updates )
- finally: self.EndModal( wx.ID_OK )
+ finally:
+
+ self.EndModal( wx.ID_OK )
+
def EventServiceChanged( self, event ):
@@ -4080,6 +4076,11 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
return ( self._service_key, content_updates )
+ def HasUncommittedPair( self ):
+
+ return len( self._children.GetTags() ) > 0 and len( self._parents.GetTags() ) > 0
+
+
def SetTagBoxFocus( self ):
if len( self._children.GetTags() ) == 0: self._child_input.SetFocus()
@@ -4215,6 +4216,19 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
def EventOK( self, event ):
+ if self._tag_repositories.GetCurrentPage().HasUncommittedPair():
+
+ message = 'Are you sure you want to OK? You have an uncommitted pair.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
+
+ if dlg.ShowModal() != wx.ID_YES:
+
+ return
+
+
+
+
service_keys_to_content_updates = {}
try:
@@ -4739,6 +4753,11 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
return ( self._service_key, content_updates )
+ def HasUncommittedPair( self ):
+
+ return len( self._old_siblings.GetTags() ) > 0 and self._current_new is not None
+
+
def SetNew( self, new_tags ):
if len( new_tags ) == 0:
@@ -4763,8 +4782,14 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
def SetTagBoxFocus( self ):
- if len( self._old_siblings.GetTags() ) == 0: self._old_input.SetFocus()
- else: self._new_input.SetFocus()
+ if len( self._old_siblings.GetTags() ) == 0:
+
+ self._old_input.SetFocus()
+
+ else:
+
+ self._new_input.SetFocus()
+
def THREADInitialise( self, tags ):
diff --git a/include/ClientGUIManagement.py b/include/ClientGUIManagement.py
index 4b0ba36c..0034eb1e 100755
--- a/include/ClientGUIManagement.py
+++ b/include/ClientGUIManagement.py
@@ -937,9 +937,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self.SetSizer( vbox )
- self.Bind( wx.EVT_TIMER, self.TIMEREventUpdateDBJob, id = ID_TIMER_UPDATE )
- self._update_db_job_timer = wx.Timer( self, id = ID_TIMER_UPDATE )
-
HG.client_controller.sub( self, 'RefreshAndUpdateStatus', 'refresh_dupe_numbers' )
@@ -978,9 +975,13 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _RebalanceTree( self ):
- self._job = 'branches'
+ job_key = ClientThreading.JobKey( cancellable = True )
- self._StartStopDBJob()
+ self._controller.Write( 'maintain_similar_files_tree', job_key = job_key )
+
+ self._controller.pub( 'modal_message', job_key )
+
+ self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _RefreshAndUpdateStatus( self ):
@@ -994,9 +995,13 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _RegeneratePhashes( self ):
- self._job = 'phashes'
+ job_key = ClientThreading.JobKey( cancellable = True )
- self._StartStopDBJob()
+ self._controller.Write( 'maintain_similar_files_phashes', job_key = job_key )
+
+ self._controller.pub( 'modal_message', job_key )
+
+ self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _ResetUnknown( self ):
@@ -1017,9 +1022,15 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _SearchForDuplicates( self ):
- self._job = 'search'
+ job_key = ClientThreading.JobKey( cancellable = True )
- self._StartStopDBJob()
+ search_distance = self._search_distance_spinctrl.GetValue()
+
+ self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = job_key )
+
+ self._controller.pub( 'modal_message', job_key )
+
+ self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _SetFileDomain( self, service_key ):
@@ -1054,7 +1065,7 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
message += os.linesep
message += '3 - Walking through the pairs or groups of potential duplicates and telling the client how they are related.'
message += os.linesep * 2
- message += 'For the first two steps, you likely just want to click the play buttons and wait for them to complete. They are very CPU intensive and lock the database heavily as they work. If you want to use the client for anything else while they are running, pause them first. You can also set them to run in idle time from the cog icon. For the search \'distance\', start at the fast and limited \'exact match\', or 0 \'hamming distance\' search and slowly expand it as you gain experience with the system.'
+ message += 'For the first two steps, you likely just want to click the play buttons and wait for them to complete. They are CPU intensive and lock the client as they work. You can also set them to run in idle time from the cog icon. For the search \'distance\', start at the fast and limited \'exact match\' (0 \'hamming distance\') and slowly expand it as you gain experience with the system.'
message += os.linesep * 2
message += 'Once you have found some potential pairs, you can either show some random groups as thumbnails (and process them manually however you prefer), or you can launch the specialised duplicate filter, which lets you quickly assign duplicate status to pairs of files and will automatically merge files and tags between dupes however you prefer.'
message += os.linesep * 2
@@ -1092,125 +1103,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._controller.pub( 'swap_media_panel', self._page_key, panel )
- def _StartStopDBJob( self ):
-
- if self._job_key is None:
-
- self._cog_button.Disable()
- self._phashes_button.Disable()
- self._branches_button.Disable()
- self._search_button.Disable()
- self._search_distance_button.Disable()
- self._search_distance_spinctrl.Disable()
- self._show_some_dupes.Disable()
- self._launch_filter.Disable()
-
- self._job_key = ClientThreading.JobKey( cancellable = True )
-
- if self._job == 'phashes':
-
- self._phashes_button.Enable()
- self._phashes_button.SetBitmap( CC.GlobalBMPs.stop )
-
- self._controller.Write( 'maintain_similar_files_phashes', job_key = self._job_key )
-
- elif self._job == 'branches':
-
- self._branches_button.Enable()
- self._branches_button.SetBitmap( CC.GlobalBMPs.stop )
-
- self._controller.Write( 'maintain_similar_files_tree', job_key = self._job_key )
-
- elif self._job == 'search':
-
- self._search_button.Enable()
- self._search_button.SetBitmap( CC.GlobalBMPs.stop )
-
- search_distance = self._search_distance_spinctrl.GetValue()
-
- self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = self._job_key )
-
-
- self._update_db_job_timer.Start( 250, wx.TIMER_CONTINUOUS )
-
- else:
-
- self._job_key.Cancel()
-
-
-
- def _UpdateJob( self ):
-
- if self._in_break:
-
- if HG.client_controller.DBCurrentlyDoingJob():
-
- return
-
- else:
-
- self._in_break = False
-
- self._StartStopDBJob()
-
- return
-
-
-
- if self._job_key.TimeRunning() > 10:
-
- self._job_key.Cancel()
-
- self._job_key = None
-
- self._in_break = True
-
- return
-
-
- if self._job_key.IsDone():
-
- self._job_key = None
-
- self._update_db_job_timer.Stop()
-
- self._RefreshAndUpdateStatus()
-
- return
-
-
- if self._job == 'phashes':
-
- text = self._job_key.GetIfHasVariable( 'popup_text_1' )
-
- if text is not None:
-
- self._num_phashes_to_regen.SetLabelText( text )
-
-
- elif self._job == 'branches':
-
- text = self._job_key.GetIfHasVariable( 'popup_text_1' )
-
- if text is not None:
-
- self._num_branches_to_regen.SetLabelText( text )
-
-
- elif self._job == 'search':
-
- text = self._job_key.GetIfHasVariable( 'popup_text_1' )
- gauge = self._job_key.GetIfHasVariable( 'popup_gauge_1' )
-
- if text is not None and gauge is not None:
-
- ( value, range ) = gauge
-
- self._num_searched.SetValue( text, value, range )
-
-
-
-
def _UpdateStatus( self ):
( num_phashes_to_regen, num_branches_to_regen, searched_distances_to_count, duplicate_types_to_count ) = self._similar_files_maintenance_status
@@ -1221,7 +1113,7 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._branches_button.SetBitmap( CC.GlobalBMPs.play )
self._search_button.SetBitmap( CC.GlobalBMPs.play )
- total_num_files = sum( searched_distances_to_count.values() )
+ total_num_files = max( num_phashes_to_regen, sum( searched_distances_to_count.values() ) )
if num_phashes_to_regen == 0:
@@ -1312,6 +1204,21 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
+ def _THREADWaitOnJob( self, job_key ):
+
+ while not job_key.IsDone():
+
+ if HG.model_shutdown:
+
+ return
+
+
+ time.sleep( 0.25 )
+
+
+ self._RefreshAndUpdateStatus()
+
+
def EventSearchDistanceChanged( self, event ):
self._UpdateStatus()
@@ -1322,11 +1229,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._RefreshAndUpdateStatus()
- def TIMEREventUpdateDBJob( self, event ):
-
- self._UpdateJob()
-
-
management_panel_types_to_classes[ MANAGEMENT_TYPE_DUPLICATE_FILTER ] = ManagementPanelDuplicateFilter
class ManagementPanelImporter( ManagementPanel ):
@@ -2169,7 +2071,7 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
def EventPaste( self, event ):
-
+
if wx.TheClipboard.Open():
data = wx.TextDataObject()
@@ -2627,8 +2529,10 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
self._file_download_control = ClientGUIControls.NetworkJobControl( self._url_panel )
self._overall_gauge = ClientGUICommon.Gauge( self._url_panel )
- self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._url_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
- self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
+ self._urls_import = self._management_controller.GetVariable( 'urls_import' )
+
+ # replace all this with a seed cache panel sometime
+ self._seed_cache_button = ClientGUISeedCache.SeedCacheButton( self, self._controller, self._urls_import.GetSeedCache )
self._url_input = wx.TextCtrl( self._url_panel, style = wx.TE_PROCESS_ENTER )
self._url_input.Bind( wx.EVT_KEY_DOWN, self.EventKeyDown )
@@ -2636,8 +2540,6 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
self._url_paste = wx.Button( self._url_panel, label = 'paste urls' )
self._url_paste.Bind( wx.EVT_BUTTON, self.EventPaste )
- self._urls_import = self._management_controller.GetVariable( 'urls_import' )
-
file_import_options = self._urls_import.GetOptions()
self._file_import_options = ClientGUIImport.FileImportOptionsButton( self._url_panel, file_import_options, self._urls_import.SetFileImportOptions )
@@ -2681,20 +2583,6 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
HG.client_controller.sub( self, 'SetURLInput', 'set_page_url_input' )
- def _SeedCache( self ):
-
- seed_cache = self._urls_import.GetSeedCache()
-
- title = 'file import status'
- frame_key = 'file_import_status'
-
- frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
-
- panel = ClientGUISeedCache.EditSeedCachePanel( frame, self._controller, seed_cache )
-
- frame.SetPanel( panel )
-
-
def _UpdateStatus( self ):
( ( overall_status, ( overall_value, overall_range ) ), paused ) = self._urls_import.GetStatus()
diff --git a/include/ClientGUIMedia.py b/include/ClientGUIMedia.py
index a29bf371..562d993e 100755
--- a/include/ClientGUIMedia.py
+++ b/include/ClientGUIMedia.py
@@ -157,6 +157,20 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
+ def _ArchiveDeleteFilter( self ):
+
+ media_results = self.GenerateMediaResults( discriminant = CC.DISCRIMINANT_LOCAL_BUT_NOT_IN_TRASH, selected_media = set( self._selected_media ), for_media_viewer = True )
+
+ if len( media_results ) > 0:
+
+ canvas_frame = ClientGUICanvas.CanvasFrame( self.GetTopLevelParent() )
+
+ canvas_window = ClientGUICanvas.CanvasMediaListFilterArchiveDelete( canvas_frame, self._page_key, media_results )
+
+ canvas_frame.SetCanvas( canvas_window )
+
+
+
def _CopyBMPToClipboard( self ):
media = self._focussed_media.GetDisplayMedia()
@@ -520,20 +534,6 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
- def _ArchiveDeleteFilter( self ):
-
- media_results = self.GenerateMediaResults( discriminant = CC.DISCRIMINANT_LOCAL_BUT_NOT_IN_TRASH, selected_media = set( self._selected_media ), for_media_viewer = True )
-
- if len( media_results ) > 0:
-
- canvas_frame = ClientGUICanvas.CanvasFrame( self.GetTopLevelParent() )
-
- canvas_window = ClientGUICanvas.CanvasMediaListFilterArchiveDelete( canvas_frame, self._page_key, media_results )
-
- canvas_frame.SetCanvas( canvas_window )
-
-
-
def _GetNumSelected( self ):
return sum( [ media.GetNumFiles() for media in self._selected_media ] )
@@ -1279,6 +1279,21 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
media_pairs = list( itertools.combinations( flat_media, 2 ) )
+ if len( media_pairs ) > 100:
+
+ message = 'The duplicate system does not yet work well for large groups of duplicates. This is about to ask if you want to apply a dupe status for more than 100 pairs.'
+ message += os.linesep * 2
+ message += 'Unless you are testing the system or have another good reason to try this, I recommend you step back for now.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message, yes_label = 'I know what I am doing', no_label = 'step back for now' ) as dlg:
+
+ if dlg.ShowModal() != wx.ID_YES:
+
+ return
+
+
+
+
message = 'Are you sure you want to ' + yes_no_text + ' for the ' + HydrusData.ConvertIntToPrettyString( len( media_pairs ) ) + ' pairs?'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
@@ -1432,13 +1447,13 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
if hashes is not None and len( hashes ) > 0:
- HG.client_controller.pub( 'new_page_query', self._file_service_key, initial_hashes = hashes )
+ HG.client_controller.pub( 'new_page_query', self._file_service_key, initial_hashes = hashes, do_sort = True )
def _ShowSelectionInNewPage( self ):
- hashes = self._GetSelectedHashes()
+ hashes = self._GetSelectedHashes( ordered = True )
if hashes is not None and len( hashes ) > 0:
diff --git a/include/ClientGUIPages.py b/include/ClientGUIPages.py
index bd43304f..d435e433 100755
--- a/include/ClientGUIPages.py
+++ b/include/ClientGUIPages.py
@@ -9,6 +9,7 @@ import ClientGUIMenus
import ClientGUICanvas
import ClientDownloading
import ClientSearch
+import hashlib
import HydrusData
import HydrusExceptions
import HydrusSerialisable
@@ -729,6 +730,8 @@ class PagesNotebook( wx.Notebook ):
self._page_key = HydrusData.GenerateKey()
+ self._last_last_session_hash = None
+
self._controller.sub( self, 'RefreshPageName', 'refresh_page_name' )
self._controller.sub( self, 'NotifyPageUnclosed', 'notify_page_unclosed' )
@@ -1773,6 +1776,9 @@ class PagesNotebook( wx.Notebook ):
page_name = page.GetName()
+ # in some unusual circumstances, this gets out of whack
+ insertion_index = min( insertion_index, self.GetPageCount() )
+
self.InsertPage( insertion_index, page, page_name, select = True )
self._controller.pub( 'refresh_page_name', page.GetPageKey() )
@@ -1838,7 +1844,7 @@ class PagesNotebook( wx.Notebook ):
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook )
- def NewPageQuery( self, file_service_key, initial_hashes = None, initial_predicates = None, page_name = None, on_deepest_notebook = False ):
+ def NewPageQuery( self, file_service_key, initial_hashes = None, initial_predicates = None, page_name = None, on_deepest_notebook = False, do_sort = False ):
if initial_hashes is None:
@@ -1870,7 +1876,14 @@ class PagesNotebook( wx.Notebook ):
management_controller = ClientGUIManagement.CreateManagementControllerQuery( page_name, file_service_key, file_search_context, search_enabled )
- return self.NewPage( management_controller, initial_hashes = initial_hashes, on_deepest_notebook = on_deepest_notebook )
+ page = self.NewPage( management_controller, initial_hashes = initial_hashes, on_deepest_notebook = on_deepest_notebook )
+
+ if do_sort:
+
+ HG.client_controller.pub( 'do_page_sort', page.GetPageKey() )
+
+
+ return page
def NewPagesNotebook( self, name = 'pages', forced_insertion_index = None, on_deepest_notebook = False, give_it_a_blank_page = True ):
@@ -2200,6 +2213,8 @@ class PagesNotebook( wx.Notebook ):
+ #
+
session = GUISession( name )
for page in self._GetPages():
@@ -2207,6 +2222,20 @@ class PagesNotebook( wx.Notebook ):
session.AddPage( page )
+ #
+
+ if name == 'last session':
+
+ session_hash = hashlib.sha256( session.DumpToString() ).digest()
+
+ if session_hash == self._last_last_session_hash:
+
+ return
+
+
+ self._last_last_session_hash = session_hash
+
+
self._controller.Write( 'serialisable', session )
self._controller.pub( 'notify_new_sessions' )
diff --git a/include/ClientGUIScrolledPanels.py b/include/ClientGUIScrolledPanels.py
index deb40d0d..bbe52d3d 100644
--- a/include/ClientGUIScrolledPanels.py
+++ b/include/ClientGUIScrolledPanels.py
@@ -20,6 +20,11 @@ class ResizingScrolledPanel( wx.lib.scrolledpanel.ScrolledPanel ):
class EditPanel( ResizingScrolledPanel ):
+ def CanCancel( self ):
+
+ return True
+
+
def GetValue( self ):
raise NotImplementedError()
@@ -27,6 +32,11 @@ class EditPanel( ResizingScrolledPanel ):
class ManagePanel( ResizingScrolledPanel ):
+ def CanCancel( self ):
+
+ return True
+
+
def CommitChanges( self ):
raise NotImplementedError()
diff --git a/include/ClientGUIScrolledPanelsEdit.py b/include/ClientGUIScrolledPanelsEdit.py
index 5a9eac9b..0b4ee7c0 100644
--- a/include/ClientGUIScrolledPanelsEdit.py
+++ b/include/ClientGUIScrolledPanelsEdit.py
@@ -1859,7 +1859,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
self._paused = wx.CheckBox( self._control_panel )
- self._retry_failed = ClientGUICommon.BetterButton( self._control_panel, 'retry failed', self.RetryFailed )
+ self._retry_failures = ClientGUICommon.BetterButton( self._control_panel, 'retry failed', self.RetryFailures )
self._check_now_button = ClientGUICommon.BetterButton( self._control_panel, 'force check on dialog ok', self.CheckNow )
@@ -1964,7 +1964,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
gridbox = ClientGUICommon.WrapInGrid( self._control_panel, rows )
self._control_panel.AddF( gridbox, CC.FLAGS_LONE_BUTTON )
- self._control_panel.AddF( self._retry_failed, CC.FLAGS_LONE_BUTTON )
+ self._control_panel.AddF( self._retry_failures, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._check_now_button, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._reset_cache_button, CC.FLAGS_LONE_BUTTON )
@@ -2041,11 +2041,11 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
if no_failures:
- self._retry_failed.Disable()
+ self._retry_failures.Disable()
else:
- self._retry_failed.Enable()
+ self._retry_failures.Enable()
if can_check:
@@ -2202,11 +2202,9 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
- def RetryFailed( self ):
+ def RetryFailures( self ):
- failed_seeds = self._seed_cache.GetSeeds( CC.STATUS_FAILED )
-
- self._seed_cache.UpdateSeedsStatus( failed_seeds, CC.STATUS_UNKNOWN )
+ self._seed_cache.RetryFailures()
self._last_error = 0
diff --git a/include/ClientGUIScrolledPanelsManagement.py b/include/ClientGUIScrolledPanelsManagement.py
index 1d493ae8..4bbd3889 100644
--- a/include/ClientGUIScrolledPanelsManagement.py
+++ b/include/ClientGUIScrolledPanelsManagement.py
@@ -1867,6 +1867,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._permit_watchers_to_name_their_pages = wx.CheckBox( thread_checker )
+ self._thread_watcher_not_found_page_string = ClientGUICommon.NoneableTextCtrl( thread_checker, none_phrase = 'do not show' )
+ self._thread_watcher_dead_page_string = ClientGUICommon.NoneableTextCtrl( thread_checker, none_phrase = 'do not show' )
+
watcher_options = self._new_options.GetDefaultThreadWatcherOptions()
self._thread_watcher_options = ClientGUIScrolledPanelsEdit.EditWatcherOptions( thread_checker, watcher_options )
@@ -1879,6 +1882,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._permit_watchers_to_name_their_pages.SetValue( self._new_options.GetBoolean( 'permit_watchers_to_name_their_pages' ) )
+ self._thread_watcher_not_found_page_string.SetValue( self._new_options.GetNoneableString( 'thread_watcher_not_found_page_string' ) )
+ self._thread_watcher_dead_page_string.SetValue( self._new_options.GetNoneableString( 'thread_watcher_dead_page_string' ) )
+
#
rows = []
@@ -1899,6 +1905,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows = []
rows.append( ( 'Permit thread checkers to name their own pages:', self._permit_watchers_to_name_their_pages ) )
+ rows.append( ( 'Prepend thread checker page names with this on 404:', self._thread_watcher_not_found_page_string ) )
+ rows.append( ( 'Prepend thread checker page names with this on death:', self._thread_watcher_dead_page_string ) )
gridbox = ClientGUICommon.WrapInGrid( thread_checker, rows )
@@ -1925,6 +1933,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetDefaultThreadWatcherOptions( self._thread_watcher_options.GetValue() )
+ self._new_options.SetNoneableString( 'thread_watcher_not_found_page_string', self._thread_watcher_not_found_page_string.GetValue() )
+ self._new_options.SetNoneableString( 'thread_watcher_dead_page_string', self._thread_watcher_dead_page_string.GetValue() )
+
class _MaintenanceAndProcessingPanel( wx.Panel ):
@@ -2478,6 +2489,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._default_gui_session = wx.Choice( self )
+ self._last_session_save_period_minutes = wx.SpinCtrl( self, min = 1, max = 1440 )
+
self._default_new_page_goes = ClientGUICommon.BetterChoice( self )
for value in [ CC.NEW_PAGE_GOES_FAR_LEFT, CC.NEW_PAGE_GOES_LEFT_OF_CURRENT, CC.NEW_PAGE_GOES_RIGHT_OF_CURRENT, CC.NEW_PAGE_GOES_FAR_RIGHT ]:
@@ -2538,14 +2551,28 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
gui_session_names = HG.client_controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION )
- if 'last session' not in gui_session_names: gui_session_names.insert( 0, 'last session' )
+ if 'last session' not in gui_session_names:
+
+ gui_session_names.insert( 0, 'last session' )
+
self._default_gui_session.Append( 'just a blank page', None )
- for name in gui_session_names: self._default_gui_session.Append( name, name )
+ for name in gui_session_names:
+
+ self._default_gui_session.Append( name, name )
+
- try: self._default_gui_session.SetStringSelection( HC.options[ 'default_gui_session' ] )
- except: self._default_gui_session.SetSelection( 0 )
+ try:
+
+ self._default_gui_session.SetStringSelection( HC.options[ 'default_gui_session' ] )
+
+ except:
+
+ self._default_gui_session.SetSelection( 0 )
+
+
+ self._last_session_save_period_minutes.SetValue( self._new_options.GetInteger( 'last_session_save_period_minutes' ) )
self._default_new_page_goes.SelectClientData( self._new_options.GetInteger( 'default_new_page_goes' ) )
@@ -2595,6 +2622,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'Main gui title: ', self._main_gui_title ) )
rows.append( ( 'Default session on startup: ', self._default_gui_session ) )
+ rows.append( ( 'If \'last session\' above, autosave it how often (minutes)?', self._last_session_save_period_minutes ) )
rows.append( ( 'By default, new page tabs: ', self._default_new_page_goes ) )
rows.append( ( 'Confirm client exit: ', self._confirm_client_exit ) )
rows.append( ( 'Confirm sending files to trash: ', self._confirm_trash ) )
@@ -2686,6 +2714,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetString( 'main_gui_title', title )
+ self._new_options.SetInteger( 'last_session_save_period_minutes', self._last_session_save_period_minutes.GetValue() )
+
self._new_options.SetInteger( 'default_new_page_goes', self._default_new_page_goes.GetChoice() )
self._new_options.SetInteger( 'max_page_name_chars', self._max_page_name_chars.GetValue() )
@@ -2737,6 +2767,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._use_system_ffmpeg = wx.CheckBox( self )
self._use_system_ffmpeg.SetToolTipString( 'Check this to always default to the system ffmpeg in your path, rather than using the static ffmpeg in hydrus\'s bin directory. (requires restart)' )
+ self._anchor_and_hide_canvas_drags = wx.CheckBox( self )
+
self._media_zooms = wx.TextCtrl( self )
self._media_zooms.Bind( wx.EVT_TEXT, self.EventZoomsChanged )
@@ -2754,6 +2786,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._load_images_with_pil.SetValue( self._new_options.GetBoolean( 'load_images_with_pil' ) )
self._do_not_import_decompression_bombs.SetValue( self._new_options.GetBoolean( 'do_not_import_decompression_bombs' ) )
self._use_system_ffmpeg.SetValue( self._new_options.GetBoolean( 'use_system_ffmpeg' ) )
+ self._anchor_and_hide_canvas_drags.SetValue( self._new_options.GetBoolean( 'anchor_and_hide_canvas_drags' ) )
media_zooms = self._new_options.GetMediaZooms()
@@ -2785,6 +2818,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'Load images with PIL: ', self._load_images_with_pil ) )
rows.append( ( 'Do not import Decompression Bombs: ', self._do_not_import_decompression_bombs ) )
rows.append( ( 'Prefer system FFMPEG: ', self._use_system_ffmpeg ) )
+ rows.append( ( 'WINDOWS ONLY: Hide and anchor mouse cursor on slow canvas drags: ', self._anchor_and_hide_canvas_drags ) )
rows.append( ( 'Media zooms: ', self._media_zooms ) )
gridbox = ClientGUICommon.WrapInGrid( self, rows )
@@ -2882,6 +2916,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetBoolean( 'load_images_with_pil', self._load_images_with_pil.GetValue() )
self._new_options.SetBoolean( 'do_not_import_decompression_bombs', self._do_not_import_decompression_bombs.GetValue() )
self._new_options.SetBoolean( 'use_system_ffmpeg', self._use_system_ffmpeg.GetValue() )
+ self._new_options.SetBoolean( 'anchor_and_hide_canvas_drags', self._anchor_and_hide_canvas_drags.GetValue() )
try:
@@ -3072,6 +3107,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
disk_panel = ClientGUICommon.StaticBox( self, 'disk cache' )
+ disk_cache_help_button = ClientGUICommon.BetterBitmapButton( disk_panel, CC.GlobalBMPs.help, self._ShowDiskCacheHelp )
+ disk_cache_help_button.SetToolTipString( 'Show help regarding the disk cache.' )
+
self._disk_cache_init_period = ClientGUICommon.NoneableSpinCtrl( disk_panel, 'max disk cache init period', none_phrase = 'do not run', min = 1, max = 120 )
self._disk_cache_init_period.SetToolTipString( 'When the client boots, it can speed up operation by reading the front of the database into memory. This sets the max number of seconds it can spend doing that.' )
@@ -3165,8 +3203,19 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
#
+
+ help_hbox = wx.BoxSizer( wx.HORIZONTAL )
+
+ st = ClientGUICommon.BetterStaticText( disk_panel, 'help for this panel -->' )
+
+ st.SetForegroundColour( wx.Colour( 0, 0, 255 ) )
+
+ help_hbox.AddF( st, CC.FLAGS_VCENTER )
+ help_hbox.AddF( disk_cache_help_button, CC.FLAGS_VCENTER )
+
vbox = wx.BoxSizer( wx.VERTICAL )
+ disk_panel.AddF( help_hbox, CC.FLAGS_LONE_BUTTON )
disk_panel.AddF( self._disk_cache_init_period, CC.FLAGS_EXPAND_PERPENDICULAR )
disk_panel.AddF( self._disk_cache_maintenance_mb, CC.FLAGS_EXPAND_PERPENDICULAR )
@@ -3268,6 +3317,21 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
wx.CallAfter( self.Layout ) # draws the static texts correctly
+ def _ShowDiskCacheHelp( self ):
+
+ message = 'The hydrus database runs best on a drive with fast random access latency. Important and heavy read and write operations can function up to 100 times faster when started raw from an SSD rather than an HDD.'
+ message += os.linesep * 2
+ message += 'To get around this, the client populates a pre-boot and ongoing disk cache. By contiguously frontloading the database into memory, the most important functions do not need to wait on your disk for most of their work.'
+ message += os.linesep * 2
+ message += 'If you tend to leave your client on in the background and have a slow drive but a lot of ram, you might like to pump these numbers up. 15s boot cache and 2048MB ongoing can really make a difference on, for instance, a slow laptop drive.'
+ message += os.linesep * 2
+ message += 'If you run the database from an SSD, you can reduce or entirely eliminate these values, as the benefit is not so stark. 2s and 256MB is fine.'
+ message += os.linesep * 2
+ message += 'Unless you are testing, do not go crazy with this stuff. You can set 8192MB if you like, but there are diminishing returns.'
+
+ wx.MessageBox( message )
+
+
def EventFetchAuto( self, event ):
if self._fetch_ac_results_automatically.GetValue() == True:
@@ -5253,12 +5317,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
seed_cache = subscription.GetSeedCache()
- failed_seeds = seed_cache.GetSeeds( CC.STATUS_FAILED )
-
- for seed in failed_seeds:
-
- seed_cache.UpdateSeedStatus( seed, CC.STATUS_UNKNOWN )
-
+ seed_cache.RetryFailures()
self._subscriptions.UpdateDatas( subscriptions )
@@ -5338,6 +5397,23 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
+ def _GetServiceKeysToContentUpdates( self ):
+
+ service_keys_to_content_updates = {}
+
+ for page in self._tag_repositories.GetActivePages():
+
+ ( service_key, content_updates ) = page.GetContentUpdates()
+
+ if len( content_updates ) > 0:
+
+ service_keys_to_content_updates[ service_key ] = content_updates
+
+
+
+ return service_keys_to_content_updates
+
+
def _OKParent( self ):
wx.PostEvent( self.GetParent(), wx.CommandEvent( commandType = wx.wxEVT_COMMAND_MENU_SELECTED, winid = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'ok' ) ) )
@@ -5413,19 +5489,29 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
+ def CanCancel( self ):
+
+ service_keys_to_content_updates = self._GetServiceKeysToContentUpdates()
+
+ if len( service_keys_to_content_updates ) > 0:
+
+ message = 'Are you sure you want to cancel? You have uncommitted changes that will be lost.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
+
+ if dlg.ShowModal() != wx.ID_YES:
+
+ return False
+
+
+
+
+ return True
+
+
def CommitChanges( self ):
- service_keys_to_content_updates = {}
-
- for page in self._tag_repositories.GetActivePages():
-
- ( service_key, content_updates ) = page.GetContentUpdates()
-
- if len( content_updates ) > 0:
-
- service_keys_to_content_updates[ service_key ] = content_updates
-
-
+ service_keys_to_content_updates = self._GetServiceKeysToContentUpdates()
if len( service_keys_to_content_updates ) > 0:
@@ -6121,6 +6207,9 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._url_input = wx.TextCtrl( self, style = wx.TE_PROCESS_ENTER )
self._url_input.Bind( wx.EVT_CHAR_HOOK, self.EventInputCharHook )
+ self._copy_button = ClientGUICommon.BetterButton( self, 'copy all', self._Copy )
+ self._paste_button = ClientGUICommon.BetterButton( self, 'paste', self._Paste )
+
self._urls_to_add = set()
self._urls_to_remove = set()
@@ -6139,20 +6228,42 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
#
+ hbox = wx.BoxSizer( wx.HORIZONTAL )
+
+ hbox.AddF( self._copy_button, CC.FLAGS_VCENTER )
+ hbox.AddF( self._paste_button, CC.FLAGS_VCENTER )
+
vbox = wx.BoxSizer( wx.VERTICAL )
vbox.AddF( self._urls_listbox, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.AddF( self._url_input, CC.FLAGS_EXPAND_PERPENDICULAR )
+ vbox.AddF( hbox, CC.FLAGS_BUTTON_SIZER )
self.SetSizer( vbox )
wx.CallAfter( self._url_input.SetFocus )
- def _EnterURL( self, url ):
+ def _Copy( self ):
+
+ urls = list( self._current_urls )
+
+ urls.sort()
+
+ text = os.linesep.join( urls )
+
+ HG.client_controller.pub( 'clipboard', 'text', text )
+
+
+ def _EnterURL( self, url, only_add = False ):
if url in self._current_urls:
+ if only_add:
+
+ return
+
+
for index in range( self._urls_listbox.GetCount() ):
existing_url = self._urls_listbox.GetClientData( index )
@@ -6178,6 +6289,39 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
+ def _Paste( self ):
+
+ if wx.TheClipboard.Open():
+
+ data = wx.TextDataObject()
+
+ wx.TheClipboard.GetData( data )
+
+ wx.TheClipboard.Close()
+
+ raw_text = data.GetText()
+
+ try:
+
+ for url in HydrusData.SplitByLinesep( raw_text ):
+
+ if url != '':
+
+ self._EnterURL( url, only_add = True )
+
+
+
+ except:
+
+ wx.MessageBox( 'I could not understand what was in the clipboard' )
+
+
+ else:
+
+ wx.MessageBox( 'I could not get permission to access the clipboard.' )
+
+
+
def _RemoveURL( self, index ):
url = self._urls_listbox.GetClientData( index )
diff --git a/include/ClientGUISeedCache.py b/include/ClientGUISeedCache.py
index 780f64a3..caff0b58 100644
--- a/include/ClientGUISeedCache.py
+++ b/include/ClientGUISeedCache.py
@@ -230,6 +230,160 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ):
+class SeedCacheButton( ClientGUICommon.BetterBitmapButton ):
+
+ def __init__( self, parent, controller, seed_cache_get_callable, seed_cache_set_callable = None ):
+
+ ClientGUICommon.BetterBitmapButton.__init__( self, parent, CC.GlobalBMPs.seed_cache, self._ShowSeedCacheFrame )
+
+ self._controller = controller
+ self._seed_cache_get_callable = seed_cache_get_callable
+ self._seed_cache_set_callable = seed_cache_set_callable
+
+ self.SetToolTipString( 'open detailed file import status--right-click for quick actions, if applicable' )
+
+ self.Bind( wx.EVT_RIGHT_DOWN, self.EventShowMenu )
+
+
+ def _ClearProcessed( self ):
+
+ message = 'Are you sure you want to delete all the processed (i.e. anything with a non-blank status in the larger window) files? This is useful for cleaning up and de-laggifying a very large list, but not much else.'
+
+ with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
+
+ if dlg.ShowModal() == wx.ID_YES:
+
+ seed_cache = self._seed_cache_get_callable()
+
+ seed_cache.RemoveProcessedSeeds()
+
+
+
+
+ def _RetryFailures( self ):
+
+ message = 'Are you sure you want to retry all the failed files?'
+
+ with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
+
+ if dlg.ShowModal() == wx.ID_YES:
+
+ seed_cache = self._seed_cache_get_callable()
+
+ seed_cache.RetryFailures()
+
+
+
+
+ def _ShowSeedCacheFrame( self ):
+
+ seed_cache = self._seed_cache_get_callable()
+
+ tlp = ClientGUICommon.GetTLP( self )
+
+ if isinstance( tlp, wx.Dialog ):
+
+ if self._seed_cache_set_callable is None: # throw up a dialog that edits the seed cache in place
+
+ with ClientGUITopLevelWindows.DialogNullipotent( self, 'file import status' ) as dlg:
+
+ panel = EditSeedCachePanel( dlg, self._controller, seed_cache )
+
+ dlg.SetPanel( panel )
+
+ dlg.ShowModal()
+
+
+ else: # throw up a dialog that edits the seed cache but can be cancelled
+
+ dupe_seed_cache = seed_cache.Duplicate()
+
+ with ClientGUITopLevelWindows.DialogEdit( self, 'file import status' ) as dlg:
+
+ panel = EditSeedCachePanel( dlg, self._controller, dupe_seed_cache )
+
+ dlg.SetPanel( panel )
+
+ if dlg.ShowModal() == wx.ID_OK:
+
+ self._seed_cache_set_callable( dupe_seed_cache )
+
+
+
+
+ else: # throw up a frame that edits the seed cache in place
+
+ title = 'file import status'
+ frame_key = 'file_import_status'
+
+ frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
+
+ panel = EditSeedCachePanel( frame, self._controller, seed_cache )
+
+ frame.SetPanel( panel )
+
+
+
+ def EventShowMenu( self, event ):
+
+ seed_cache = self._seed_cache_get_callable()
+
+ menu_items = []
+
+ num_failures = seed_cache.GetSeedCount( CC.STATUS_FAILED )
+
+ if num_failures > 0:
+
+ menu_items.append( ( 'normal', 'retry ' + HydrusData.ConvertIntToPrettyString( num_failures ) + ' failures', 'Tell this cache to reattempt all its failures.', self._RetryFailures ) )
+
+
+ num_unknown = seed_cache.GetSeedCount( CC.STATUS_UNKNOWN )
+
+ num_processed = len( seed_cache ) - num_unknown
+
+ if num_processed > 0:
+
+ menu_items.append( ( 'normal', 'delete ' + HydrusData.ConvertIntToPrettyString( num_processed ) + ' \'processed\' files from the queue', 'Tell this cache to clear out processed files, reducing the size of the queue.', self._ClearProcessed ) )
+
+
+ if len( menu_items ) > 0:
+
+ menu = wx.Menu()
+
+ for ( item_type, title, description, data ) in menu_items:
+
+ if item_type == 'normal':
+
+ func = data
+
+ ClientGUIMenus.AppendMenuItem( self, menu, title, description, func )
+
+ elif item_type == 'check':
+
+ check_manager = data
+
+ current_value = check_manager.GetCurrentValue()
+ func = check_manager.Invert
+
+ if current_value is not None:
+
+ ClientGUIMenus.AppendMenuCheckItem( self, menu, title, description, current_value, func )
+
+
+ elif item_type == 'separator':
+
+ ClientGUIMenus.AppendSeparator( menu )
+
+
+
+ HG.client_controller.PopupMenu( self, menu )
+
+ else:
+
+ event.Skip()
+
+
+
class SeedCacheStatusControl( wx.Panel ):
def __init__( self, parent, controller ):
@@ -243,8 +397,7 @@ class SeedCacheStatusControl( wx.Panel ):
self._import_summary_st = ClientGUICommon.BetterStaticText( self )
self._progress_st = ClientGUICommon.BetterStaticText( self )
- self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self, CC.GlobalBMPs.seed_cache, self._ShowSeedCacheFrame )
- self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
+ self._seed_cache_button = SeedCacheButton( self, self._controller, self._GetSeedCache )
self._progress_gauge = ClientGUICommon.Gauge( self )
@@ -274,32 +427,9 @@ class SeedCacheStatusControl( wx.Panel ):
self._update_timer = wx.Timer( self )
- def _ShowSeedCacheFrame( self ):
+ def _GetSeedCache( self ):
- tlp = ClientGUICommon.GetTLP( self )
-
- if isinstance( tlp, wx.Dialog ):
-
- with ClientGUITopLevelWindows.DialogNullipotent( self, 'file import status' ) as dlg:
-
- panel = EditSeedCachePanel( dlg, self._controller, self._seed_cache )
-
- dlg.SetPanel( panel )
-
- dlg.ShowModal()
-
-
- else:
-
- title = 'file import status'
- frame_key = 'file_import_status'
-
- frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
-
- panel = EditSeedCachePanel( frame, self._controller, self._seed_cache )
-
- frame.SetPanel( panel )
-
+ return self._seed_cache
def _Update( self ):
diff --git a/include/ClientGUITopLevelWindows.py b/include/ClientGUITopLevelWindows.py
index 8f52f32b..43cbf1ca 100644
--- a/include/ClientGUITopLevelWindows.py
+++ b/include/ClientGUITopLevelWindows.py
@@ -308,6 +308,11 @@ class NewDialog( wx.Dialog ):
HG.client_controller.ResetIdleTimer()
+ def _CanCancel( self ):
+
+ return True
+
+
def EventMenuClose( self, event ):
menu = event.GetMenu()
@@ -368,7 +373,17 @@ class NewDialog( wx.Dialog ):
def EventDialogButton( self, event ):
- self.EndModal( event.GetId() )
+ event_id = event.GetId()
+
+ if event_id == wx.ID_CANCEL:
+
+ if not self._CanCancel():
+
+ return
+
+
+
+ self.EndModal( event_id )
class DialogThatResizes( NewDialog ):
@@ -394,6 +409,11 @@ class DialogThatTakesScrollablePanel( DialogThatResizes ):
self.Bind( CC.EVT_SIZE_CHANGED, self.EventChildSizeChanged )
+ def _CanCancel( self ):
+
+ return self._panel.CanCancel()
+
+
def _GetButtonBox( self ):
raise NotImplementedError()
diff --git a/include/ClientImporting.py b/include/ClientImporting.py
index 95821fa3..864e89e8 100644
--- a/include/ClientImporting.py
+++ b/include/ClientImporting.py
@@ -370,7 +370,7 @@ class FileImportJob( object ):
new_options = HG.client_controller.GetNewOptions()
- if mime in HC.IMAGES and new_options.GetBoolean( 'do_not_import_decompression_bombs' ):
+ if mime in HC.DECOMPRESSION_BOMB_IMAGES and new_options.GetBoolean( 'do_not_import_decompression_bombs' ):
if HydrusImageHandling.IsDecompressionBomb( self._temp_path ):
@@ -2466,7 +2466,7 @@ HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIAL
class SeedCache( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_SEED_CACHE
- SERIALISABLE_VERSION = 6
+ SERIALISABLE_VERSION = 7
def __init__( self ):
@@ -2722,6 +2722,33 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
return ( 6, new_serialisable_info )
+ if version == 6:
+
+ new_serialisable_info = []
+
+ for ( seed, seed_info ) in old_serialisable_info:
+
+ try:
+
+ magic_phrase = '//media.tumblr.com'
+ replacement = '//data.tumblr.com'
+
+ if magic_phrase in seed:
+
+ seed = seed.replace( magic_phrase, replacement )
+
+
+ except:
+
+ pass
+
+
+ new_serialisable_info.append( ( seed, seed_info ) )
+
+
+ return ( 7, new_serialisable_info )
+
+
def AddSeeds( self, seeds ):
@@ -2990,6 +3017,35 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
+ def RemoveProcessedSeeds( self ):
+
+ with self._lock:
+
+ seeds_to_delete = set()
+
+ for ( seed, seed_info ) in self._seeds_to_info.items():
+
+ if seed_info[ 'status' ] != CC.STATUS_UNKNOWN:
+
+ seeds_to_delete.add( seed )
+
+
+
+ for seed in seeds_to_delete:
+
+ del self._seeds_to_info[ seed ]
+
+ self._seeds_ordered.remove( seed )
+
+
+ self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) }
+
+ self._SetDirty()
+
+
+ HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, seeds_to_delete )
+
+
def RemoveSeeds( self, seeds ):
with self._lock:
@@ -3041,6 +3097,13 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, seeds_to_delete )
+ def RetryFailures( self ):
+
+ failed_seeds = self.GetSeeds( CC.STATUS_FAILED )
+
+ self.UpdateSeedsStatus( failed_seeds, CC.STATUS_UNKNOWN )
+
+
def UpdateSeedSourceTime( self, seed, source_timestamp ):
# this is ugly--this should all be moved to the seed when it becomes a cleverer object, rather than jimmying it through the cache
@@ -4233,11 +4296,21 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
if self._thread_status == THREAD_STATUS_404:
- page_name = '[404] ' + page_name
+ thread_watcher_not_found_page_string = new_options.GetNoneableString( 'thread_watcher_not_found_page_string' )
+
+ if thread_watcher_not_found_page_string is not None:
+
+ page_name = thread_watcher_not_found_page_string + ' ' + page_name
+
elif self._thread_status == THREAD_STATUS_DEAD:
- page_name = '[DEAD] ' + page_name
+ thread_watcher_dead_page_string = new_options.GetNoneableString( 'thread_watcher_dead_page_string' )
+
+ if thread_watcher_dead_page_string is not None:
+
+ page_name = thread_watcher_dead_page_string + ' ' + page_name
+
if page_name != self._last_pubbed_page_name:
@@ -4274,20 +4347,20 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
else:
- if self._watcher_options.IsDead( self._urls_cache, self._last_check_time ):
+ if self._thread_status != THREAD_STATUS_404:
- if self._thread_status != THREAD_STATUS_404:
+ if self._watcher_options.IsDead( self._urls_cache, self._last_check_time ):
self._thread_status = THREAD_STATUS_DEAD
-
- self._watcher_status = ''
-
- self._thread_paused = True
-
- else:
-
- self._thread_status = THREAD_STATUS_OK
+ self._watcher_status = ''
+
+ self._thread_paused = True
+
+ else:
+
+ self._thread_status = THREAD_STATUS_OK
+
self._next_check_time = self._watcher_options.GetNextCheckTime( self._urls_cache, self._last_check_time )
diff --git a/include/ClientNetworking.py b/include/ClientNetworking.py
index 47b3f034..d7d0443f 100644
--- a/include/ClientNetworking.py
+++ b/include/ClientNetworking.py
@@ -2356,7 +2356,9 @@ class NetworkJob( object ):
else:
- return self.engine.login_manager.NeedsLogin( self._network_contexts )
+ session_network_context = self._GetSessionNetworkContext()
+
+ return self.engine.login_manager.NeedsLogin( session_network_context )
@@ -2729,73 +2731,6 @@ class NetworkJobThreadWatcher( NetworkJob ):
return self._network_contexts[-2] # the domain one
-class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
-
- SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
- SERIALISABLE_VERSION = 1
-
- def __init__( self ):
-
- HydrusSerialisable.SerialisableBase.__init__( self )
-
- self.engine = None
-
- self._lock = threading.Lock()
-
- self._network_contexts_to_logins = {}
-
- # a login has:
- # a network_context it works for (PRIMARY KEY)
- # a login script
- # rules to check validity in cookies in a current session (fold that into the login script, which may have several stages of this)
- # current user/pass/whatever
- # current script validity
- # current credentials validity
- # recent error? some way of dealing with 'domain is currently down, so try again later'
-
- # so, we fetch all the logins, ask them for the network contexts so we can set up the dict
-
-
- def _GetSerialisableInfo( self ):
-
- return {}
-
-
- def _InitialiseFromSerialisableInfo( self, serialisable_info ):
-
- self._network_contexts_to_logins = {}
-
-
- def CanLogin( self, network_contexts ):
-
- # look them up in our structure
- # if they have a login, is it valid?
- # valid means we have tested credentials and it hasn't been invalidated by a parsing error or similar
- # I think this just means saying Login.CanLogin( credentials )
-
- return False
-
-
- def GenerateLoginProcess( self, network_contexts ):
-
- # look up the logins
- # login_process = Login.GenerateLoginProcess
- # return login_process
-
- raise NotImplementedError()
-
-
- def NeedsLogin( self, network_contexts ):
-
- # look up the network contexts in our structure
- # if they have a login, see if they match the 'is logged in' predicates
- # otherwise:
-
- return False
-
-
-HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager
-
class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER
@@ -2861,6 +2796,8 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
del self._network_contexts_to_sessions[ network_context ]
+ self._SetDirty()
+
@@ -2873,6 +2810,14 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
self._network_contexts_to_sessions[ network_context ] = self._GenerateSession( network_context )
+ # tumblr can't into ssl for some reason, and the data subdomain they use has weird cert properties, looking like amazon S3
+ # perhaps it is inward-facing somehow? whatever the case, let's just say fuck it for tumblr
+
+ if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and network_context.context_data == 'tumblr.com':
+
+ self._network_contexts_to_sessions[ network_context ].verify = False
+
+
self._SetDirty()
return self._network_contexts_to_sessions[ network_context ]
diff --git a/include/ClientNetworkingLogin.py b/include/ClientNetworkingLogin.py
index 2113ed36..923e715a 100644
--- a/include/ClientNetworkingLogin.py
+++ b/include/ClientNetworkingLogin.py
@@ -1,8 +1,19 @@
import ClientConstants as CC
+import ClientDefaults
+import ClientDownloading
+import ClientNetworking
+import ClientNetworkingDomain
import HydrusConstants as HC
import HydrusGlobals as HG
import HydrusData
import HydrusExceptions
+import HydrusSerialisable
+import json
+import requests
+
+import threading
+import time
+import urllib
VALIDITY_VALID = 0
VALIDITY_UNTESTED = 1
@@ -20,8 +31,449 @@ class LoginCredentials( object ):
# test current values, including fail for not having enough/having too many
+class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
+
+ SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
+ SERIALISABLE_VERSION = 1
+
+ SESSION_TIMEOUT = 60 * 60
+
+ def __init__( self ):
+
+ HydrusSerialisable.SerialisableBase.__init__( self )
+
+ self.engine = None
+
+ self._lock = threading.Lock()
+
+ self._domains_to_logins = {}
+
+ # a login has:
+ # a login script
+ # rules to check validity in cookies in a current session (fold that into the login script, which may have several stages of this)
+ # current user/pass/whatever
+ # current script validity
+ # current credentials validity
+ # recent error? some way of dealing with 'domain is currently down, so try again later'
+
+ # so, we fetch all the logins, ask them for the network contexts so we can set up the dict
+ # variables from old object here
+ self._error_names = set()
+
+ self._network_contexts_to_session_timeouts = {}
+
+
+ def _GetSerialisableInfo( self ):
+
+ return {}
+
+
+ def _InitialiseFromSerialisableInfo( self, serialisable_info ):
+
+ self._network_contexts_to_logins = {}
+
+
+ def CanLogin( self, network_contexts ):
+
+ # look them up in our structure
+ # if they have a login, is it valid?
+ # valid means we have tested credentials and it hasn't been invalidated by a parsing error or similar
+ # I think this just means saying Login.CanLogin( credentials )
+
+ return False
+
+
+ def GenerateLoginProcess( self, network_contexts ):
+
+ # look up the logins
+ # login_process = Login.GenerateLoginProcess
+ # return login_process
+
+ raise NotImplementedError()
+
+
+ def NeedsLogin( self, network_context ):
+
+ with self._lock:
+
+ if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
+
+ nc_domain = network_context.context_data
+
+ domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( nc_domain )
+
+ for domain in domains:
+
+ if domain in self._domains_to_logins:
+
+ # fetch session
+ # does the login script reckon the session is logged in?
+ # if not, return True
+
+ pass
+
+
+
+ elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
+
+ service_key = network_context.context_data
+
+ # figure it out here
+
+
+ return False
+
+
+
+ # these methods are from the old object:
+
+ def _GetCookiesDict( self, network_context ):
+
+ session = self._GetSession( network_context )
+
+ cookies = session.cookies
+
+ cookies.clear_expired_cookies()
+
+ domains = cookies.list_domains()
+
+ for domain in domains:
+
+ if domain.endswith( network_context.context_data ):
+
+ return cookies.get_dict( domain )
+
+
+
+ return {}
+
+
+ def _GetSession( self, network_context ):
+
+ session = self.engine.controller.network_engine.session_manager.GetSession( network_context )
+
+ if network_context not in self._network_contexts_to_session_timeouts:
+
+ self._network_contexts_to_session_timeouts[ network_context ] = 0
+
+
+ if HydrusData.TimeHasPassed( self._network_contexts_to_session_timeouts[ network_context ] ):
+
+ session.cookies.clear_session_cookies()
+
+
+ self._network_contexts_to_session_timeouts[ network_context ] = HydrusData.GetNow() + self.SESSION_TIMEOUT
+
+ return session
+
+
+ def _IsLoggedIn( self, network_context, required_cookies ):
+
+ cookie_dict = self._GetCookiesDict( network_context )
+
+ for name in required_cookies:
+
+ if name not in cookie_dict:
+
+ return False
+
+
+
+ return True
+
+
+ def EnsureHydrusSessionIsOK( self, service_key ):
+
+ with self._lock:
+
+ if not self.engine.controller.services_manager.ServiceExists( service_key ):
+
+ raise HydrusExceptions.DataMissing( 'Service does not exist!' )
+
+
+ name = self.engine.controller.services_manager.GetService( service_key ).GetName()
+
+ if service_key in self._error_names:
+
+ raise Exception( 'Could not establish a hydrus network session for ' + name + '! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
+
+
+ network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, service_key )
+
+ required_cookies = [ 'session_key' ]
+
+ if self._IsLoggedIn( network_context, required_cookies ):
+
+ return
+
+
+ try:
+
+ self.SetupHydrusSession( service_key )
+
+ if not self._IsLoggedIn( network_context, required_cookies ):
+
+ return
+
+
+ HydrusData.Print( 'Successfully logged into ' + name + '.' )
+
+ except:
+
+ self._error_names.add( service_key )
+
+ raise
+
+
+
+
+ def EnsureLoggedIn( self, name ):
+
+ with self._lock:
+
+ if name in self._error_names:
+
+ raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
+
+
+ if name == 'hentai foundry':
+
+ network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
+
+ required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
+
+ elif name == 'pixiv':
+
+ network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
+
+ required_cookies = [ 'PHPSESSID' ]
+
+
+ if self._IsLoggedIn( network_context, required_cookies ):
+
+ return
+
+
+ try:
+
+ if name == 'hentai foundry':
+
+ self.LoginHF( network_context )
+
+ elif name == 'pixiv':
+
+ result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
+
+ if result is None:
+
+ raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
+
+
+ ( pixiv_id, password ) = result
+
+ self.LoginPixiv( network_context, pixiv_id, password )
+
+
+ if not self._IsLoggedIn( network_context, required_cookies ):
+
+ raise Exception( name + ' login did not work correctly!' )
+
+
+ HydrusData.Print( 'Successfully logged into ' + name + '.' )
+
+ except:
+
+ self._error_names.add( name )
+
+ raise
+
+
+
+
+ def LoginHF( self, network_context ):
+
+ session = self._GetSession( network_context )
+
+ response = session.get( 'https://www.hentai-foundry.com/' )
+
+ time.sleep( 1 )
+
+ response = session.get( 'https://www.hentai-foundry.com/?enterAgree=1' )
+
+ time.sleep( 1 )
+
+ cookie_dict = self._GetCookiesDict( network_context )
+
+ raw_csrf = cookie_dict[ 'YII_CSRF_TOKEN' ] # 19b05b536885ec60b8b37650a32f8deb11c08cd1s%3A40%3A%222917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32%22%3B
+
+ processed_csrf = urllib.unquote( raw_csrf ) # 19b05b536885ec60b8b37650a32f8deb11c08cd1s:40:"2917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32";
+
+ csrf_token = processed_csrf.split( '"' )[1] # the 2917... bit
+
+ hentai_foundry_form_info = ClientDefaults.GetDefaultHentaiFoundryInfo()
+
+ hentai_foundry_form_info[ 'YII_CSRF_TOKEN' ] = csrf_token
+
+ response = session.post( 'http://www.hentai-foundry.com/site/filters', data = hentai_foundry_form_info )
+
+ time.sleep( 1 )
+
+
+ # This updated login form is cobbled together from the example in PixivUtil2
+ # it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
+ # Pixiv 400s if cookies and referrers aren't passed correctly
+ # I am leaving this as a mess with the hope the eventual login engine will replace it
+ def LoginPixiv( self, network_context, pixiv_id, password ):
+
+ session = self._GetSession( network_context )
+
+ response = session.get( 'https://accounts.pixiv.net/login' )
+
+ soup = ClientDownloading.GetSoup( response.content )
+
+ # some whocking 20kb bit of json tucked inside a hidden form input wew lad
+ i = soup.find( 'input', id = 'init-config' )
+
+ raw_json = i['value']
+
+ j = json.loads( raw_json )
+
+ if 'pixivAccount.postKey' not in j:
+
+ raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
+
+
+ post_key = j[ 'pixivAccount.postKey' ]
+
+ form_fields = {}
+
+ form_fields[ 'pixiv_id' ] = pixiv_id
+ form_fields[ 'password' ] = password
+ form_fields[ 'captcha' ] = ''
+ form_fields[ 'g_recaptcha_response' ] = ''
+ form_fields[ 'return_to' ] = 'https://www.pixiv.net'
+ form_fields[ 'lang' ] = 'en'
+ form_fields[ 'post_key' ] = post_key
+ form_fields[ 'source' ] = 'pc'
+
+ headers = {}
+
+ headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
+ headers[ 'origin' ] = "https://accounts.pixiv.net"
+
+ session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
+
+ time.sleep( 1 )
+
+
+ def SetupHydrusSession( self, service_key ):
+
+ # nah, replace this with a proper login script
+
+ service = self.engine.controller.services_manager.GetService( service_key )
+
+ if not service.HasAccessKey():
+
+ raise HydrusExceptions.DataMissing( 'No access key for this service, so cannot set up session!' )
+
+
+ access_key = service.GetAccessKey()
+
+ url = 'blah'
+
+ network_job = ClientNetworking.NetworkJobHydrus( service_key, 'GET', url )
+
+ network_job.SetForLogin( True )
+
+ network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.encode( 'hex' ) )
+
+ self.engine.controller.network_engine.AddJob( network_job )
+
+ network_job.WaitUntilDone()
+
+
+ def TestPixiv( self, pixiv_id, password ):
+
+ # this is just an ugly copy, but fuck it for the minute
+ # we'll figure out a proper testing engine later with the login engine and tie the manage gui into it as well
+
+ session = requests.Session()
+
+ response = session.get( 'https://accounts.pixiv.net/login' )
+
+ soup = ClientDownloading.GetSoup( response.content )
+
+ # some whocking 20kb bit of json tucked inside a hidden form input wew lad
+ i = soup.find( 'input', id = 'init-config' )
+
+ raw_json = i['value']
+
+ j = json.loads( raw_json )
+
+ if 'pixivAccount.postKey' not in j:
+
+ return ( False, 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
+
+
+ post_key = j[ 'pixivAccount.postKey' ]
+
+ form_fields = {}
+
+ form_fields[ 'pixiv_id' ] = pixiv_id
+ form_fields[ 'password' ] = password
+ form_fields[ 'captcha' ] = ''
+ form_fields[ 'g_recaptcha_response' ] = ''
+ form_fields[ 'return_to' ] = 'https://www.pixiv.net'
+ form_fields[ 'lang' ] = 'en'
+ form_fields[ 'post_key' ] = post_key
+ form_fields[ 'source' ] = 'pc'
+
+ headers = {}
+
+ headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
+ headers[ 'origin' ] = "https://accounts.pixiv.net"
+
+ r = session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
+
+ if not r.ok:
+
+ HydrusData.ShowText( r.content )
+
+ return ( False, 'Login request failed! Info printed to log.' )
+
+
+ cookies = session.cookies
+
+ cookies.clear_expired_cookies()
+
+ domains = cookies.list_domains()
+
+ for domain in domains:
+
+ if domain.endswith( 'pixiv.net' ):
+
+ d = cookies.get_dict( domain )
+
+ if 'PHPSESSID' not in d:
+
+ HydrusData.ShowText( r.content )
+
+ return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
+
+
+ return ( True, '' )
+
+
+
+ HydrusData.ShowText( r.content )
+
+ return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
+
+
+HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager
+
# make this serialisable
-class LoginScript( object ):
+class LoginProcess( object ):
def __init__( self ):
@@ -29,6 +481,8 @@ class LoginScript( object ):
self._login_steps = []
self._validity = VALIDITY_UNTESTED
+ # possible error info
+
self._temp_variables = {}
@@ -36,6 +490,7 @@ class LoginScript( object ):
# this maybe takes some job_key or something so it can present to the user login process status
# this will be needed in the dialog where we test this. we need good feedback on how it is going
+ # irl, this could be a 'login popup' message as well, just to inform the user on the progress of any delay
for step in self._login_steps:
@@ -45,7 +500,15 @@ class LoginScript( object ):
except HydrusExceptions.VetoException: # or something--invalidscript exception?
- # also figure out a way to deal with connection errors and so on which will want a haderror time delay before giving it another go
+ # set error info
+
+ self._validity = VALIDITY_INVALID
+
+ return False
+
+ except Exception as e:
+
+ # set error info
self._validity = VALIDITY_INVALID
diff --git a/include/HydrusConstants.py b/include/HydrusConstants.py
index e68d3cdf..e4a3d4f9 100755
--- a/include/HydrusConstants.py
+++ b/include/HydrusConstants.py
@@ -49,7 +49,7 @@ options = {}
# Misc
NETWORK_VERSION = 18
-SOFTWARE_VERSION = 277
+SOFTWARE_VERSION = 278
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@@ -434,6 +434,8 @@ APPLICATION_UNKNOWN = 101
ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV, APPLICATION_HYDRUS_UPDATE_CONTENT, APPLICATION_HYDRUS_UPDATE_DEFINITIONS )
SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV )
+DECOMPRESSION_BOMB_IMAGES = ( IMAGE_JPEG, IMAGE_PNG )
+
IMAGES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP )
AUDIO = ( AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA )
diff --git a/include/TestClientNetworking.py b/include/TestClientNetworking.py
index 69be2616..20f8aa5e 100644
--- a/include/TestClientNetworking.py
+++ b/include/TestClientNetworking.py
@@ -1,6 +1,7 @@
import ClientConstants as CC
import ClientNetworking
import ClientNetworkingDomain
+import ClientNetworkingLogin
import collections
import HydrusConstants as HC
import HydrusData
@@ -222,7 +223,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@@ -252,7 +253,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@@ -280,7 +281,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@@ -332,7 +333,7 @@ class TestNetworkingJob( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@@ -541,7 +542,7 @@ class TestNetworkingJobHydrus( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
- login_manager = ClientNetworking.NetworkLoginManager()
+ login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
diff --git a/test.py b/test.py
index e6b2ebdd..82265a46 100644
--- a/test.py
+++ b/test.py
@@ -139,7 +139,6 @@ class Controller( object ):
self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
- self._managers[ 'web_sessions' ] = TestConstants.FakeWebSessionManager()
self._server_session_manager = HydrusSessions.HydrusSessionManagerServer()
self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache( self )