Rooba
/
hydrus
mirror of https://github.com/hydrusnetwork/hydrus.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Version 278

This commit is contained in:
Hydrus Network Developer 2017-10-18 14:41:25 -05:00
parent b4a288b994
commit 52d14cdb51
27 changed files with 1364 additions and 777 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
help/changelog.html
View File

@ -8,6 +8,38 @@
<div class="content">
<h3>changelog</h3>
<ul>
<li><h3>version 278</h3></li>
<ul>
<li>fixed the tumblr raw url converter to now point to the data.tumblr.com domain</li>
<li>added a hardcoded ssl verify exception for data.tumblr.com, which has an incorrectly defined ssl cert (at least for public-facing interactions), wew</li>
<li>all existing db urls and file import cache urls for media.tumblr.com will be updated to data.tumblr.com on update! (everything _should_ just magically work again)</li>
<li>fixed apng import, which the decompressionbomb detection code was not handling correctly</li>
<li>collapsed the different instatiations of the 'file import status' button down to one class</li>
<li>the file import status button now has a right-click menu that supports 'retry failures' and 'delete processed', if applicable</li>
<li>misc import status cache cleanup and refactoring</li>
<li>you can now edit or completely turn off the [404] and [DEAD] thread watcher page name prefixes under options->downloading</li>
<li>thread watchers should more reliably keep 404 status</li>
<li>'open selection in a new page' now preserves file order!</li>
<li>'view this file's duplicates' now sorts the files!</li>
<li>options->gui now has an option to change how often 'last session' is saved</li>
<li>'last session' will no longer autosave to the database if there are no changes</li>
<li>tags exported to neighbouring .txt files are now correctly sibling-collapsed</li>
<li>tags imported or exported via neighbouring .txt files are now correctly tag censored</li>
<li>the manage tags dialog will now protest with a yes/no dialog if you attempt to cancel it with uncommitted changes</li>
<li>the manage parents and siblings dialogs will now protest with a yes/no on an ok event if there are 'uncommitted' pairs in the lower boxes (e.g. if you forgot to click the 'add' button)</li>
<li>fixed an issue that would sometimes stop old sessions from loading properly</li>
<li>the duplicates page now does its maintenance jobs in modal popups!</li>
<li>attempting to apply a duplicate status to more than 100 pairs now throws up a warning yes/no dialog</li>
<li>the manage urls dialog now has copy/paste buttons</li>
<li>added a (somewhat debug) option to disable the mouse hide&anchor behaviour on slow Windows canvas drags to options->media</li>
<li>added a 'regen all phashes' command to the database regen menu</li>
<li>the disk cache options in help now have a help button to explain good values for ssd vs hdd</li>
<li>the edit bandwidth rules dialog now uses the new listctrl</li>
<li>merged the old and new login managers</li>
<li>misc login work</li>
<li>misc refactoring</li>
<li>misc cleanup</li>
</ul>
<li><h3>version 277</h3></li>
<ul>
<li>expanded the domain manager into a legit serialisable object that holds data and saves itself on changes. to begin with, it supports custom http headers for particular on network contexts</li>

357
include/ClientCaches.py
View File

@ -3174,360 +3174,3 @@ class UndoManager( object ):
class WebSessionManagerClient( object ):
SESSION_TIMEOUT = 60 * 60
def __init__( self, controller ):
self._controller = controller
self._error_names = set()
self._network_contexts_to_session_timeouts = {}
self._lock = threading.Lock()
def _GetCookiesDict( self, network_context ):
session = self._GetSession( network_context )
cookies = session.cookies
cookies.clear_expired_cookies()
domains = cookies.list_domains()
for domain in domains:
if domain.endswith( network_context.context_data ):
return cookies.get_dict( domain )
return {}
def _GetSession( self, network_context ):
session = self._controller.network_engine.session_manager.GetSession( network_context )
if network_context not in self._network_contexts_to_session_timeouts:
self._network_contexts_to_session_timeouts[ network_context ] = 0
if HydrusData.TimeHasPassed( self._network_contexts_to_session_timeouts[ network_context ] ):
session.cookies.clear_session_cookies()
self._network_contexts_to_session_timeouts[ network_context ] = HydrusData.GetNow() + self.SESSION_TIMEOUT
return session
def _IsLoggedIn( self, network_context, required_cookies ):
cookie_dict = self._GetCookiesDict( network_context )
for name in required_cookies:
if name not in cookie_dict:
return False
return True
def EnsureHydrusSessionIsOK( self, service_key ):
with self._lock:
if not self._controller.services_manager.ServiceExists( service_key ):
raise HydrusExceptions.DataMissing( 'Service does not exist!' )
name = self._controller.services_manager.GetService( service_key ).GetName()
if service_key in self._error_names:
raise Exception( 'Could not establish a hydrus network session for ' + name + '! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, service_key )
required_cookies = [ 'session_key' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
self.SetupHydrusSession( service_key )
if not self._IsLoggedIn( network_context, required_cookies ):
return
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( service_key )
raise
def EnsureLoggedIn( self, name ):
with self._lock:
if name in self._error_names:
raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
if name == 'hentai foundry':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
elif name == 'pixiv':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
required_cookies = [ 'PHPSESSID' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
if name == 'hentai foundry':
self.LoginHF( network_context )
elif name == 'pixiv':
result = self._controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
( pixiv_id, password ) = result
self.LoginPixiv( network_context, pixiv_id, password )
if not self._IsLoggedIn( network_context, required_cookies ):
raise Exception( name + ' login did not work correctly!' )
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( name )
raise
def LoginHF( self, network_context ):
session = self._GetSession( network_context )
response = session.get( 'https://www.hentai-foundry.com/' )
time.sleep( 1 )
response = session.get( 'https://www.hentai-foundry.com/?enterAgree=1' )
time.sleep( 1 )
cookie_dict = self._GetCookiesDict( network_context )
raw_csrf = cookie_dict[ 'YII_CSRF_TOKEN' ] # 19b05b536885ec60b8b37650a32f8deb11c08cd1s%3A40%3A%222917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32%22%3B
processed_csrf = urllib.unquote( raw_csrf ) # 19b05b536885ec60b8b37650a32f8deb11c08cd1s:40:"2917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32";
csrf_token = processed_csrf.split( '"' )[1] # the 2917... bit
hentai_foundry_form_info = ClientDefaults.GetDefaultHentaiFoundryInfo()
hentai_foundry_form_info[ 'YII_CSRF_TOKEN' ] = csrf_token
response = session.post( 'http://www.hentai-foundry.com/site/filters', data = hentai_foundry_form_info )
time.sleep( 1 )
# This updated login form is cobbled together from the example in PixivUtil2
# it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
# Pixiv 400s if cookies and referrers aren't passed correctly
# I am leaving this as a mess with the hope the eventual login engine will replace it
def LoginPixiv( self, network_context, pixiv_id, password ):
session = self._GetSession( network_context )
response = session.get( 'https://accounts.pixiv.net/login' )
soup = ClientDownloading.GetSoup( response.content )
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
i = soup.find( 'input', id = 'init-config' )
raw_json = i['value']
j = json.loads( raw_json )
if 'pixivAccount.postKey' not in j:
raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
post_key = j[ 'pixivAccount.postKey' ]
form_fields = {}
form_fields[ 'pixiv_id' ] = pixiv_id
form_fields[ 'password' ] = password
form_fields[ 'captcha' ] = ''
form_fields[ 'g_recaptcha_response' ] = ''
form_fields[ 'return_to' ] = 'https://www.pixiv.net'
form_fields[ 'lang' ] = 'en'
form_fields[ 'post_key' ] = post_key
form_fields[ 'source' ] = 'pc'
headers = {}
headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
headers[ 'origin' ] = "https://accounts.pixiv.net"
session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
time.sleep( 1 )
def SetupHydrusSession( self, service_key ):
# nah, replace this with a proper login script
service = self._controller.services_manager.GetService( service_key )
if not service.HasAccessKey():
raise HydrusExceptions.DataMissing( 'No access key for this service, so cannot set up session!' )
access_key = service.GetAccessKey()
url = 'blah'
network_job = ClientNetworking.NetworkJobHydrus( service_key, 'GET', url )
network_job.SetForLogin( True )
network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.encode( 'hex' ) )
self._controller.network_engine.AddJob( network_job )
network_job.WaitUntilDone()
def TestPixiv( self, pixiv_id, password ):
# this is just an ugly copy, but fuck it for the minute
# we'll figure out a proper testing engine later with the login engine and tie the manage gui into it as well
session = requests.Session()
response = session.get( 'https://accounts.pixiv.net/login' )
soup = ClientDownloading.GetSoup( response.content )
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
i = soup.find( 'input', id = 'init-config' )
raw_json = i['value']
j = json.loads( raw_json )
if 'pixivAccount.postKey' not in j:
return ( False, 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
post_key = j[ 'pixivAccount.postKey' ]
form_fields = {}
form_fields[ 'pixiv_id' ] = pixiv_id
form_fields[ 'password' ] = password
form_fields[ 'captcha' ] = ''
form_fields[ 'g_recaptcha_response' ] = ''
form_fields[ 'return_to' ] = 'https://www.pixiv.net'
form_fields[ 'lang' ] = 'en'
form_fields[ 'post_key' ] = post_key
form_fields[ 'source' ] = 'pc'
headers = {}
headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
headers[ 'origin' ] = "https://accounts.pixiv.net"
r = session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
if not r.ok:
HydrusData.ShowText( r.content )
return ( False, 'Login request failed! Info printed to log.' )
cookies = session.cookies
cookies.clear_expired_cookies()
domains = cookies.list_domains()
for domain in domains:
if domain.endswith( 'pixiv.net' ):
d = cookies.get_dict( domain )
if 'PHPSESSID' not in d:
HydrusData.ShowText( r.content )
return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
return ( True, '' )
HydrusData.ShowText( r.content )
return ( False, 'Pixiv login failed to establish session! Info printed to log.' )

50
include/ClientController.py
View File

@ -5,6 +5,7 @@ import ClientDefaults
import ClientGUIMenus
import ClientNetworking
import ClientNetworkingDomain
import ClientNetworkingLogin
import ClientThreading
import hashlib
import HydrusConstants as HC
@ -51,10 +52,10 @@ class Controller( HydrusController.HydrusController ):
HG.client_controller = self
# just to set up some defaults, in case some db update expects something for an odd yaml-loading reason
self._options = ClientDefaults.GetClientDefaultOptions()
self._new_options = ClientData.ClientOptions( self.db_dir )
self.options = ClientDefaults.GetClientDefaultOptions()
self.new_options = ClientData.ClientOptions( self.db_dir )
HC.options = self._options
HC.options = self.options
self._last_mouse_position = None
self._menu_open = False
@ -324,9 +325,9 @@ class Controller( HydrusController.HydrusController ):
return False
idle_normal = self._options[ 'idle_normal' ]
idle_period = self._options[ 'idle_period' ]
idle_mouse_period = self._options[ 'idle_mouse_period' ]
idle_normal = self.options[ 'idle_normal' ]
idle_period = self.options[ 'idle_period' ]
idle_mouse_period = self.options[ 'idle_mouse_period' ]
if idle_normal:
@ -389,11 +390,11 @@ class Controller( HydrusController.HydrusController ):
def DoIdleShutdownWork( self ):
stop_time = HydrusData.GetNow() + ( self._options[ 'idle_shutdown_max_minutes' ] * 60 )
stop_time = HydrusData.GetNow() + ( self.options[ 'idle_shutdown_max_minutes' ] * 60 )
self.MaintainDB( stop_time = stop_time )
if not self._options[ 'pause_repo_sync' ]:
if not self.options[ 'pause_repo_sync' ]:
services = self.services_manager.GetServices( HC.REPOSITORIES )
@ -422,11 +423,11 @@ class Controller( HydrusController.HydrusController ):
self._CreateSplash()
idle_shutdown_action = self._options[ 'idle_shutdown' ]
idle_shutdown_action = self.options[ 'idle_shutdown' ]
if idle_shutdown_action in ( CC.IDLE_ON_SHUTDOWN, CC.IDLE_ON_SHUTDOWN_ASK_FIRST ):
idle_shutdown_max_minutes = self._options[ 'idle_shutdown_max_minutes' ]
idle_shutdown_max_minutes = self.options[ 'idle_shutdown_max_minutes' ]
time_to_stop = HydrusData.GetNow() + ( idle_shutdown_max_minutes * 60 )
@ -497,12 +498,12 @@ class Controller( HydrusController.HydrusController ):
def GetOptions( self ):
return self._options
return self.options
def GetNewOptions( self ):
return self._new_options
return self.new_options
def GoodTimeToDoForegroundWork( self ):
@ -555,12 +556,12 @@ class Controller( HydrusController.HydrusController ):
self.services_manager = ClientCaches.ServicesManager( self )
self._options = self.Read( 'options' )
self._new_options = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
self.options = self.Read( 'options' )
self.new_options = self.Read( 'serialisable', HydrusSerialisable.SERIALISABLE_TYPE_CLIENT_OPTIONS )
HC.options = self._options
HC.options = self.options
if self._new_options.GetBoolean( 'use_system_ffmpeg' ):
if self.new_options.GetBoolean( 'use_system_ffmpeg' ):
if HydrusVideoHandling.FFMPEG_PATH.startswith( HC.BIN_DIR ):
@ -600,14 +601,14 @@ class Controller( HydrusController.HydrusController ):
if domain_manager is None:
domain_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
domain_manager._dirty = True
wx.MessageBox( 'Your domain manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
self.network_engine = ClientNetworking.NetworkEngine( self, bandwidth_manager, session_manager, domain_manager, login_manager )
@ -624,7 +625,6 @@ class Controller( HydrusController.HydrusController ):
self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
self._managers[ 'web_sessions' ] = ClientCaches.WebSessionManagerClient( self )
def wx_code():
@ -642,7 +642,7 @@ class Controller( HydrusController.HydrusController ):
def InitView( self ):
if self._options[ 'password' ] is not None:
if self.options[ 'password' ] is not None:
self.pub( 'splash_set_status_text', 'waiting for password' )
@ -657,7 +657,7 @@ class Controller( HydrusController.HydrusController ):
# this can produce unicode with cyrillic or w/e keyboards, which hashlib can't handle
password = HydrusData.ToByteString( dlg.GetValue() )
if hashlib.sha256( password ).digest() == self._options[ 'password' ]: break
if hashlib.sha256( password ).digest() == self.options[ 'password' ]: break
else:
@ -743,7 +743,7 @@ class Controller( HydrusController.HydrusController ):
def MaintainDB( self, stop_time = None ):
if self._new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ):
if self.new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ):
phashes_stop_time = stop_time
@ -763,7 +763,7 @@ class Controller( HydrusController.HydrusController ):
self.WriteInterruptable( 'maintain_similar_files_tree', stop_time = tree_stop_time, abandon_if_other_work_to_do = True )
search_distance = self._new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
search_distance = self.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
search_stop_time = stop_time
@ -815,7 +815,7 @@ class Controller( HydrusController.HydrusController ):
self._timestamps[ 'last_page_change' ] = HydrusData.GetNow()
disk_cache_maintenance_mb = self._new_options.GetNoneableInteger( 'disk_cache_maintenance_mb' )
disk_cache_maintenance_mb = self.new_options.GetNoneableInteger( 'disk_cache_maintenance_mb' )
if disk_cache_maintenance_mb is not None:
@ -1141,7 +1141,7 @@ class Controller( HydrusController.HydrusController ):
return False
max_cpu = self._options[ 'idle_cpu_max' ]
max_cpu = self.options[ 'idle_cpu_max' ]
if max_cpu is None:

28
include/ClientDB.py
View File

@ -1499,7 +1499,7 @@ class DB( HydrusDB.HydrusDB ):
path = client_files_manager.GetFilePath( hash, mime )
if mime in ( HC.IMAGE_JPEG, HC.IMAGE_PNG ):
if mime in HC.MIMES_WE_CAN_PHASH:
try:
@ -9830,6 +9830,31 @@ class DB( HydrusDB.HydrusDB ):
self.pub_initial_message( message )
if version == 277:
self._controller.pub( 'splash_set_status_text', 'updating tumblr urls' )
urls = self._c.execute( 'SELECT hash_id, url FROM urls;' ).fetchall()
# don't catch the 68.media.whatever, as these may be valid, not raw urls
magic_phrase = '//media.tumblr.com'
replacement = '//data.tumblr.com'
updates = []
for ( hash_id, url ) in urls:
if magic_phrase in url:
fixed_url = url.replace( magic_phrase, replacement )
updates.append( ( fixed_url, hash_id ) )
self._c.executemany( 'UPDATE OR IGNORE urls SET url = ? WHERE hash_id = ?;', updates )
self._controller.pub( 'splash_set_title_text', 'updated db to v' + str( version + 1 ) )
self._c.execute( 'UPDATE version SET version = ?;', ( version + 1, ) )
@ -10426,6 +10451,7 @@ class DB( HydrusDB.HydrusDB ):
elif action == 'repair_client_files': result = self._RepairClientFiles( *args, **kwargs )
elif action == 'reset_repository': result = self._ResetRepository( *args, **kwargs )
elif action == 'save_options': result = self._SaveOptions( *args, **kwargs )
elif action == 'schedule_full_phash_regen': result = self._CacheSimilarFilesSchedulePHashRegeneration()
elif action == 'serialisable_simple': result = self._SetJSONSimple( *args, **kwargs )
elif action == 'serialisable': result = self._SetJSONDump( *args, **kwargs )
elif action == 'serialisables_overwrite': result = self._OverwriteJSONDumps( *args, **kwargs )

6
include/ClientData.py
View File

@ -853,6 +853,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'booleans' ][ 'reverse_page_shift_drag_behaviour' ] = False
self._dictionary[ 'booleans' ][ 'anchor_and_hide_canvas_drags' ] = HC.PLATFORM_WINDOWS
#
self._dictionary[ 'colours' ] = HydrusSerialisable.SerialisableDictionary()
@ -919,6 +921,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'integers' ][ 'thumbnail_visibility_scroll_percent' ] = 75
self._dictionary[ 'integers' ][ 'last_session_save_period_minutes' ] = 5
#
self._dictionary[ 'keys' ] = {}
@ -951,6 +955,8 @@ class ClientOptions( HydrusSerialisable.SerialisableBase ):
self._dictionary[ 'noneable_strings' ][ 'favourite_file_lookup_script' ] = 'gelbooru md5'
self._dictionary[ 'noneable_strings' ][ 'suggested_tags_layout' ] = 'notebook'
self._dictionary[ 'noneable_strings' ][ 'backup_path' ] = None
self._dictionary[ 'noneable_strings' ][ 'thread_watcher_not_found_page_string' ] = '[404]'
self._dictionary[ 'noneable_strings' ][ 'thread_watcher_dead_page_string' ] = '[DEAD]'
self._dictionary[ 'strings' ] = {}

2
include/ClientDefaults.py
View File

@ -146,7 +146,7 @@ def GetClientDefaultOptions():
options[ 'idle_shutdown_max_minutes' ] = 5
options[ 'maintenance_delete_orphans_period' ] = 86400 * 3
options[ 'trash_max_age' ] = 72
options[ 'trash_max_size' ] = 512
options[ 'trash_max_size' ] = 2048
options[ 'remove_trashed_files' ] = False
options[ 'remove_filtered_files' ] = False
options[ 'external_host' ] = None

17
include/ClientDownloading.py
View File

@ -1239,9 +1239,7 @@ class GalleryHentaiFoundry( Gallery ):
def _EnsureLoggedIn( self ):
manager = HG.client_controller.GetManager( 'web_sessions' )
manager.EnsureLoggedIn( 'hentai foundry' )
HG.client_controller.network_engine.login_manager.EnsureLoggedIn( 'hentai foundry' )
def _GetFileURLAndTags( self, url ):
@ -1564,9 +1562,7 @@ class GalleryPixiv( Gallery ):
def _EnsureLoggedIn( self ):
manager = HG.client_controller.GetManager( 'web_sessions' )
manager.EnsureLoggedIn( 'pixiv' )
HG.client_controller.network_engine.login_manager.EnsureLoggedIn( 'pixiv' )
def _ParseGalleryPage( self, html, url_base ):
@ -1753,6 +1749,8 @@ class GalleryTumblr( Gallery ):
# I am not sure if it is always 68, but let's not assume
# Indeed, this is apparently now 78, wew!
( scheme, rest ) = long_url.split( '://', 1 )
if rest.startswith( 'media.tumblr.com' ):
@ -1767,6 +1765,11 @@ class GalleryTumblr( Gallery ):
return shorter_url
def MediaToDataSubdomain( url ):
return url.replace( 'media', 'data', 1 )
definitely_no_more_pages = False
processed_raw_json = data.split( 'var tumblr_api_read = ' )[1][:-2] # -1 takes a js ';' off the end
@ -1823,6 +1826,8 @@ class GalleryTumblr( Gallery ):
url = Remove68Subdomain( url )
url = MediaToDataSubdomain( url )
url = ClientData.ConvertHTTPToHTTPS( url )

36
include/ClientGUI.py
View File

@ -1154,7 +1154,8 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
submenu = wx.Menu()
ClientGUIMenus.AppendMenuItem( self, submenu, 'autocomplete cache', 'Delete and recreate the tag autocomplete cache, fixing any miscounts.', self._RegenerateACCache )
ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search data', 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesData )
ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search metadata', 'Delete and recreate the similar files search phashes.', self._RegenerateSimilarFilesPhashes )
ClientGUIMenus.AppendMenuItem( self, submenu, 'similar files search tree', 'Delete and recreate the similar files search tree.', self._RegenerateSimilarFilesTree )
ClientGUIMenus.AppendMenuItem( self, submenu, 'all thumbnails', 'Delete all thumbnails and regenerate them from their original files.', self._RegenerateThumbnails )
ClientGUIMenus.AppendMenu( menu, submenu, 'regenerate' )
@ -1656,7 +1657,9 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
self._notebook.LoadGUISession( default_gui_session )
wx.CallLater( 5 * 60 * 1000, self.SaveLastSession )
last_session_save_period_minutes = self._controller.new_options.GetInteger( 'last_session_save_period_minutes' )
wx.CallLater( last_session_save_period_minutes * 60 * 1000, self.SaveLastSession )
def _ManageAccountTypes( self, service_key ):
@ -2101,7 +2104,26 @@ class FrameGUI( ClientGUITopLevelWindows.FrameThatResizes ):
def _RegenerateSimilarFilesData( self ):
def _RegenerateSimilarFilesPhashes( self ):
message = 'This will schedule all similar files \'phash\' metadata to be regenerated. This is a very expensive operation that will occur in future idle time.'
message += os.linesep * 2
message += 'This ultimately requires a full read for all valid files. It is a large investment of CPU and HDD time.'
message += os.linesep * 2
message += 'Do not run this unless you know your phashes need to be regenerated.'
with ClientGUIDialogs.DialogYesNo( self, message, yes_label = 'do it', no_label = 'forget it' ) as dlg:
result = dlg.ShowModal()
if result == wx.ID_YES:
self._controller.Write( 'schedule_full_phash_regen' )
def _RegenerateSimilarFilesTree( self ):
message = 'This will delete and then recreate the similar files search tree. This is useful if it has somehow become unbalanced and similar files searches are running slow.'
message += os.linesep * 2
@ -3125,7 +3147,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._notebook.NewPage( management_controller, on_deepest_notebook = True )
def NewPageQuery( self, service_key, initial_hashes = None, initial_predicates = None, page_name = None ):
def NewPageQuery( self, service_key, initial_hashes = None, initial_predicates = None, page_name = None, do_sort = False ):
if initial_hashes is None:
@ -3137,7 +3159,7 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
initial_predicates = []
self._notebook.NewPageQuery( service_key, initial_hashes = initial_hashes, initial_predicates = initial_predicates, page_name = page_name, on_deepest_notebook = True )
self._notebook.NewPageQuery( service_key, initial_hashes = initial_hashes, initial_predicates = initial_predicates, page_name = page_name, on_deepest_notebook = True, do_sort = do_sort )
def NotifyClosedPage( self, page ):
@ -3312,7 +3334,9 @@ The password is cleartext here but obscured in the entry dialog. Enter a blank p
self._notebook.SaveGUISession( 'last session' )
wx.CallLater( 5 * 60 * 1000, self.SaveLastSession )
last_session_save_period_minutes = self._controller.new_options.GetInteger( 'last_session_save_period_minutes' )
wx.CallLater( last_session_save_period_minutes * 60 * 1000, self.SaveLastSession )
def SetMediaFocus( self ): self._SetMediaFocus()

4
include/ClientGUICanvas.py
View File

@ -2945,7 +2945,9 @@ class CanvasWithHovers( CanvasWithDetails ):
self._current_drag_is_touch = True
if HC.PLATFORM_WINDOWS and not self._current_drag_is_touch:
anchor_and_hide_canvas_drags = HG.client_controller.new_options.GetBoolean( 'anchor_and_hide_canvas_drags' )
if HC.PLATFORM_WINDOWS and anchor_and_hide_canvas_drags and not self._current_drag_is_touch:
# touch events obviously don't mix with warping well. the touch just warps it back and again and we get a massive delta!

107
include/ClientGUICommon.py
View File

@ -733,6 +733,7 @@ class ChoiceSort( wx.Panel ):
self._sort_asc_choice.Bind( wx.EVT_CHOICE, self.EventSortAscChoice )
HG.client_controller.sub( self, 'ACollectHappened', 'collect_media' )
HG.client_controller.sub( self, 'BroadcastSort', 'do_page_sort' )
if self._management_controller is not None and self._management_controller.HasVariable( 'media_sort' ):
@ -815,7 +816,12 @@ class ChoiceSort( wx.Panel ):
def BroadcastSort( self ):
def BroadcastSort( self, page_key = None ):
if page_key is not None and page_key != self._management_controller.GetKey( 'page' ):
return
self._BroadcastSort()
@ -1635,11 +1641,20 @@ class NoneableSpinCtrl( wx.Panel ):
def GetValue( self ):
if self._checkbox.GetValue(): return None
if self._checkbox.GetValue():
return None
else:
if self._num_dimensions == 2: return ( self._one.GetValue() * self._multiplier, self._two.GetValue() * self._multiplier )
else: return self._one.GetValue() * self._multiplier
if self._num_dimensions == 2:
return ( self._one.GetValue() * self._multiplier, self._two.GetValue() * self._multiplier )
else:
return self._one.GetValue() * self._multiplier
@ -1681,6 +1696,90 @@ class NoneableSpinCtrl( wx.Panel ):
class NoneableTextCtrl( wx.Panel ):
def __init__( self, parent, message = '', none_phrase = 'no limit' ):
wx.Panel.__init__( self, parent )
self._checkbox = wx.CheckBox( self )
self._checkbox.Bind( wx.EVT_CHECKBOX, self.EventCheckBox )
self._checkbox.SetLabelText( none_phrase )
self._text = wx.TextCtrl( self )
hbox = wx.BoxSizer( wx.HORIZONTAL )
if len( message ) > 0:
hbox.AddF( BetterStaticText( self, message + ': ' ), CC.FLAGS_VCENTER )
hbox.AddF( self._text, CC.FLAGS_VCENTER )
hbox.AddF( self._checkbox, CC.FLAGS_VCENTER )
self.SetSizer( hbox )
def Bind( self, event_type, callback ):
self._checkbox.Bind( wx.EVT_CHECKBOX, callback )
self._text.Bind( wx.EVT_TEXT, callback )
def EventCheckBox( self, event ):
if self._checkbox.GetValue():
self._text.Disable()
else:
self._text.Enable()
def GetValue( self ):
if self._checkbox.GetValue():
return None
else:
return self._text.GetValue()
def SetToolTipString( self, text ):
wx.Panel.SetToolTipString( self, text )
for c in self.GetChildren():
c.SetToolTipString( text )
def SetValue( self, value ):
if value is None:
self._checkbox.SetValue( True )
self._text.Disable()
else:
self._checkbox.SetValue( False )
self._text.Enable()
self._text.SetValue( value )
class OnOffButton( wx.Button ):
def __init__( self, parent, page_key, topic, on_label, off_label = None, start_on = True ):

52
include/ClientGUIControls.py
View File

@ -20,11 +20,9 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
ClientGUICommon.StaticBox.__init__( self, parent, 'bandwidth rules' )
columns = [ ( 'type', -1 ), ( 'time delta', 120 ), ( 'max allowed', 80 ) ]
listctrl_panel = ClientGUIListCtrl.BetterListCtrlPanel( self )
listctrl_panel = ClientGUIListCtrl.SaneListCtrlPanel( self )
self._listctrl = ClientGUIListCtrl.SaneListCtrl( listctrl_panel, 100, columns, delete_key_callback = self._Delete, activation_callback = self._Edit )
self._listctrl = ClientGUIListCtrl.BetterListCtrl( listctrl_panel, 'bandwidth_rules', 8, 10, [ ( 'type', -1 ), ( 'time delta', 16 ), ( 'max allowed', 14 ) ], self._ConvertRuleToListctrlTuples, delete_key_callback = self._Delete, activation_callback = self._Edit )
listctrl_panel.SetListCtrl( self._listctrl )
@ -34,18 +32,13 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
#
for rule in bandwidth_rules.GetRules():
sort_tuple = rule
display_tuple = self._GetDisplayTuple( sort_tuple )
self._listctrl.Append( display_tuple, sort_tuple )
self._listctrl.AddDatas( bandwidth_rules.GetRules() )
self._listctrl.Sort( 0 )
#
self.AddF( listctrl_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
self.AddF( listctrl_panel, CC.FLAGS_EXPAND_BOTH_WAYS )
def _Add( self ):
@ -62,16 +55,14 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
new_rule = panel.GetValue()
sort_tuple = new_rule
self._listctrl.AddDatas( ( new_rule, ) )
display_tuple = self._GetDisplayTuple( sort_tuple )
self._listctrl.Append( display_tuple, sort_tuple )
self._listctrl.Sort()
def _GetDisplayTuple( self, rule ):
def _ConvertRuleToListctrlTuples( self, rule ):
( bandwidth_type, time_delta, max_allowed ) = rule
@ -88,7 +79,10 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
pretty_max_allowed = HydrusData.ConvertIntToPrettyString( max_allowed )
return ( pretty_bandwidth_type, pretty_time_delta, pretty_max_allowed )
sort_tuple = ( pretty_bandwidth_type, time_delta, max_allowed )
display_tuple = ( pretty_bandwidth_type, pretty_time_delta, pretty_max_allowed )
return ( display_tuple, sort_tuple )
def _Delete( self ):
@ -97,18 +91,16 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
if dlg.ShowModal() == wx.ID_YES:
self._listctrl.RemoveAllSelected()
self._listctrl.DeleteSelected()
def _Edit( self ):
all_selected = self._listctrl.GetAllSelected()
selected_rules = self._listctrl.GetData( only_selected = True )
for index in all_selected:
rule = self._listctrl.GetClientData( index )
for rule in selected_rules:
with ClientGUITopLevelWindows.DialogEdit( self, 'edit rule' ) as dlg:
@ -120,11 +112,9 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
edited_rule = panel.GetValue()
sort_tuple = edited_rule
self._listctrl.DeleteDatas( ( rule, ) )
display_tuple = self._GetDisplayTuple( sort_tuple )
self._listctrl.UpdateRow( index, display_tuple, sort_tuple )
self._listctrl.AddDatas( ( edited_rule, ) )
else:
@ -133,12 +123,16 @@ class BandwidthRulesCtrl( ClientGUICommon.StaticBox ):
self._listctrl.Sort()
def GetValue( self ):
bandwidth_rules = HydrusNetworking.BandwidthRules()
for ( bandwidth_type, time_delta, max_allowed ) in self._listctrl.GetClientData():
for rule in self._listctrl.GetData():
( bandwidth_type, time_delta, max_allowed ) = rule
bandwidth_rules.AddRule( bandwidth_type, time_delta, max_allowed )

16
include/ClientGUIDialogs.py
View File

@ -2525,6 +2525,10 @@ class DialogPathsToTags( Dialog ):
txt_tags = siblings_manager.CollapseTags( self._service_key, txt_tags )
tag_censorship_manager = HG.client_controller.GetManager( 'tag_censorship' )
txt_tags = tag_censorship_manager.FilterTags( self._service_key, txt_tags )
tags.extend( txt_tags )
except:
@ -3337,9 +3341,19 @@ class DialogSetupExport( Dialog ):
tags = set()
siblings_manager = HG.controller.GetManager( 'tag_siblings' )
tag_censorship_manager = HG.client_controller.GetManager( 'tag_censorship' )
for service_key in self._neighbouring_txt_tag_service_keys:
tags.update( tags_manager.GetCurrent( service_key ) )
current_tags = tags_manager.GetCurrent( service_key )
current_tags = siblings_manager.CollapseTags( service_key, current_tags )
current_tags = tag_censorship_manager.FilterTags( service_key, current_tags )
tags.update( current_tags )
tags = list( tags )

75
include/ClientGUIDialogsManage.py
View File

@ -2363,8 +2363,7 @@ class DialogManageImportFoldersEdit( ClientGUIDialogs.Dialog ):
self._paused = wx.CheckBox( self._folder_box )
self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._folder_box, CC.GlobalBMPs.seed_cache, self.ShowSeedCache )
self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
self._seed_cache_button = ClientGUISeedCache.SeedCacheButton( self, HG.client_controller, self._import_folder.GetSeedCache, seed_cache_set_callable = self._import_folder.SetSeedCache )
#
@ -2756,25 +2755,6 @@ class DialogManageImportFoldersEdit( ClientGUIDialogs.Dialog ):
return self._import_folder
def ShowSeedCache( self ):
seed_cache = self._import_folder.GetSeedCache()
dupe_seed_cache = seed_cache.Duplicate()
with ClientGUITopLevelWindows.DialogEdit( self, 'file import status' ) as dlg:
panel = ClientGUISeedCache.EditSeedCachePanel( dlg, HG.client_controller, dupe_seed_cache )
dlg.SetPanel( panel )
if dlg.ShowModal() == wx.ID_OK:
self._import_folder.SetSeedCache( dupe_seed_cache )
class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
def __init__( self, parent ):
@ -2872,7 +2852,7 @@ class DialogManagePixivAccount( ClientGUIDialogs.Dialog ):
try:
manager = HG.client_controller.GetManager( 'web_sessions' )
manager = HG.client_controller.network_engine.login_manager
( result, message ) = manager.TestPixiv( pixiv_id, password )
@ -3611,6 +3591,19 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
def EventOK( self, event ):
if self._tag_repositories.GetCurrentPage().HasUncommittedPair():
message = 'Are you sure you want to OK? You have an uncommitted pair.'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
if dlg.ShowModal() != wx.ID_YES:
return
service_keys_to_content_updates = {}
try:
@ -3624,7 +3617,10 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
HG.client_controller.Write( 'content_updates', service_keys_to_content_updates )
finally: self.EndModal( wx.ID_OK )
finally:
self.EndModal( wx.ID_OK )
def EventServiceChanged( self, event ):
@ -4080,6 +4076,11 @@ class DialogManageTagParents( ClientGUIDialogs.Dialog ):
return ( self._service_key, content_updates )
def HasUncommittedPair( self ):
return len( self._children.GetTags() ) > 0 and len( self._parents.GetTags() ) > 0
def SetTagBoxFocus( self ):
if len( self._children.GetTags() ) == 0: self._child_input.SetFocus()
@ -4215,6 +4216,19 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
def EventOK( self, event ):
if self._tag_repositories.GetCurrentPage().HasUncommittedPair():
message = 'Are you sure you want to OK? You have an uncommitted pair.'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
if dlg.ShowModal() != wx.ID_YES:
return
service_keys_to_content_updates = {}
try:
@ -4739,6 +4753,11 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
return ( self._service_key, content_updates )
def HasUncommittedPair( self ):
return len( self._old_siblings.GetTags() ) > 0 and self._current_new is not None
def SetNew( self, new_tags ):
if len( new_tags ) == 0:
@ -4763,8 +4782,14 @@ class DialogManageTagSiblings( ClientGUIDialogs.Dialog ):
def SetTagBoxFocus( self ):
if len( self._old_siblings.GetTags() ) == 0: self._old_input.SetFocus()
else: self._new_input.SetFocus()
if len( self._old_siblings.GetTags() ) == 0:
self._old_input.SetFocus()
else:
self._new_input.SetFocus()
def THREADInitialise( self, tags ):

196
include/ClientGUIManagement.py
View File

@ -937,9 +937,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self.SetSizer( vbox )
self.Bind( wx.EVT_TIMER, self.TIMEREventUpdateDBJob, id = ID_TIMER_UPDATE )
self._update_db_job_timer = wx.Timer( self, id = ID_TIMER_UPDATE )
HG.client_controller.sub( self, 'RefreshAndUpdateStatus', 'refresh_dupe_numbers' )
@ -978,9 +975,13 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _RebalanceTree( self ):
self._job = 'branches'
job_key = ClientThreading.JobKey( cancellable = True )
self._StartStopDBJob()
self._controller.Write( 'maintain_similar_files_tree', job_key = job_key )
self._controller.pub( 'modal_message', job_key )
self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _RefreshAndUpdateStatus( self ):
@ -994,9 +995,13 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _RegeneratePhashes( self ):
self._job = 'phashes'
job_key = ClientThreading.JobKey( cancellable = True )
self._StartStopDBJob()
self._controller.Write( 'maintain_similar_files_phashes', job_key = job_key )
self._controller.pub( 'modal_message', job_key )
self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _ResetUnknown( self ):
@ -1017,9 +1022,15 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _SearchForDuplicates( self ):
self._job = 'search'
job_key = ClientThreading.JobKey( cancellable = True )
self._StartStopDBJob()
search_distance = self._search_distance_spinctrl.GetValue()
self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = job_key )
self._controller.pub( 'modal_message', job_key )
self._controller.CallToThread( self._THREADWaitOnJob, job_key )
def _SetFileDomain( self, service_key ):
@ -1054,7 +1065,7 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
message += os.linesep
message += '3 - Walking through the pairs or groups of potential duplicates and telling the client how they are related.'
message += os.linesep * 2
message += 'For the first two steps, you likely just want to click the play buttons and wait for them to complete. They are very CPU intensive and lock the database heavily as they work. If you want to use the client for anything else while they are running, pause them first. You can also set them to run in idle time from the cog icon. For the search \'distance\', start at the fast and limited \'exact match\', or 0 \'hamming distance\' search and slowly expand it as you gain experience with the system.'
message += 'For the first two steps, you likely just want to click the play buttons and wait for them to complete. They are CPU intensive and lock the client as they work. You can also set them to run in idle time from the cog icon. For the search \'distance\', start at the fast and limited \'exact match\' (0 \'hamming distance\') and slowly expand it as you gain experience with the system.'
message += os.linesep * 2
message += 'Once you have found some potential pairs, you can either show some random groups as thumbnails (and process them manually however you prefer), or you can launch the specialised duplicate filter, which lets you quickly assign duplicate status to pairs of files and will automatically merge files and tags between dupes however you prefer.'
message += os.linesep * 2
@ -1092,125 +1103,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._controller.pub( 'swap_media_panel', self._page_key, panel )
def _StartStopDBJob( self ):
if self._job_key is None:
self._cog_button.Disable()
self._phashes_button.Disable()
self._branches_button.Disable()
self._search_button.Disable()
self._search_distance_button.Disable()
self._search_distance_spinctrl.Disable()
self._show_some_dupes.Disable()
self._launch_filter.Disable()
self._job_key = ClientThreading.JobKey( cancellable = True )
if self._job == 'phashes':
self._phashes_button.Enable()
self._phashes_button.SetBitmap( CC.GlobalBMPs.stop )
self._controller.Write( 'maintain_similar_files_phashes', job_key = self._job_key )
elif self._job == 'branches':
self._branches_button.Enable()
self._branches_button.SetBitmap( CC.GlobalBMPs.stop )
self._controller.Write( 'maintain_similar_files_tree', job_key = self._job_key )
elif self._job == 'search':
self._search_button.Enable()
self._search_button.SetBitmap( CC.GlobalBMPs.stop )
search_distance = self._search_distance_spinctrl.GetValue()
self._controller.Write( 'maintain_similar_files_duplicate_pairs', search_distance, job_key = self._job_key )
self._update_db_job_timer.Start( 250, wx.TIMER_CONTINUOUS )
else:
self._job_key.Cancel()
def _UpdateJob( self ):
if self._in_break:
if HG.client_controller.DBCurrentlyDoingJob():
return
else:
self._in_break = False
self._StartStopDBJob()
return
if self._job_key.TimeRunning() > 10:
self._job_key.Cancel()
self._job_key = None
self._in_break = True
return
if self._job_key.IsDone():
self._job_key = None
self._update_db_job_timer.Stop()
self._RefreshAndUpdateStatus()
return
if self._job == 'phashes':
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
if text is not None:
self._num_phashes_to_regen.SetLabelText( text )
elif self._job == 'branches':
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
if text is not None:
self._num_branches_to_regen.SetLabelText( text )
elif self._job == 'search':
text = self._job_key.GetIfHasVariable( 'popup_text_1' )
gauge = self._job_key.GetIfHasVariable( 'popup_gauge_1' )
if text is not None and gauge is not None:
( value, range ) = gauge
self._num_searched.SetValue( text, value, range )
def _UpdateStatus( self ):
( num_phashes_to_regen, num_branches_to_regen, searched_distances_to_count, duplicate_types_to_count ) = self._similar_files_maintenance_status
@ -1221,7 +1113,7 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._branches_button.SetBitmap( CC.GlobalBMPs.play )
self._search_button.SetBitmap( CC.GlobalBMPs.play )
total_num_files = sum( searched_distances_to_count.values() )
total_num_files = max( num_phashes_to_regen, sum( searched_distances_to_count.values() ) )
if num_phashes_to_regen == 0:
@ -1312,6 +1204,21 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
def _THREADWaitOnJob( self, job_key ):
while not job_key.IsDone():
if HG.model_shutdown:
return
time.sleep( 0.25 )
self._RefreshAndUpdateStatus()
def EventSearchDistanceChanged( self, event ):
self._UpdateStatus()
@ -1322,11 +1229,6 @@ class ManagementPanelDuplicateFilter( ManagementPanel ):
self._RefreshAndUpdateStatus()
def TIMEREventUpdateDBJob( self, event ):
self._UpdateJob()
management_panel_types_to_classes[ MANAGEMENT_TYPE_DUPLICATE_FILTER ] = ManagementPanelDuplicateFilter
class ManagementPanelImporter( ManagementPanel ):
@ -2169,7 +2071,7 @@ class ManagementPanelImporterPageOfImages( ManagementPanelImporter ):
def EventPaste( self, event ):
if wx.TheClipboard.Open():
data = wx.TextDataObject()
@ -2627,8 +2529,10 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
self._file_download_control = ClientGUIControls.NetworkJobControl( self._url_panel )
self._overall_gauge = ClientGUICommon.Gauge( self._url_panel )
self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self._url_panel, CC.GlobalBMPs.seed_cache, self._SeedCache )
self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
self._urls_import = self._management_controller.GetVariable( 'urls_import' )
# replace all this with a seed cache panel sometime
self._seed_cache_button = ClientGUISeedCache.SeedCacheButton( self, self._controller, self._urls_import.GetSeedCache )
self._url_input = wx.TextCtrl( self._url_panel, style = wx.TE_PROCESS_ENTER )
self._url_input.Bind( wx.EVT_KEY_DOWN, self.EventKeyDown )
@ -2636,8 +2540,6 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
self._url_paste = wx.Button( self._url_panel, label = 'paste urls' )
self._url_paste.Bind( wx.EVT_BUTTON, self.EventPaste )
self._urls_import = self._management_controller.GetVariable( 'urls_import' )
file_import_options = self._urls_import.GetOptions()
self._file_import_options = ClientGUIImport.FileImportOptionsButton( self._url_panel, file_import_options, self._urls_import.SetFileImportOptions )
@ -2681,20 +2583,6 @@ class ManagementPanelImporterURLs( ManagementPanelImporter ):
HG.client_controller.sub( self, 'SetURLInput', 'set_page_url_input' )
def _SeedCache( self ):
seed_cache = self._urls_import.GetSeedCache()
title = 'file import status'
frame_key = 'file_import_status'
frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
panel = ClientGUISeedCache.EditSeedCachePanel( frame, self._controller, seed_cache )
frame.SetPanel( panel )
def _UpdateStatus( self ):
( ( overall_status, ( overall_value, overall_range ) ), paused ) = self._urls_import.GetStatus()

47
include/ClientGUIMedia.py
View File

@ -157,6 +157,20 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
def _ArchiveDeleteFilter( self ):
media_results = self.GenerateMediaResults( discriminant = CC.DISCRIMINANT_LOCAL_BUT_NOT_IN_TRASH, selected_media = set( self._selected_media ), for_media_viewer = True )
if len( media_results ) > 0:
canvas_frame = ClientGUICanvas.CanvasFrame( self.GetTopLevelParent() )
canvas_window = ClientGUICanvas.CanvasMediaListFilterArchiveDelete( canvas_frame, self._page_key, media_results )
canvas_frame.SetCanvas( canvas_window )
def _CopyBMPToClipboard( self ):
media = self._focussed_media.GetDisplayMedia()
@ -520,20 +534,6 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
def _ArchiveDeleteFilter( self ):
media_results = self.GenerateMediaResults( discriminant = CC.DISCRIMINANT_LOCAL_BUT_NOT_IN_TRASH, selected_media = set( self._selected_media ), for_media_viewer = True )
if len( media_results ) > 0:
canvas_frame = ClientGUICanvas.CanvasFrame( self.GetTopLevelParent() )
canvas_window = ClientGUICanvas.CanvasMediaListFilterArchiveDelete( canvas_frame, self._page_key, media_results )
canvas_frame.SetCanvas( canvas_window )
def _GetNumSelected( self ):
return sum( [ media.GetNumFiles() for media in self._selected_media ] )
@ -1279,6 +1279,21 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
media_pairs = list( itertools.combinations( flat_media, 2 ) )
if len( media_pairs ) > 100:
message = 'The duplicate system does not yet work well for large groups of duplicates. This is about to ask if you want to apply a dupe status for more than 100 pairs.'
message += os.linesep * 2
message += 'Unless you are testing the system or have another good reason to try this, I recommend you step back for now.'
with ClientGUIDialogs.DialogYesNo( self, message, yes_label = 'I know what I am doing', no_label = 'step back for now' ) as dlg:
if dlg.ShowModal() != wx.ID_YES:
return
message = 'Are you sure you want to ' + yes_no_text + ' for the ' + HydrusData.ConvertIntToPrettyString( len( media_pairs ) ) + ' pairs?'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
@ -1432,13 +1447,13 @@ class MediaPanel( ClientMedia.ListeningMediaList, wx.ScrolledWindow ):
if hashes is not None and len( hashes ) > 0:
HG.client_controller.pub( 'new_page_query', self._file_service_key, initial_hashes = hashes )
HG.client_controller.pub( 'new_page_query', self._file_service_key, initial_hashes = hashes, do_sort = True )
def _ShowSelectionInNewPage( self ):
hashes = self._GetSelectedHashes()
hashes = self._GetSelectedHashes( ordered = True )
if hashes is not None and len( hashes ) > 0:

33
include/ClientGUIPages.py
View File

@ -9,6 +9,7 @@ import ClientGUIMenus
import ClientGUICanvas
import ClientDownloading
import ClientSearch
import hashlib
import HydrusData
import HydrusExceptions
import HydrusSerialisable
@ -729,6 +730,8 @@ class PagesNotebook( wx.Notebook ):
self._page_key = HydrusData.GenerateKey()
self._last_last_session_hash = None
self._controller.sub( self, 'RefreshPageName', 'refresh_page_name' )
self._controller.sub( self, 'NotifyPageUnclosed', 'notify_page_unclosed' )
@ -1773,6 +1776,9 @@ class PagesNotebook( wx.Notebook ):
page_name = page.GetName()
# in some unusual circumstances, this gets out of whack
insertion_index = min( insertion_index, self.GetPageCount() )
self.InsertPage( insertion_index, page, page_name, select = True )
self._controller.pub( 'refresh_page_name', page.GetPageKey() )
@ -1838,7 +1844,7 @@ class PagesNotebook( wx.Notebook ):
return self.NewPage( management_controller, on_deepest_notebook = on_deepest_notebook )
def NewPageQuery( self, file_service_key, initial_hashes = None, initial_predicates = None, page_name = None, on_deepest_notebook = False ):
def NewPageQuery( self, file_service_key, initial_hashes = None, initial_predicates = None, page_name = None, on_deepest_notebook = False, do_sort = False ):
if initial_hashes is None:
@ -1870,7 +1876,14 @@ class PagesNotebook( wx.Notebook ):
management_controller = ClientGUIManagement.CreateManagementControllerQuery( page_name, file_service_key, file_search_context, search_enabled )
return self.NewPage( management_controller, initial_hashes = initial_hashes, on_deepest_notebook = on_deepest_notebook )
page = self.NewPage( management_controller, initial_hashes = initial_hashes, on_deepest_notebook = on_deepest_notebook )
if do_sort:
HG.client_controller.pub( 'do_page_sort', page.GetPageKey() )
return page
def NewPagesNotebook( self, name = 'pages', forced_insertion_index = None, on_deepest_notebook = False, give_it_a_blank_page = True ):
@ -2200,6 +2213,8 @@ class PagesNotebook( wx.Notebook ):
#
session = GUISession( name )
for page in self._GetPages():
@ -2207,6 +2222,20 @@ class PagesNotebook( wx.Notebook ):
session.AddPage( page )
#
if name == 'last session':
session_hash = hashlib.sha256( session.DumpToString() ).digest()
if session_hash == self._last_last_session_hash:
return
self._last_last_session_hash = session_hash
self._controller.Write( 'serialisable', session )
self._controller.pub( 'notify_new_sessions' )

10
include/ClientGUIScrolledPanels.py
View File

@ -20,6 +20,11 @@ class ResizingScrolledPanel( wx.lib.scrolledpanel.ScrolledPanel ):
class EditPanel( ResizingScrolledPanel ):
def CanCancel( self ):
return True
def GetValue( self ):
raise NotImplementedError()
@ -27,6 +32,11 @@ class EditPanel( ResizingScrolledPanel ):
class ManagePanel( ResizingScrolledPanel ):
def CanCancel( self ):
return True
def CommitChanges( self ):
raise NotImplementedError()

14
include/ClientGUIScrolledPanelsEdit.py
View File

@ -1859,7 +1859,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
self._paused = wx.CheckBox( self._control_panel )
self._retry_failed = ClientGUICommon.BetterButton( self._control_panel, 'retry failed', self.RetryFailed )
self._retry_failures = ClientGUICommon.BetterButton( self._control_panel, 'retry failed', self.RetryFailures )
self._check_now_button = ClientGUICommon.BetterButton( self._control_panel, 'force check on dialog ok', self.CheckNow )
@ -1964,7 +1964,7 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
gridbox = ClientGUICommon.WrapInGrid( self._control_panel, rows )
self._control_panel.AddF( gridbox, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._retry_failed, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._retry_failures, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._check_now_button, CC.FLAGS_LONE_BUTTON )
self._control_panel.AddF( self._reset_cache_button, CC.FLAGS_LONE_BUTTON )
@ -2041,11 +2041,11 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
if no_failures:
self._retry_failed.Disable()
self._retry_failures.Disable()
else:
self._retry_failed.Enable()
self._retry_failures.Enable()
if can_check:
@ -2202,11 +2202,9 @@ class EditSubscriptionPanel( ClientGUIScrolledPanels.EditPanel ):
def RetryFailed( self ):
def RetryFailures( self ):
failed_seeds = self._seed_cache.GetSeeds( CC.STATUS_FAILED )
self._seed_cache.UpdateSeedsStatus( failed_seeds, CC.STATUS_UNKNOWN )
self._seed_cache.RetryFailures()
self._last_error = 0

188
include/ClientGUIScrolledPanelsManagement.py
View File

@ -1867,6 +1867,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._permit_watchers_to_name_their_pages = wx.CheckBox( thread_checker )
self._thread_watcher_not_found_page_string = ClientGUICommon.NoneableTextCtrl( thread_checker, none_phrase = 'do not show' )
self._thread_watcher_dead_page_string = ClientGUICommon.NoneableTextCtrl( thread_checker, none_phrase = 'do not show' )
watcher_options = self._new_options.GetDefaultThreadWatcherOptions()
self._thread_watcher_options = ClientGUIScrolledPanelsEdit.EditWatcherOptions( thread_checker, watcher_options )
@ -1879,6 +1882,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._permit_watchers_to_name_their_pages.SetValue( self._new_options.GetBoolean( 'permit_watchers_to_name_their_pages' ) )
self._thread_watcher_not_found_page_string.SetValue( self._new_options.GetNoneableString( 'thread_watcher_not_found_page_string' ) )
self._thread_watcher_dead_page_string.SetValue( self._new_options.GetNoneableString( 'thread_watcher_dead_page_string' ) )
#
rows = []
@ -1899,6 +1905,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows = []
rows.append( ( 'Permit thread checkers to name their own pages:', self._permit_watchers_to_name_their_pages ) )
rows.append( ( 'Prepend thread checker page names with this on 404:', self._thread_watcher_not_found_page_string ) )
rows.append( ( 'Prepend thread checker page names with this on death:', self._thread_watcher_dead_page_string ) )
gridbox = ClientGUICommon.WrapInGrid( thread_checker, rows )
@ -1925,6 +1933,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetDefaultThreadWatcherOptions( self._thread_watcher_options.GetValue() )
self._new_options.SetNoneableString( 'thread_watcher_not_found_page_string', self._thread_watcher_not_found_page_string.GetValue() )
self._new_options.SetNoneableString( 'thread_watcher_dead_page_string', self._thread_watcher_dead_page_string.GetValue() )
class _MaintenanceAndProcessingPanel( wx.Panel ):
@ -2478,6 +2489,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._default_gui_session = wx.Choice( self )
self._last_session_save_period_minutes = wx.SpinCtrl( self, min = 1, max = 1440 )
self._default_new_page_goes = ClientGUICommon.BetterChoice( self )
for value in [ CC.NEW_PAGE_GOES_FAR_LEFT, CC.NEW_PAGE_GOES_LEFT_OF_CURRENT, CC.NEW_PAGE_GOES_RIGHT_OF_CURRENT, CC.NEW_PAGE_GOES_FAR_RIGHT ]:
@ -2538,14 +2551,28 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
gui_session_names = HG.client_controller.Read( 'serialisable_names', HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION )
if 'last session' not in gui_session_names: gui_session_names.insert( 0, 'last session' )
if 'last session' not in gui_session_names:
gui_session_names.insert( 0, 'last session' )
self._default_gui_session.Append( 'just a blank page', None )
for name in gui_session_names: self._default_gui_session.Append( name, name )
for name in gui_session_names:
self._default_gui_session.Append( name, name )
try: self._default_gui_session.SetStringSelection( HC.options[ 'default_gui_session' ] )
except: self._default_gui_session.SetSelection( 0 )
try:
self._default_gui_session.SetStringSelection( HC.options[ 'default_gui_session' ] )
except:
self._default_gui_session.SetSelection( 0 )
self._last_session_save_period_minutes.SetValue( self._new_options.GetInteger( 'last_session_save_period_minutes' ) )
self._default_new_page_goes.SelectClientData( self._new_options.GetInteger( 'default_new_page_goes' ) )
@ -2595,6 +2622,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'Main gui title: ', self._main_gui_title ) )
rows.append( ( 'Default session on startup: ', self._default_gui_session ) )
rows.append( ( 'If \'last session\' above, autosave it how often (minutes)?', self._last_session_save_period_minutes ) )
rows.append( ( 'By default, new page tabs: ', self._default_new_page_goes ) )
rows.append( ( 'Confirm client exit: ', self._confirm_client_exit ) )
rows.append( ( 'Confirm sending files to trash: ', self._confirm_trash ) )
@ -2686,6 +2714,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetString( 'main_gui_title', title )
self._new_options.SetInteger( 'last_session_save_period_minutes', self._last_session_save_period_minutes.GetValue() )
self._new_options.SetInteger( 'default_new_page_goes', self._default_new_page_goes.GetChoice() )
self._new_options.SetInteger( 'max_page_name_chars', self._max_page_name_chars.GetValue() )
@ -2737,6 +2767,8 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._use_system_ffmpeg = wx.CheckBox( self )
self._use_system_ffmpeg.SetToolTipString( 'Check this to always default to the system ffmpeg in your path, rather than using the static ffmpeg in hydrus\'s bin directory. (requires restart)' )
self._anchor_and_hide_canvas_drags = wx.CheckBox( self )
self._media_zooms = wx.TextCtrl( self )
self._media_zooms.Bind( wx.EVT_TEXT, self.EventZoomsChanged )
@ -2754,6 +2786,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._load_images_with_pil.SetValue( self._new_options.GetBoolean( 'load_images_with_pil' ) )
self._do_not_import_decompression_bombs.SetValue( self._new_options.GetBoolean( 'do_not_import_decompression_bombs' ) )
self._use_system_ffmpeg.SetValue( self._new_options.GetBoolean( 'use_system_ffmpeg' ) )
self._anchor_and_hide_canvas_drags.SetValue( self._new_options.GetBoolean( 'anchor_and_hide_canvas_drags' ) )
media_zooms = self._new_options.GetMediaZooms()
@ -2785,6 +2818,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
rows.append( ( 'Load images with PIL: ', self._load_images_with_pil ) )
rows.append( ( 'Do not import Decompression Bombs: ', self._do_not_import_decompression_bombs ) )
rows.append( ( 'Prefer system FFMPEG: ', self._use_system_ffmpeg ) )
rows.append( ( 'WINDOWS ONLY: Hide and anchor mouse cursor on slow canvas drags: ', self._anchor_and_hide_canvas_drags ) )
rows.append( ( 'Media zooms: ', self._media_zooms ) )
gridbox = ClientGUICommon.WrapInGrid( self, rows )
@ -2882,6 +2916,7 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._new_options.SetBoolean( 'load_images_with_pil', self._load_images_with_pil.GetValue() )
self._new_options.SetBoolean( 'do_not_import_decompression_bombs', self._do_not_import_decompression_bombs.GetValue() )
self._new_options.SetBoolean( 'use_system_ffmpeg', self._use_system_ffmpeg.GetValue() )
self._new_options.SetBoolean( 'anchor_and_hide_canvas_drags', self._anchor_and_hide_canvas_drags.GetValue() )
try:
@ -3072,6 +3107,9 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
disk_panel = ClientGUICommon.StaticBox( self, 'disk cache' )
disk_cache_help_button = ClientGUICommon.BetterBitmapButton( disk_panel, CC.GlobalBMPs.help, self._ShowDiskCacheHelp )
disk_cache_help_button.SetToolTipString( 'Show help regarding the disk cache.' )
self._disk_cache_init_period = ClientGUICommon.NoneableSpinCtrl( disk_panel, 'max disk cache init period', none_phrase = 'do not run', min = 1, max = 120 )
self._disk_cache_init_period.SetToolTipString( 'When the client boots, it can speed up operation by reading the front of the database into memory. This sets the max number of seconds it can spend doing that.' )
@ -3165,8 +3203,19 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
#
help_hbox = wx.BoxSizer( wx.HORIZONTAL )
st = ClientGUICommon.BetterStaticText( disk_panel, 'help for this panel -->' )
st.SetForegroundColour( wx.Colour( 0, 0, 255 ) )
help_hbox.AddF( st, CC.FLAGS_VCENTER )
help_hbox.AddF( disk_cache_help_button, CC.FLAGS_VCENTER )
vbox = wx.BoxSizer( wx.VERTICAL )
disk_panel.AddF( help_hbox, CC.FLAGS_LONE_BUTTON )
disk_panel.AddF( self._disk_cache_init_period, CC.FLAGS_EXPAND_PERPENDICULAR )
disk_panel.AddF( self._disk_cache_maintenance_mb, CC.FLAGS_EXPAND_PERPENDICULAR )
@ -3268,6 +3317,21 @@ class ManageOptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
wx.CallAfter( self.Layout ) # draws the static texts correctly
def _ShowDiskCacheHelp( self ):
message = 'The hydrus database runs best on a drive with fast random access latency. Important and heavy read and write operations can function up to 100 times faster when started raw from an SSD rather than an HDD.'
message += os.linesep * 2
message += 'To get around this, the client populates a pre-boot and ongoing disk cache. By contiguously frontloading the database into memory, the most important functions do not need to wait on your disk for most of their work.'
message += os.linesep * 2
message += 'If you tend to leave your client on in the background and have a slow drive but a lot of ram, you might like to pump these numbers up. 15s boot cache and 2048MB ongoing can really make a difference on, for instance, a slow laptop drive.'
message += os.linesep * 2
message += 'If you run the database from an SSD, you can reduce or entirely eliminate these values, as the benefit is not so stark. 2s and 256MB is fine.'
message += os.linesep * 2
message += 'Unless you are testing, do not go crazy with this stuff. You can set 8192MB if you like, but there are diminishing returns.'
wx.MessageBox( message )
def EventFetchAuto( self, event ):
if self._fetch_ac_results_automatically.GetValue() == True:
@ -5253,12 +5317,7 @@ class ManageSubscriptionsPanel( ClientGUIScrolledPanels.ManagePanel ):
seed_cache = subscription.GetSeedCache()
failed_seeds = seed_cache.GetSeeds( CC.STATUS_FAILED )
for seed in failed_seeds:
seed_cache.UpdateSeedStatus( seed, CC.STATUS_UNKNOWN )
seed_cache.RetryFailures()
self._subscriptions.UpdateDatas( subscriptions )
@ -5338,6 +5397,23 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
def _GetServiceKeysToContentUpdates( self ):
service_keys_to_content_updates = {}
for page in self._tag_repositories.GetActivePages():
( service_key, content_updates ) = page.GetContentUpdates()
if len( content_updates ) > 0:
service_keys_to_content_updates[ service_key ] = content_updates
return service_keys_to_content_updates
def _OKParent( self ):
wx.PostEvent( self.GetParent(), wx.CommandEvent( commandType = wx.wxEVT_COMMAND_MENU_SELECTED, winid = ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetTemporaryId( 'ok' ) ) )
@ -5413,19 +5489,29 @@ class ManageTagsPanel( ClientGUIScrolledPanels.ManagePanel ):
def CanCancel( self ):
service_keys_to_content_updates = self._GetServiceKeysToContentUpdates()
if len( service_keys_to_content_updates ) > 0:
message = 'Are you sure you want to cancel? You have uncommitted changes that will be lost.'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
if dlg.ShowModal() != wx.ID_YES:
return False
return True
def CommitChanges( self ):
service_keys_to_content_updates = {}
for page in self._tag_repositories.GetActivePages():
( service_key, content_updates ) = page.GetContentUpdates()
if len( content_updates ) > 0:
service_keys_to_content_updates[ service_key ] = content_updates
service_keys_to_content_updates = self._GetServiceKeysToContentUpdates()
if len( service_keys_to_content_updates ) > 0:
@ -6121,6 +6207,9 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
self._url_input = wx.TextCtrl( self, style = wx.TE_PROCESS_ENTER )
self._url_input.Bind( wx.EVT_CHAR_HOOK, self.EventInputCharHook )
self._copy_button = ClientGUICommon.BetterButton( self, 'copy all', self._Copy )
self._paste_button = ClientGUICommon.BetterButton( self, 'paste', self._Paste )
self._urls_to_add = set()
self._urls_to_remove = set()
@ -6139,20 +6228,42 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
#
hbox = wx.BoxSizer( wx.HORIZONTAL )
hbox.AddF( self._copy_button, CC.FLAGS_VCENTER )
hbox.AddF( self._paste_button, CC.FLAGS_VCENTER )
vbox = wx.BoxSizer( wx.VERTICAL )
vbox.AddF( self._urls_listbox, CC.FLAGS_EXPAND_BOTH_WAYS )
vbox.AddF( self._url_input, CC.FLAGS_EXPAND_PERPENDICULAR )
vbox.AddF( hbox, CC.FLAGS_BUTTON_SIZER )
self.SetSizer( vbox )
wx.CallAfter( self._url_input.SetFocus )
def _EnterURL( self, url ):
def _Copy( self ):
urls = list( self._current_urls )
urls.sort()
text = os.linesep.join( urls )
HG.client_controller.pub( 'clipboard', 'text', text )
def _EnterURL( self, url, only_add = False ):
if url in self._current_urls:
if only_add:
return
for index in range( self._urls_listbox.GetCount() ):
existing_url = self._urls_listbox.GetClientData( index )
@ -6178,6 +6289,39 @@ class ManageURLsPanel( ClientGUIScrolledPanels.ManagePanel ):
def _Paste( self ):
if wx.TheClipboard.Open():
data = wx.TextDataObject()
wx.TheClipboard.GetData( data )
wx.TheClipboard.Close()
raw_text = data.GetText()
try:
for url in HydrusData.SplitByLinesep( raw_text ):
if url != '':
self._EnterURL( url, only_add = True )
except:
wx.MessageBox( 'I could not understand what was in the clipboard' )
else:
wx.MessageBox( 'I could not get permission to access the clipboard.' )
def _RemoveURL( self, index ):
url = self._urls_listbox.GetClientData( index )

184
include/ClientGUISeedCache.py
View File

@ -230,6 +230,160 @@ class EditSeedCachePanel( ClientGUIScrolledPanels.EditPanel ):
class SeedCacheButton( ClientGUICommon.BetterBitmapButton ):
def __init__( self, parent, controller, seed_cache_get_callable, seed_cache_set_callable = None ):
ClientGUICommon.BetterBitmapButton.__init__( self, parent, CC.GlobalBMPs.seed_cache, self._ShowSeedCacheFrame )
self._controller = controller
self._seed_cache_get_callable = seed_cache_get_callable
self._seed_cache_set_callable = seed_cache_set_callable
self.SetToolTipString( 'open detailed file import status--right-click for quick actions, if applicable' )
self.Bind( wx.EVT_RIGHT_DOWN, self.EventShowMenu )
def _ClearProcessed( self ):
message = 'Are you sure you want to delete all the processed (i.e. anything with a non-blank status in the larger window) files? This is useful for cleaning up and de-laggifying a very large list, but not much else.'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
if dlg.ShowModal() == wx.ID_YES:
seed_cache = self._seed_cache_get_callable()
seed_cache.RemoveProcessedSeeds()
def _RetryFailures( self ):
message = 'Are you sure you want to retry all the failed files?'
with ClientGUIDialogs.DialogYesNo( self, message ) as dlg:
if dlg.ShowModal() == wx.ID_YES:
seed_cache = self._seed_cache_get_callable()
seed_cache.RetryFailures()
def _ShowSeedCacheFrame( self ):
seed_cache = self._seed_cache_get_callable()
tlp = ClientGUICommon.GetTLP( self )
if isinstance( tlp, wx.Dialog ):
if self._seed_cache_set_callable is None: # throw up a dialog that edits the seed cache in place
with ClientGUITopLevelWindows.DialogNullipotent( self, 'file import status' ) as dlg:
panel = EditSeedCachePanel( dlg, self._controller, seed_cache )
dlg.SetPanel( panel )
dlg.ShowModal()
else: # throw up a dialog that edits the seed cache but can be cancelled
dupe_seed_cache = seed_cache.Duplicate()
with ClientGUITopLevelWindows.DialogEdit( self, 'file import status' ) as dlg:
panel = EditSeedCachePanel( dlg, self._controller, dupe_seed_cache )
dlg.SetPanel( panel )
if dlg.ShowModal() == wx.ID_OK:
self._seed_cache_set_callable( dupe_seed_cache )
else: # throw up a frame that edits the seed cache in place
title = 'file import status'
frame_key = 'file_import_status'
frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
panel = EditSeedCachePanel( frame, self._controller, seed_cache )
frame.SetPanel( panel )
def EventShowMenu( self, event ):
seed_cache = self._seed_cache_get_callable()
menu_items = []
num_failures = seed_cache.GetSeedCount( CC.STATUS_FAILED )
if num_failures > 0:
menu_items.append( ( 'normal', 'retry ' + HydrusData.ConvertIntToPrettyString( num_failures ) + ' failures', 'Tell this cache to reattempt all its failures.', self._RetryFailures ) )
num_unknown = seed_cache.GetSeedCount( CC.STATUS_UNKNOWN )
num_processed = len( seed_cache ) - num_unknown
if num_processed > 0:
menu_items.append( ( 'normal', 'delete ' + HydrusData.ConvertIntToPrettyString( num_processed ) + ' \'processed\' files from the queue', 'Tell this cache to clear out processed files, reducing the size of the queue.', self._ClearProcessed ) )
if len( menu_items ) > 0:
menu = wx.Menu()
for ( item_type, title, description, data ) in menu_items:
if item_type == 'normal':
func = data
ClientGUIMenus.AppendMenuItem( self, menu, title, description, func )
elif item_type == 'check':
check_manager = data
current_value = check_manager.GetCurrentValue()
func = check_manager.Invert
if current_value is not None:
ClientGUIMenus.AppendMenuCheckItem( self, menu, title, description, current_value, func )
elif item_type == 'separator':
ClientGUIMenus.AppendSeparator( menu )
HG.client_controller.PopupMenu( self, menu )
else:
event.Skip()
class SeedCacheStatusControl( wx.Panel ):
def __init__( self, parent, controller ):
@ -243,8 +397,7 @@ class SeedCacheStatusControl( wx.Panel ):
self._import_summary_st = ClientGUICommon.BetterStaticText( self )
self._progress_st = ClientGUICommon.BetterStaticText( self )
self._seed_cache_button = ClientGUICommon.BetterBitmapButton( self, CC.GlobalBMPs.seed_cache, self._ShowSeedCacheFrame )
self._seed_cache_button.SetToolTipString( 'open detailed file import status' )
self._seed_cache_button = SeedCacheButton( self, self._controller, self._GetSeedCache )
self._progress_gauge = ClientGUICommon.Gauge( self )
@ -274,32 +427,9 @@ class SeedCacheStatusControl( wx.Panel ):
self._update_timer = wx.Timer( self )
def _ShowSeedCacheFrame( self ):
def _GetSeedCache( self ):
tlp = ClientGUICommon.GetTLP( self )
if isinstance( tlp, wx.Dialog ):
with ClientGUITopLevelWindows.DialogNullipotent( self, 'file import status' ) as dlg:
panel = EditSeedCachePanel( dlg, self._controller, self._seed_cache )
dlg.SetPanel( panel )
dlg.ShowModal()
else:
title = 'file import status'
frame_key = 'file_import_status'
frame = ClientGUITopLevelWindows.FrameThatTakesScrollablePanel( self, title, frame_key )
panel = EditSeedCachePanel( frame, self._controller, self._seed_cache )
frame.SetPanel( panel )
return self._seed_cache
def _Update( self ):

22
include/ClientGUITopLevelWindows.py
View File

@ -308,6 +308,11 @@ class NewDialog( wx.Dialog ):
HG.client_controller.ResetIdleTimer()
def _CanCancel( self ):
return True
def EventMenuClose( self, event ):
menu = event.GetMenu()
@ -368,7 +373,17 @@ class NewDialog( wx.Dialog ):
def EventDialogButton( self, event ):
self.EndModal( event.GetId() )
event_id = event.GetId()
if event_id == wx.ID_CANCEL:
if not self._CanCancel():
return
self.EndModal( event_id )
class DialogThatResizes( NewDialog ):
@ -394,6 +409,11 @@ class DialogThatTakesScrollablePanel( DialogThatResizes ):
self.Bind( CC.EVT_SIZE_CHANGED, self.EventChildSizeChanged )
def _CanCancel( self ):
return self._panel.CanCancel()
def _GetButtonBox( self ):
raise NotImplementedError()

101
include/ClientImporting.py
View File

@ -370,7 +370,7 @@ class FileImportJob( object ):
new_options = HG.client_controller.GetNewOptions()
if mime in HC.IMAGES and new_options.GetBoolean( 'do_not_import_decompression_bombs' ):
if mime in HC.DECOMPRESSION_BOMB_IMAGES and new_options.GetBoolean( 'do_not_import_decompression_bombs' ):
if HydrusImageHandling.IsDecompressionBomb( self._temp_path ):
@ -2466,7 +2466,7 @@ HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIAL
class SeedCache( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_SEED_CACHE
SERIALISABLE_VERSION = 6
SERIALISABLE_VERSION = 7
def __init__( self ):
@ -2722,6 +2722,33 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
return ( 6, new_serialisable_info )
if version == 6:
new_serialisable_info = []
for ( seed, seed_info ) in old_serialisable_info:
try:
magic_phrase = '//media.tumblr.com'
replacement = '//data.tumblr.com'
if magic_phrase in seed:
seed = seed.replace( magic_phrase, replacement )
except:
pass
new_serialisable_info.append( ( seed, seed_info ) )
return ( 7, new_serialisable_info )
def AddSeeds( self, seeds ):
@ -2990,6 +3017,35 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
def RemoveProcessedSeeds( self ):
with self._lock:
seeds_to_delete = set()
for ( seed, seed_info ) in self._seeds_to_info.items():
if seed_info[ 'status' ] != CC.STATUS_UNKNOWN:
seeds_to_delete.add( seed )
for seed in seeds_to_delete:
del self._seeds_to_info[ seed ]
self._seeds_ordered.remove( seed )
self._seeds_to_indices = { seed : index for ( index, seed ) in enumerate( self._seeds_ordered ) }
self._SetDirty()
HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, seeds_to_delete )
def RemoveSeeds( self, seeds ):
with self._lock:
@ -3041,6 +3097,13 @@ class SeedCache( HydrusSerialisable.SerialisableBase ):
HG.client_controller.pub( 'seed_cache_seeds_updated', self._seed_cache_key, seeds_to_delete )
def RetryFailures( self ):
failed_seeds = self.GetSeeds( CC.STATUS_FAILED )
self.UpdateSeedsStatus( failed_seeds, CC.STATUS_UNKNOWN )
def UpdateSeedSourceTime( self, seed, source_timestamp ):
# this is ugly--this should all be moved to the seed when it becomes a cleverer object, rather than jimmying it through the cache
@ -4233,11 +4296,21 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
if self._thread_status == THREAD_STATUS_404:
page_name = '[404] ' + page_name
thread_watcher_not_found_page_string = new_options.GetNoneableString( 'thread_watcher_not_found_page_string' )
if thread_watcher_not_found_page_string is not None:
page_name = thread_watcher_not_found_page_string + ' ' + page_name
elif self._thread_status == THREAD_STATUS_DEAD:
page_name = '[DEAD] ' + page_name
thread_watcher_dead_page_string = new_options.GetNoneableString( 'thread_watcher_dead_page_string' )
if thread_watcher_dead_page_string is not None:
page_name = thread_watcher_dead_page_string + ' ' + page_name
if page_name != self._last_pubbed_page_name:
@ -4274,20 +4347,20 @@ class ThreadWatcherImport( HydrusSerialisable.SerialisableBase ):
else:
if self._watcher_options.IsDead( self._urls_cache, self._last_check_time ):
if self._thread_status != THREAD_STATUS_404:
if self._thread_status != THREAD_STATUS_404:
if self._watcher_options.IsDead( self._urls_cache, self._last_check_time ):
self._thread_status = THREAD_STATUS_DEAD
self._watcher_status = ''
self._thread_paused = True
else:
self._thread_status = THREAD_STATUS_OK
self._watcher_status = ''
self._thread_paused = True
else:
self._thread_status = THREAD_STATUS_OK
self._next_check_time = self._watcher_options.GetNextCheckTime( self._urls_cache, self._last_check_time )

81
include/ClientNetworking.py
View File

@ -2356,7 +2356,9 @@ class NetworkJob( object ):
else:
return self.engine.login_manager.NeedsLogin( self._network_contexts )
session_network_context = self._GetSessionNetworkContext()
return self.engine.login_manager.NeedsLogin( session_network_context )
@ -2729,73 +2731,6 @@ class NetworkJobThreadWatcher( NetworkJob ):
return self._network_contexts[-2] # the domain one
class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
SERIALISABLE_VERSION = 1
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
self.engine = None
self._lock = threading.Lock()
self._network_contexts_to_logins = {}
# a login has:
# a network_context it works for (PRIMARY KEY)
# a login script
# rules to check validity in cookies in a current session (fold that into the login script, which may have several stages of this)
# current user/pass/whatever
# current script validity
# current credentials validity
# recent error? some way of dealing with 'domain is currently down, so try again later'
# so, we fetch all the logins, ask them for the network contexts so we can set up the dict
def _GetSerialisableInfo( self ):
return {}
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
self._network_contexts_to_logins = {}
def CanLogin( self, network_contexts ):
# look them up in our structure
# if they have a login, is it valid?
# valid means we have tested credentials and it hasn't been invalidated by a parsing error or similar
# I think this just means saying Login.CanLogin( credentials )
return False
def GenerateLoginProcess( self, network_contexts ):
# look up the logins
# login_process = Login.GenerateLoginProcess
# return login_process
raise NotImplementedError()
def NeedsLogin( self, network_contexts ):
# look up the network contexts in our structure
# if they have a login, see if they match the 'is logged in' predicates
# otherwise:
return False
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager
class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER
@ -2861,6 +2796,8 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
del self._network_contexts_to_sessions[ network_context ]
self._SetDirty()
@ -2873,6 +2810,14 @@ class NetworkSessionManager( HydrusSerialisable.SerialisableBase ):
self._network_contexts_to_sessions[ network_context ] = self._GenerateSession( network_context )
# tumblr can't into ssl for some reason, and the data subdomain they use has weird cert properties, looking like amazon S3
# perhaps it is inward-facing somehow? whatever the case, let's just say fuck it for tumblr
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN and network_context.context_data == 'tumblr.com':
self._network_contexts_to_sessions[ network_context ].verify = False
self._SetDirty()
return self._network_contexts_to_sessions[ network_context ]

467
include/ClientNetworkingLogin.py
View File

@ -1,8 +1,19 @@
import ClientConstants as CC
import ClientDefaults
import ClientDownloading
import ClientNetworking
import ClientNetworkingDomain
import HydrusConstants as HC
import HydrusGlobals as HG
import HydrusData
import HydrusExceptions
import HydrusSerialisable
import json
import requests
import threading
import time
import urllib
VALIDITY_VALID = 0
VALIDITY_UNTESTED = 1
@ -20,8 +31,449 @@ class LoginCredentials( object ):
# test current values, including fail for not having enough/having too many
class NetworkLoginManager( HydrusSerialisable.SerialisableBase ):
SERIALISABLE_TYPE = HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER
SERIALISABLE_VERSION = 1
SESSION_TIMEOUT = 60 * 60
def __init__( self ):
HydrusSerialisable.SerialisableBase.__init__( self )
self.engine = None
self._lock = threading.Lock()
self._domains_to_logins = {}
# a login has:
# a login script
# rules to check validity in cookies in a current session (fold that into the login script, which may have several stages of this)
# current user/pass/whatever
# current script validity
# current credentials validity
# recent error? some way of dealing with 'domain is currently down, so try again later'
# so, we fetch all the logins, ask them for the network contexts so we can set up the dict
# variables from old object here
self._error_names = set()
self._network_contexts_to_session_timeouts = {}
def _GetSerialisableInfo( self ):
return {}
def _InitialiseFromSerialisableInfo( self, serialisable_info ):
self._network_contexts_to_logins = {}
def CanLogin( self, network_contexts ):
# look them up in our structure
# if they have a login, is it valid?
# valid means we have tested credentials and it hasn't been invalidated by a parsing error or similar
# I think this just means saying Login.CanLogin( credentials )
return False
def GenerateLoginProcess( self, network_contexts ):
# look up the logins
# login_process = Login.GenerateLoginProcess
# return login_process
raise NotImplementedError()
def NeedsLogin( self, network_context ):
with self._lock:
if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
nc_domain = network_context.context_data
domains = ClientNetworkingDomain.ConvertDomainIntoAllApplicableDomains( nc_domain )
for domain in domains:
if domain in self._domains_to_logins:
# fetch session
# does the login script reckon the session is logged in?
# if not, return True
pass
elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
service_key = network_context.context_data
# figure it out here
return False
# these methods are from the old object:
def _GetCookiesDict( self, network_context ):
session = self._GetSession( network_context )
cookies = session.cookies
cookies.clear_expired_cookies()
domains = cookies.list_domains()
for domain in domains:
if domain.endswith( network_context.context_data ):
return cookies.get_dict( domain )
return {}
def _GetSession( self, network_context ):
session = self.engine.controller.network_engine.session_manager.GetSession( network_context )
if network_context not in self._network_contexts_to_session_timeouts:
self._network_contexts_to_session_timeouts[ network_context ] = 0
if HydrusData.TimeHasPassed( self._network_contexts_to_session_timeouts[ network_context ] ):
session.cookies.clear_session_cookies()
self._network_contexts_to_session_timeouts[ network_context ] = HydrusData.GetNow() + self.SESSION_TIMEOUT
return session
def _IsLoggedIn( self, network_context, required_cookies ):
cookie_dict = self._GetCookiesDict( network_context )
for name in required_cookies:
if name not in cookie_dict:
return False
return True
def EnsureHydrusSessionIsOK( self, service_key ):
with self._lock:
if not self.engine.controller.services_manager.ServiceExists( service_key ):
raise HydrusExceptions.DataMissing( 'Service does not exist!' )
name = self.engine.controller.services_manager.GetService( service_key ).GetName()
if service_key in self._error_names:
raise Exception( 'Could not establish a hydrus network session for ' + name + '! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, service_key )
required_cookies = [ 'session_key' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
self.SetupHydrusSession( service_key )
if not self._IsLoggedIn( network_context, required_cookies ):
return
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( service_key )
raise
def EnsureLoggedIn( self, name ):
with self._lock:
if name in self._error_names:
raise Exception( name + ' could not establish a session! This ugly error is temporary due to the network engine rewrite. Please restart the client to reattempt this network context.' )
if name == 'hentai foundry':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )
required_cookies = [ 'PHPSESSID', 'YII_CSRF_TOKEN' ]
elif name == 'pixiv':
network_context = ClientNetworking.NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
required_cookies = [ 'PHPSESSID' ]
if self._IsLoggedIn( network_context, required_cookies ):
return
try:
if name == 'hentai foundry':
self.LoginHF( network_context )
elif name == 'pixiv':
result = self.engine.controller.Read( 'serialisable_simple', 'pixiv_account' )
if result is None:
raise HydrusExceptions.DataMissing( 'You need to set up your pixiv credentials in services->manage pixiv account.' )
( pixiv_id, password ) = result
self.LoginPixiv( network_context, pixiv_id, password )
if not self._IsLoggedIn( network_context, required_cookies ):
raise Exception( name + ' login did not work correctly!' )
HydrusData.Print( 'Successfully logged into ' + name + '.' )
except:
self._error_names.add( name )
raise
def LoginHF( self, network_context ):
session = self._GetSession( network_context )
response = session.get( 'https://www.hentai-foundry.com/' )
time.sleep( 1 )
response = session.get( 'https://www.hentai-foundry.com/?enterAgree=1' )
time.sleep( 1 )
cookie_dict = self._GetCookiesDict( network_context )
raw_csrf = cookie_dict[ 'YII_CSRF_TOKEN' ] # 19b05b536885ec60b8b37650a32f8deb11c08cd1s%3A40%3A%222917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32%22%3B
processed_csrf = urllib.unquote( raw_csrf ) # 19b05b536885ec60b8b37650a32f8deb11c08cd1s:40:"2917dcfbfbf2eda2c1fbe43f4d4c4ec4b6902b32";
csrf_token = processed_csrf.split( '"' )[1] # the 2917... bit
hentai_foundry_form_info = ClientDefaults.GetDefaultHentaiFoundryInfo()
hentai_foundry_form_info[ 'YII_CSRF_TOKEN' ] = csrf_token
response = session.post( 'http://www.hentai-foundry.com/site/filters', data = hentai_foundry_form_info )
time.sleep( 1 )
# This updated login form is cobbled together from the example in PixivUtil2
# it is breddy shid because I'm not using mechanize or similar browser emulation (like requests's sessions) yet
# Pixiv 400s if cookies and referrers aren't passed correctly
# I am leaving this as a mess with the hope the eventual login engine will replace it
def LoginPixiv( self, network_context, pixiv_id, password ):
session = self._GetSession( network_context )
response = session.get( 'https://accounts.pixiv.net/login' )
soup = ClientDownloading.GetSoup( response.content )
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
i = soup.find( 'input', id = 'init-config' )
raw_json = i['value']
j = json.loads( raw_json )
if 'pixivAccount.postKey' not in j:
raise HydrusExceptions.ForbiddenException( 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
post_key = j[ 'pixivAccount.postKey' ]
form_fields = {}
form_fields[ 'pixiv_id' ] = pixiv_id
form_fields[ 'password' ] = password
form_fields[ 'captcha' ] = ''
form_fields[ 'g_recaptcha_response' ] = ''
form_fields[ 'return_to' ] = 'https://www.pixiv.net'
form_fields[ 'lang' ] = 'en'
form_fields[ 'post_key' ] = post_key
form_fields[ 'source' ] = 'pc'
headers = {}
headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
headers[ 'origin' ] = "https://accounts.pixiv.net"
session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
time.sleep( 1 )
def SetupHydrusSession( self, service_key ):
# nah, replace this with a proper login script
service = self.engine.controller.services_manager.GetService( service_key )
if not service.HasAccessKey():
raise HydrusExceptions.DataMissing( 'No access key for this service, so cannot set up session!' )
access_key = service.GetAccessKey()
url = 'blah'
network_job = ClientNetworking.NetworkJobHydrus( service_key, 'GET', url )
network_job.SetForLogin( True )
network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.encode( 'hex' ) )
self.engine.controller.network_engine.AddJob( network_job )
network_job.WaitUntilDone()
def TestPixiv( self, pixiv_id, password ):
# this is just an ugly copy, but fuck it for the minute
# we'll figure out a proper testing engine later with the login engine and tie the manage gui into it as well
session = requests.Session()
response = session.get( 'https://accounts.pixiv.net/login' )
soup = ClientDownloading.GetSoup( response.content )
# some whocking 20kb bit of json tucked inside a hidden form input wew lad
i = soup.find( 'input', id = 'init-config' )
raw_json = i['value']
j = json.loads( raw_json )
if 'pixivAccount.postKey' not in j:
return ( False, 'When trying to log into Pixiv, I could not find the POST key! This is a problem with hydrus\'s pixiv parsing, not your login! Please contact hydrus dev!' )
post_key = j[ 'pixivAccount.postKey' ]
form_fields = {}
form_fields[ 'pixiv_id' ] = pixiv_id
form_fields[ 'password' ] = password
form_fields[ 'captcha' ] = ''
form_fields[ 'g_recaptcha_response' ] = ''
form_fields[ 'return_to' ] = 'https://www.pixiv.net'
form_fields[ 'lang' ] = 'en'
form_fields[ 'post_key' ] = post_key
form_fields[ 'source' ] = 'pc'
headers = {}
headers[ 'referer' ] = "https://accounts.pixiv.net/login?lang=en^source=pc&view_type=page&ref=wwwtop_accounts_index"
headers[ 'origin' ] = "https://accounts.pixiv.net"
r = session.post( 'https://accounts.pixiv.net/api/login?lang=en', data = form_fields, headers = headers )
if not r.ok:
HydrusData.ShowText( r.content )
return ( False, 'Login request failed! Info printed to log.' )
cookies = session.cookies
cookies.clear_expired_cookies()
domains = cookies.list_domains()
for domain in domains:
if domain.endswith( 'pixiv.net' ):
d = cookies.get_dict( domain )
if 'PHPSESSID' not in d:
HydrusData.ShowText( r.content )
return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
return ( True, '' )
HydrusData.ShowText( r.content )
return ( False, 'Pixiv login failed to establish session! Info printed to log.' )
HydrusSerialisable.SERIALISABLE_TYPES_TO_OBJECT_TYPES[ HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_LOGIN_MANAGER ] = NetworkLoginManager
# make this serialisable
class LoginScript( object ):
class LoginProcess( object ):
def __init__( self ):
@ -29,6 +481,8 @@ class LoginScript( object ):
self._login_steps = []
self._validity = VALIDITY_UNTESTED
# possible error info
self._temp_variables = {}
@ -36,6 +490,7 @@ class LoginScript( object ):
# this maybe takes some job_key or something so it can present to the user login process status
# this will be needed in the dialog where we test this. we need good feedback on how it is going
# irl, this could be a 'login popup' message as well, just to inform the user on the progress of any delay
for step in self._login_steps:
@ -45,7 +500,15 @@ class LoginScript( object ):
except HydrusExceptions.VetoException: # or something--invalidscript exception?
# also figure out a way to deal with connection errors and so on which will want a haderror time delay before giving it another go
# set error info
self._validity = VALIDITY_INVALID
return False
except Exception as e:
# set error info
self._validity = VALIDITY_INVALID

4
include/HydrusConstants.py
View File

@ -49,7 +49,7 @@ options = {}
# Misc
NETWORK_VERSION = 18
SOFTWARE_VERSION = 277
SOFTWARE_VERSION = 278
UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )
@ -434,6 +434,8 @@ APPLICATION_UNKNOWN = 101
ALLOWED_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV, APPLICATION_HYDRUS_UPDATE_CONTENT, APPLICATION_HYDRUS_UPDATE_DEFINITIONS )
SEARCHABLE_MIMES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, APPLICATION_FLASH, VIDEO_AVI, VIDEO_FLV, VIDEO_MOV, VIDEO_MP4, VIDEO_MKV, VIDEO_WEBM, VIDEO_MPEG, APPLICATION_PDF, APPLICATION_ZIP, APPLICATION_RAR, APPLICATION_7Z, AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA, VIDEO_WMV )
DECOMPRESSION_BOMB_IMAGES = ( IMAGE_JPEG, IMAGE_PNG )
IMAGES = ( IMAGE_JPEG, IMAGE_PNG, IMAGE_APNG, IMAGE_GIF, IMAGE_BMP )
AUDIO = ( AUDIO_MP3, AUDIO_OGG, AUDIO_FLAC, AUDIO_WMA )

11
include/TestClientNetworking.py
View File

@ -1,6 +1,7 @@
import ClientConstants as CC
import ClientNetworking
import ClientNetworkingDomain
import ClientNetworkingLogin
import collections
import HydrusConstants as HC
import HydrusData
@ -222,7 +223,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@ -252,7 +253,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@ -280,7 +281,7 @@ class TestNetworkingEngine( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@ -332,7 +333,7 @@ class TestNetworkingJob( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )
@ -541,7 +542,7 @@ class TestNetworkingJobHydrus( unittest.TestCase ):
bandwidth_manager = ClientNetworking.NetworkBandwidthManager()
session_manager = ClientNetworking.NetworkSessionManager()
domain_manager = ClientNetworkingDomain.NetworkDomainManager()
login_manager = ClientNetworking.NetworkLoginManager()
login_manager = ClientNetworkingLogin.NetworkLoginManager()
engine = ClientNetworking.NetworkEngine( mock_controller, bandwidth_manager, session_manager, domain_manager, login_manager )

1
test.py
View File

@ -139,7 +139,6 @@ class Controller( object ):
self._managers[ 'tag_siblings' ] = ClientCaches.TagSiblingsManager( self )
self._managers[ 'tag_parents' ] = ClientCaches.TagParentsManager( self )
self._managers[ 'undo' ] = ClientCaches.UndoManager( self )
self._managers[ 'web_sessions' ] = TestConstants.FakeWebSessionManager()
self._server_session_manager = HydrusSessions.HydrusSessionManagerServer()
self._managers[ 'local_booru' ] = ClientCaches.LocalBooruCache( self )