From 07333d60c579fc17ab147a746db2e6a6a7ba14e2 Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 13 Jul 2022 16:35:17 -0500 Subject: [PATCH] Version 491 --- db/help my db is broke.txt | 23 +- db/help my db is not booting.txt | 4 +- db/help my media files are broke.txt | 7 + docs/changelog.md | 76 +++-- docs/developer_api.md | 15 +- docs/launch_arguments.md | 4 +- docs/old_changelog.html | 47 +++ hydrus/client/ClientApplicationCommand.py | 2 +- hydrus/client/ClientSearch.py | 24 +- hydrus/client/ClientServices.py | 79 ++++- hydrus/client/ClientStrings.py | 149 +++++++- hydrus/client/db/ClientDB.py | 34 ++ hydrus/client/db/ClientDBFilesStorage.py | 2 +- .../ClientDBMappingsCacheSpecificDisplay.py | 2 +- .../ClientDBMappingsCacheSpecificStorage.py | 2 +- hydrus/client/db/ClientDBMappingsCounts.py | 2 +- hydrus/client/db/ClientDBMappingsStorage.py | 2 +- hydrus/client/db/ClientDBRepositories.py | 2 +- hydrus/client/db/ClientDBTagParents.py | 2 +- hydrus/client/db/ClientDBTagSearch.py | 2 +- hydrus/client/db/ClientDBTagSiblings.py | 2 +- .../ClientExportingFiles.py} | 138 +++----- .../exporting/ClientExportingMetadata.py | 321 ++++++++++++++++++ hydrus/client/exporting/__init__.py | 1 + hydrus/client/gui/ClientGUI.py | 91 ++--- hydrus/client/gui/ClientGUIDragDrop.py | 10 +- hydrus/client/gui/ClientGUIExport.py | 239 +++++++------ hydrus/client/gui/ClientGUIFunctions.py | 2 +- hydrus/client/gui/ClientGUIRatings.py | 135 ++++++-- .../gui/ClientGUIScrolledPanelsManagement.py | 2 +- .../gui/ClientGUIScrolledPanelsReview.py | 37 +- hydrus/client/gui/ClientGUIStringPanels.py | 143 +++++++- hydrus/client/gui/ClientGUITags.py | 4 + hydrus/client/gui/canvas/ClientGUICanvas.py | 2 +- .../gui/canvas/ClientGUICanvasHoverFrames.py | 33 +- .../gui/lists/ClientGUIListConstants.py | 2 +- hydrus/client/gui/pages/ClientGUIResults.py | 4 +- .../client/gui/search/ClientGUIACDropdown.py | 119 +++++-- hydrus/client/gui/search/ClientGUILocation.py | 7 +- .../services/ClientGUIClientsideServices.py | 4 + .../importing/options/TagImportOptions.py | 49 +-- hydrus/client/metadata/ClientTagsHandling.py | 11 +- hydrus/core/HydrusConstants.py | 2 +- hydrus/core/HydrusDBBase.py | 29 +- hydrus/core/HydrusDBModule.py | 45 ++- hydrus/core/HydrusData.py | 2 +- hydrus/core/HydrusPaths.py | 45 ++- hydrus/core/HydrusSerialisable.py | 4 + hydrus/hydrus_client.py | 18 +- hydrus/hydrus_server.py | 18 +- hydrus/test/TestClientDB.py | 4 +- hydrus/test/TestHydrusNetworking.py | 4 +- .../parsers/4chan-style thread api parser.png | Bin 2605 -> 2696 bytes 53 files changed, 1484 insertions(+), 523 deletions(-) rename hydrus/client/{ClientExporting.py => exporting/ClientExportingFiles.py} (85%) create mode 100644 hydrus/client/exporting/ClientExportingMetadata.py create mode 100644 hydrus/client/exporting/__init__.py diff --git a/db/help my db is broke.txt b/db/help my db is broke.txt index 4a7686cc..46fec7da 100644 --- a/db/help my db is broke.txt +++ b/db/help my db is broke.txt @@ -1,3 +1,12 @@ +*** The purpose of this document *** + +This is mostly an explainer for a specific error some databases encounter, 'database image is malformed'. The cloning and repair steps only relate to this specific problem, but as this problem is caused by hard drive damage, users who have other hard drive problems are pointed here as background reading. + +Please take a breath and read carefully. Search online if there is anything you don't understand, and plan out what you are going to do. Whatever the problem, do not try to rush it while you are stressed out, since that often causes mistakes. + + +*** Malformed Database *** + If you are getting weird but serious-sounding errors like 'database image is malformed', your database may be corrupt! If your first language is not english, 'db' means 'database'. @@ -91,7 +100,7 @@ The four files are separate, so if one is broken but the other three are fine, y *** fix the problem *** -There are three known repair operations. Try one and see if you can pass an integrity_check on the broken file. Clone works well for most simple problems, but the others have better success in different situations. +There are three known repair operations. Try one and see if you can pass an integrity_check on the broken file. Clone works well for almost all simple problems, but the others have better success in different situations. ** clone ** @@ -127,7 +136,7 @@ Do not delete your original files for now. Just rename them to 'old' and move th ** repair ** -This command tries to fix a database file in place. It works very very slowly. While it may be able to recover some rows a clones would lose, it cannot fix everything and may leave you with a database that is still malformed, so you'll want to run integrity_check again and do a clone if needed. +This command tries to fix a database file in place. I do not recommend it as it works very very slowly. While it may be able to recover some rows a clones would lose, it cannot fix everything and may leave you with a database that is still malformed, so you'll want to run integrity_check again and do a clone if needed. It is very important you have a backup copy of the broken file(s) here, as this edits the file in place. @@ -216,15 +225,15 @@ If you are absolutely certain you restored your database files to the point wher If you are in this situation, I am happy to help you figure things out, but you know your system's hardware better than I do. As far as I know, I cannot create a 'malformed' error with my database access routines even if wanted to--it can only come from an external fault. -*** a note on Write Caching and 'Delayed Write Failed' *** +*** a note on Write Caching and 'Delayed Write Failed' on NVMe drives *** -Some users encountered 'malformed' problems alongside Windows OS errors about 'Delayed Write Failed' on the hydrus directory, usually after sleeping and then waking the computer. +Some users encountered 'malformed' or 'disk I/O error' problems alongside Windows OS errors about 'Delayed Write Failed' on the hydrus directory, usually after sleeping and then waking the computer, but also after the database has finished a lot of work. -This seems to be something to do with Write Caching failing for certain very large WAL journal files. I believe I have the underlying cause of large WAL files fixed, but if you nonetheless get this specific error, the two things to try seem to be: +This seems to be something to do with Write Caching failing for certain very large WAL journal files, and particularly affects some NVMe drive(r)s. I have attempted to relieve the large WAL files, but 'disk I/O error' is still happening for some users. The two things to try seem to be: -1) Disable Write Caching on the drive. This sucks, since write caching is normally really useful, but it seems to help/fix it completely, so it may be ok for a hydrus-exclusive drive. +1) Launch the program with "--db_journal_mode TRUNCATE" argument, turning off WAL journaling. Check the 'launch arguments' article in the help for more info. This generally fixes the problem completely. -2) Launch the program with "--db_journal_mode TRUNCATE" argument, turning off WAL journaling. Check the 'launch arguments' article in the help for more info. +2) Disable Write Caching on the drive. This sucks, since write caching is normally really useful, but it seems to help/fix it completely, so it may be ok for a hydrus-exclusive drive. *** lastly, if you did not have any backup before this *** diff --git a/db/help my db is not booting.txt b/db/help my db is not booting.txt index 908e3198..67b9220d 100644 --- a/db/help my db is not booting.txt +++ b/db/help my db is not booting.txt @@ -4,6 +4,6 @@ So, if your client seems suddenly to take a very long time to start up, just sit You can double-check this by looking at the client executable in your OS's Task Manager (Ctrl+Shift+Esc on Windows). If it is doing some CPU/HDD, there is no need to kill the process. -Please contact hydrus_dev if it really does seem stuck (say, no progress after an hour, or if CPU/HDD activity completely drops to nothing), or if every startup is delayed like this. +Please contact hydrus_dev if it really does seem stuck (say, no progress after an hour, or if CPU/HDD activity completely drops to nothing for several minutes), or if every startup is delayed like this. -One possible cause of delayed startup every time (usually a delay _before_ the splash screen appears) is overly paranoid virus scanners rechecking all of hydrus every time it starts. To relieve this, please check your anti-virus software's options and make sure your hard drive is defragged. \ No newline at end of file +One possible cause of delayed startup every time (usually a delay _before_ the splash screen appears) is overly paranoid virus scanners rechecking all of hydrus every time it starts. To relieve this, please check your anti-virus software's options and make sure, if you have an HDD, that your disk is defragged. \ No newline at end of file diff --git a/db/help my media files are broke.txt b/db/help my media files are broke.txt index 507c20dc..cc8039fa 100644 --- a/db/help my media files are broke.txt +++ b/db/help my media files are broke.txt @@ -1,3 +1,10 @@ +*** The purpose of this document *** + +If you have missing files or missing file records, this document will walk you through the specific steps on how to fix it. If you have encountered a related problem, such as losing one of your 'fxx' sub-folders, it may also help as background reading. + + +*** Missing Files *** + If you have lost lots of media files due to a mistake or a drive failing, or if you have rolled back to an older backup and now your file structure is out of sync with your database record, there are several things to do. First off, make sure you are running on good hard drives now. Check the 'help my db is broke' help document for info here. If your .db files were also on a failing disk, you'll want to run through that whole document to make sure your database itself is ok--there's no point trying to fix the file record on a malformed database. diff --git a/docs/changelog.md b/docs/changelog.md index 4bb28267..a172fb1b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,53 @@ !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). +## [Version 491](https://github.com/hydrusnetwork/hydrus/releases/tag/v491) + +### system predicates +* the advanced OR input, where you can type tags in complicated logical expressions, now supports system predicates! most system predicates are supported using their typical display strings. it uses the same engine as the client api, so check the examples here https://hydrusnetwork.github.io/hydrus/developer_api.html#get_files_search_files sorry for the delay here +* the advanced input also runs tags better through the hydrus tag 'cleaning' process, so things like whitespace between the namespace colon and the subtag are cleaned up correctly, and invalid tags should be excluded +* it also starts with the keyboard focus in the text input +* and I think I fixed an issue with '!'', 'not', or '-' negation prefixes not parsing +* highlighted the example parseable system predicate texts in the Client API help, and added 'last viewed' to it + +### misc +* altering your services in _manage services_ no longer causes a full page refresh for all currently open search pages +* in a related thing, if you click the file or tag domain of a file search page to be the same as it just was, you no longer get a page refresh +* the rating widgets now show their current rating value on their tooltips +* when setting a numerical rating by a drag, it no longer matters if your mouse strays above or below the widget--it will still set +* the String Processing system has a new 'String Tag Filter' processing step. this applies the normal tag filtering object to your list of strings and also performs the hydrus 'tag cleaning' process on them, making them all lowercase and trimming whitespace and so on +* the sibling/parent sync is now even more polite when told to do work in 'normal' time. this has been hitting a lot of new users really hard, so it should now really trickle work during normal time, throttling down when it hits a bump to avoid stunlocking you but also responding quickly to recent changes if you are fully synced +* the database repair code is now better at healing damaged fast-text-search (FTS) tables. previously, in cases of partial damage to the virtual table, the repair code would error out +* fixed a bug where certain search predicate calendar dates that are acceptable in Linux but not in Windows caused Windows to fail to load the session. if you put in 1965 as a search date, it should now revert to the current time one next load etc... +* the test to see if a directory is writeable-to is improved and now handles Windows's Program Files directory correctly +* improved how the boot scripts handle incorrect/bad database directory paths. the error handling works better, and it figures out a fallback location for crash.log better +* a new button on 'review services' now lets advanced users copy the service key to the clipboard +* the migrate tags dialog now lists file repositories, ipfs services, and 'all my files' as potential file filter domains +* when checking it has space for a large transaction like a vacuum, hydrus now tries to check if you are running on a ramdisk or other severely space-limited temp dir and offers more text if this is true +* updated the '4chan style thread api parser' to handle posts with multiple files, which fixes tvchan.moe and probably anything else running NPFchan +* some logic testing around showing 'return to inbox' and the actual operation is fixed so it only applies to local files. in some weird advanced situations, you could previously send deleted files to inbox + +### new import/export framework +* started a new modular metadata import/export pipeline. this thing starts out today by doing the work of newline-separated tags in a .txt sidecar file and will expand to do all sorts of metadata in other formats like JSON and XML. it will also, eventually, support arbitrary cross-type conversions like tags to urls or ratings to tags +* export folders now support '.txt' sidecar tag exporting! +* the '.txt' sidecar tag importing in import folders or manual imports is now handled by the new pipeline +* the '.txt' sidecar exporting in the manual export dialog is now handled by the new pipeline +* please expect the UI around '.txt' sidecar importing and exporting to change significantly in future. you'll be selecting different metadata types to import or export, make string processing steps to alter or filter what you get, and of course be able to compile it all into more complicated filetypes + +### cleanup and refactoring +* mr bones gets two new columns to line up the numbers better +* a bunch of export code got moved around. created a new module 'exporting', and moved ClientExporting.py to it, renaming to ClientExportingFiles.py +* removed an old prototype for sidecar exporting and related plans for UI +* the 'missing file folders on boot' dialog now points users to 'help my media files are broke.txt' +* brushed up the 'help my x is broke.txt' documents in the database directory a little +* fixed some surplus double backslashes in the help +* a secret tiny label change/fix, let's see if anyone notices +* cleaned up how the rating widgets manage and update rating state. it was ancient bad code +* updated how different rating values are converted to UI text +* misc cleanup of some free space checking code +* fixed some bad quote characters in client api help JSON examples +* improved some error handling for uploading pending content and sped up file uploads a little + ## [Version 490](https://github.com/hydrusnetwork/hydrus/releases/tag/v490) ### misc @@ -293,32 +340,3 @@ * added a unit test to test this * added archive timestamp and hash hex sort enum definitions to the 'search_files' client api help * client api version is now 31 - -## [Version 480](https://github.com/hydrusnetwork/hydrus/releases/tag/v480) - -### file notes and media viewer hover windows -* file notes are now shown on the media viewer! this is a first version, pretty ugly, and may have font layout bugs for some systems, but it works. they hang just below the top-right hover, both in the canvas background and with their own hover if you mouseover. clicking on any note will open 'edit notes' on that note -* the duplicate filter's always-on hover _should_ slide out of the way when there are many notes -* furthermore, I rewrote the backend of hover windows. they are now embedded into the media viewer rather than being separate frameless toolbar windows. this should relieve several problems different users had--for instance, if you click a hover, you now no longer lose focus on the main media viewer window. I hacked some of this to get it to work, but along the way I undid three other hacks, so overall it should be better. please let me know how this works for you! -* fixed a long time hover window positioning bug where the top-right window would sometimes pop in for a frame the first time you moved the mouse to the top middle before repositioning and hiding itself again -* removed the 'notes' icon from the top right hover window -* refactored a bunch of canvas background code - -### client api -* search_files/get_thumbnail now returns image/jpeg or image/png Content-Type. it _should_ be super fast, but let me know if it lags after 3k thumbs or something -* you can now ask for CBOR or JSON specifically by using the 'Accept' request header, regardless of your own request Content-Type (issue #1110) -* if you send or ask for CBOR but it is not available for that client, you now get a new 'Not Acceptable' 406 response (previously it would 500 or 200 but in JSON) -* updated the help regarding the above and wrote some unit tests to check CBOR/JSON requests and responses -* client api version is now 30 - -### misc -* added a link to 'Hyshare', at https://github.com/floogulinc/hyshare, to the Client API help. it is a neat way to share galleries with friends, just like the the old 'local booru' -* building on last week's shift-select improvement, I tweaked it and shift-select and ctrl-select are back to not setting the preview focus. you can ctrl-click a bunch of vids in quick silence again -* the menu on the 'file log' button is now attached to the downloader page lists and the menu when you right-click on the file log panel. you can now access these actions without having to highlight a big query -* the same is also true of the search/check log! -* when you select a new downloader in the gallery download page, the keyboard focus now moves immediately to the query text input box -* tweaked the zoom locking code in the duplicate filter again. the 'don't lock that way if there is spillover' test, which is meant to stop garbage site banners from being hidden just offscreen, is much more strict. it now only cares about 10% or so spillover, assuming that with a large 'B' the spillover will be obvious. this should improve some odd zoom locking situations where the first pair change was ok and the rest were weird -* if you exit the client before the first session loads (either it is really huge or a problem breaks/delays your boot) the client will not save any 'last/exit session' (previously, it was saving empty here, requiring inconvenient load from a backup) -* if you have a really really huge session, the client is now more careful about not booting delayed background tasks like subscriptions until the session is in place -* on 'migrate database', the thumbnail size estimate now has a min-max range and a tooltip to clarify that it is an estimate -* fixed a bug in the new 'sort by file hash' pre-sort when applying system:limit diff --git a/docs/developer_api.md b/docs/developer_api.md index 668fb594..b68ab78b 100644 --- a/docs/developer_api.md +++ b/docs/developer_api.md @@ -256,10 +256,10 @@ Response: "service_key": "616c6c206c6f63616c2066696c6573" } ], - 'all_local_media': [ + "all_local_media": [ { - 'name': 'all my files', - 'service_key': '616c6c206c6f63616c206d65646961' + "name": "all my files", + "service_key": "616c6c206c6f63616c206d65646961" } ], "all_known_files": [ @@ -301,7 +301,7 @@ Arguments (in JSON): : - `path`: (the path you want to import) ```json title="Example request body" -{"path": "E:\\to_import\\ayanami.jpg"} +{"path": "E:\to_import\ayanami.jpg"} ``` Arguments (as bytes): @@ -1286,7 +1286,7 @@ If the access key's permissions only permit search for certain tags, at least on Wildcards and namespace searches are supported, so if you search for 'character:sam*' or 'series:*', this will be handled correctly clientside. -Many system predicates are also supported using a text parser! The parser was designed by a clever user for human input and allows for a certain amount of error (e.g. ~= instead of ≈, or "isn't" instead of "is not") or requires more information (e.g. the specific hashes for a hash lookup). Here's a big list of current formats supported: +**Many system predicates are also supported using a text parser!** The parser was designed by a clever user for human input and allows for a certain amount of error (e.g. ~= instead of ≈, or "isn't" instead of "is not") or requires more information (e.g. the specific hashes for a hash lookup). **Here's a big list of examples that are supported:** ??? example "System Predicates" * system:everything @@ -1323,8 +1323,11 @@ Many system predicates are also supported using a text parser! The parser was de * system:hash = abcdef01 abcdef02 md5 * system:modified date < 7 years 45 days 7h * system:modified date > 2011-06-04 + * system:last viewed time < 7 years 45 days 7h + * system:last view time < 7 years 45 days 7h * system:date modified > 7 years 2 months * system:date modified < 0 years 1 month 1 day 1 hour + * system:import time < 7 years 45 days 7h * system:time imported < 7 years 45 days 7h * system:time imported > 2011-06-04 * system:time imported > 7 years 2 months @@ -1370,7 +1373,7 @@ Many system predicates are also supported using a text parser! The parser was de * system:no note with name note name * system:does not have note with name note name -More system predicate types and input formats will be available in future. Please test out the system predicates you want to send. Reverse engineering system predicate data from text is obviously tricky. If a system predicate does not parse, you'll get 400. +Please test out the system predicates you want to send. If you are in _help->advanced mode_, you can test this parser in the advanced text input dialog when you click the OR\* button on a tag autocomplete dropdown. More system predicate types and input formats will be available in future. Reverse engineering system predicate data from text is obviously tricky. If a system predicate does not parse, you'll get 400. Also, OR predicates are now supported! Just nest within the tag list, and it'll be treated like an OR. For instance: diff --git a/docs/launch_arguments.md b/docs/launch_arguments.md index 5cb2a91f..1878f0d6 100644 --- a/docs/launch_arguments.md +++ b/docs/launch_arguments.md @@ -23,7 +23,7 @@ Which gives you a full listing of all below arguments, however this will not wor Lets you customise where hydrus should use for its base database directory. This is install_dir/db by default, but many advanced deployments will move this around, as described [here](database_migration.md). When an argument takes a complicated value like a path that could itself include whitespace, you should wrap it in quote marks, like this: ``` --d="E:\\my hydrus\\hydrus db" +-d="E:\my hydrus\hydrus db" ``` ##**`--temp_dir TEMP_DIR`** @@ -31,7 +31,7 @@ Lets you customise where hydrus should use for its base database directory. This This tells all aspects of the client, including the SQLite database, to use a different path for temp operations. This would be by default your system temp path, such as: ``` -C:\\Users\\You\\AppData\\Local\\Temp +C:\Users\You\AppData\Local\Temp ``` But you can also check it in _help->about_. A handful of database operations (PTR tag processing, vacuums) require a lot of free space, so if your system drive is very full, or you have unusual ramdisk-based temp storage limits, you may want to relocate to another location or drive. diff --git a/docs/old_changelog.html b/docs/old_changelog.html index f5b1b503..b6b5a77e 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -33,6 +33,53 @@

changelog