From f4825fadf4f35092083f7b3484df0bab58ffdc77 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Wed, 24 Aug 2022 16:37:20 +1000 Subject: [PATCH] [Files refactor] Bug fixes (#2849) * Fix scene sorting * Fix folder-based gallery path sorting * Fix gallery path filter * Fix stash-box performer submission * Fix identify logging * Remove govet from linter --- .golangci.yml | 3 +- internal/identify/identify.go | 4 +- pkg/scraper/stashbox/stash_box.go | 149 +++++++++++++++--------------- pkg/sqlite/filter.go | 6 +- pkg/sqlite/gallery.go | 77 ++++++++++++++- pkg/sqlite/scene.go | 5 +- pkg/sqlite/scene_test.go | 2 +- pkg/sqlite/sql.go | 4 +- 8 files changed, 159 insertions(+), 91 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 43f7324a0..d0644f564 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -9,7 +9,8 @@ linters: # Default set of linters from golangci-lint - errcheck - gosimple - - govet + # removed to fix timeout error + # - govet - ineffassign - staticcheck - typecheck diff --git a/internal/identify/identify.go b/internal/identify/identify.go index 98bcaa34e..1845479b8 100644 --- a/internal/identify/identify.go +++ b/internal/identify/identify.go @@ -212,7 +212,7 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage // don't update anything if nothing was set if updater.IsEmpty() { - logger.Debugf("Nothing to set for %s", s.Path) + logger.Debugf("Nothing to set for %s", s.Path()) return nil } @@ -225,7 +225,7 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage if title.Ptr() != nil { as = fmt.Sprintf(" as %s", title.Value) } - logger.Infof("Successfully identified %s%s using %s", s.Path, as, result.source.Name) + logger.Infof("Successfully identified %s%s using %s", s.Path(), as, result.source.Name) return nil }); err != nil { diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 7bcc6ef64..fc8c5e5bc 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -777,8 +777,9 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo return nil, err } for _, stashID := range stashIDs { - if stashID.Endpoint == endpoint { - studioDraft.ID = &stashID.StashID + c := stashID + if c.Endpoint == endpoint { + studioDraft.ID = &c.StashID break } } @@ -891,89 +892,83 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Performer, endpoint string) (*string, error) { draft := graphql.PerformerDraftInput{} var image io.Reader - if err := txn.WithTxn(ctx, c.txnManager, func(ctx context.Context) error { - pqb := c.repository.Performer - img, _ := pqb.GetImage(ctx, performer.ID) - if img != nil { - image = bytes.NewReader(img) - } + pqb := c.repository.Performer + img, _ := pqb.GetImage(ctx, performer.ID) + if img != nil { + image = bytes.NewReader(img) + } - if performer.Name.Valid { - draft.Name = performer.Name.String - } - if performer.Birthdate.Valid { - draft.Birthdate = &performer.Birthdate.String - } - if performer.Country.Valid { - draft.Country = &performer.Country.String - } - if performer.Ethnicity.Valid { - draft.Ethnicity = &performer.Ethnicity.String - } - if performer.EyeColor.Valid { - draft.EyeColor = &performer.EyeColor.String - } - if performer.FakeTits.Valid { - draft.BreastType = &performer.FakeTits.String - } - if performer.Gender.Valid { - draft.Gender = &performer.Gender.String - } - if performer.HairColor.Valid { - draft.HairColor = &performer.HairColor.String - } - if performer.Height.Valid { - draft.Height = &performer.Height.String - } - if performer.Measurements.Valid { - draft.Measurements = &performer.Measurements.String - } - if performer.Piercings.Valid { - draft.Piercings = &performer.Piercings.String - } - if performer.Tattoos.Valid { - draft.Tattoos = &performer.Tattoos.String - } - if performer.Aliases.Valid { - draft.Aliases = &performer.Aliases.String - } + if performer.Name.Valid { + draft.Name = performer.Name.String + } + if performer.Birthdate.Valid { + draft.Birthdate = &performer.Birthdate.String + } + if performer.Country.Valid { + draft.Country = &performer.Country.String + } + if performer.Ethnicity.Valid { + draft.Ethnicity = &performer.Ethnicity.String + } + if performer.EyeColor.Valid { + draft.EyeColor = &performer.EyeColor.String + } + if performer.FakeTits.Valid { + draft.BreastType = &performer.FakeTits.String + } + if performer.Gender.Valid { + draft.Gender = &performer.Gender.String + } + if performer.HairColor.Valid { + draft.HairColor = &performer.HairColor.String + } + if performer.Height.Valid { + draft.Height = &performer.Height.String + } + if performer.Measurements.Valid { + draft.Measurements = &performer.Measurements.String + } + if performer.Piercings.Valid { + draft.Piercings = &performer.Piercings.String + } + if performer.Tattoos.Valid { + draft.Tattoos = &performer.Tattoos.String + } + if performer.Aliases.Valid { + draft.Aliases = &performer.Aliases.String + } - var urls []string - if len(strings.TrimSpace(performer.Twitter.String)) > 0 { - urls = append(urls, "https://twitter.com/"+strings.TrimSpace(performer.Twitter.String)) - } - if len(strings.TrimSpace(performer.Instagram.String)) > 0 { - urls = append(urls, "https://instagram.com/"+strings.TrimSpace(performer.Instagram.String)) - } - if len(strings.TrimSpace(performer.URL.String)) > 0 { - urls = append(urls, strings.TrimSpace(performer.URL.String)) - } - if len(urls) > 0 { - draft.Urls = urls - } + var urls []string + if len(strings.TrimSpace(performer.Twitter.String)) > 0 { + urls = append(urls, "https://twitter.com/"+strings.TrimSpace(performer.Twitter.String)) + } + if len(strings.TrimSpace(performer.Instagram.String)) > 0 { + urls = append(urls, "https://instagram.com/"+strings.TrimSpace(performer.Instagram.String)) + } + if len(strings.TrimSpace(performer.URL.String)) > 0 { + urls = append(urls, strings.TrimSpace(performer.URL.String)) + } + if len(urls) > 0 { + draft.Urls = urls + } - stashIDs, err := pqb.GetStashIDs(ctx, performer.ID) - if err != nil { - return err - } - var stashID *string - for _, v := range stashIDs { - c := v - if v.Endpoint == endpoint { - stashID = &c.StashID - break - } - } - draft.ID = stashID - - return nil - }); err != nil { + stashIDs, err := pqb.GetStashIDs(ctx, performer.ID) + if err != nil { return nil, err } + var stashID *string + for _, v := range stashIDs { + c := v + if v.Endpoint == endpoint { + stashID = &c.StashID + break + } + } + draft.ID = stashID var id *string var ret graphql.SubmitPerformerDraft - err := c.submitDraft(ctx, graphql.SubmitPerformerDraftDocument, draft, image, &ret) + err = c.submitDraft(ctx, graphql.SubmitPerformerDraftDocument, draft, image, &ret) id = ret.SubmitPerformerDraft.ID return id, err diff --git a/pkg/sqlite/filter.go b/pkg/sqlite/filter.go index ca980102a..8204adb95 100644 --- a/pkg/sqlite/filter.go +++ b/pkg/sqlite/filter.go @@ -387,11 +387,9 @@ func stringCriterionHandler(c *models.StringCriterionInput, column string) crite if modifier := c.Modifier; c.Modifier.IsValid() { switch modifier { case models.CriterionModifierIncludes: - clause, thisArgs := getSearchBinding([]string{column}, c.Value, false) - f.addWhere(clause, thisArgs...) + f.whereClauses = append(f.whereClauses, getStringSearchClause([]string{column}, c.Value, false)) case models.CriterionModifierExcludes: - clause, thisArgs := getSearchBinding([]string{column}, c.Value, true) - f.addWhere(clause, thisArgs...) + f.whereClauses = append(f.whereClauses, getStringSearchClause([]string{column}, c.Value, true)) case models.CriterionModifierEquals: f.addWhere(column+" LIKE ?", c.Value) case models.CriterionModifierNotEquals: diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 3a4b05d23..83b3d25bc 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "regexp" "time" "github.com/doug-martin/goqu/v9" @@ -595,7 +596,7 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga } })) - query.handleCriterion(ctx, pathCriterionHandler(galleryFilter.Path, "folders.path", "files.basename", qb.addFoldersTable)) + query.handleCriterion(ctx, qb.galleryPathCriterionHandler(galleryFilter.Path)) query.handleCriterion(ctx, galleryFileCountCriterionHandler(qb, galleryFilter.FileCount)) query.handleCriterion(ctx, intCriterionHandler(galleryFilter.Rating, "galleries.rating", nil)) query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.URL, "galleries.url")) @@ -716,6 +717,71 @@ func (qb *GalleryStore) QueryCount(ctx context.Context, galleryFilter *models.Ga return query.executeCount(ctx) } +func (qb *GalleryStore) galleryPathCriterionHandler(c *models.StringCriterionInput) criterionHandlerFunc { + return func(ctx context.Context, f *filterBuilder) { + if c != nil { + qb.addFoldersTable(f) + f.addLeftJoin(folderTable, "gallery_folder", "galleries.folder_id = gallery_folder.id") + + const pathColumn = "folders.path" + const basenameColumn = "files.basename" + const folderPathColumn = "gallery_folder.path" + + addWildcards := true + not := false + + if modifier := c.Modifier; c.Modifier.IsValid() { + switch modifier { + case models.CriterionModifierIncludes: + clause := getPathSearchClause(pathColumn, basenameColumn, c.Value, addWildcards, not) + clause2 := getStringSearchClause([]string{folderPathColumn}, c.Value, false) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + case models.CriterionModifierExcludes: + not = true + clause := getPathSearchClause(pathColumn, basenameColumn, c.Value, addWildcards, not) + clause2 := getStringSearchClause([]string{folderPathColumn}, c.Value, true) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + case models.CriterionModifierEquals: + addWildcards = false + clause := getPathSearchClause(pathColumn, basenameColumn, c.Value, addWildcards, not) + clause2 := makeClause(folderPathColumn+" LIKE ?", c.Value) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + case models.CriterionModifierNotEquals: + addWildcards = false + not = true + clause := getPathSearchClause(pathColumn, basenameColumn, c.Value, addWildcards, not) + clause2 := makeClause(folderPathColumn+" NOT LIKE ?", c.Value) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + case models.CriterionModifierMatchesRegex: + if _, err := regexp.Compile(c.Value); err != nil { + f.setError(err) + return + } + clause := makeClause(fmt.Sprintf("(%s IS NOT NULL AND %[1]s regexp ?) OR (%s IS NOT NULL AND %[2]s regexp ?)", pathColumn, basenameColumn), c.Value, c.Value) + clause2 := makeClause(fmt.Sprintf("(%s IS NOT NULL AND %[1]s regexp ?)", folderPathColumn), c.Value) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + case models.CriterionModifierNotMatchesRegex: + if _, err := regexp.Compile(c.Value); err != nil { + f.setError(err) + return + } + f.addWhere(fmt.Sprintf("(%s IS NULL OR %[1]s NOT regexp ?) AND (%s IS NULL OR %[2]s NOT regexp ?)", pathColumn, basenameColumn), c.Value, c.Value) + f.addWhere(fmt.Sprintf("(%s IS NULL OR %[1]s NOT regexp ?)", folderPathColumn), c.Value) + case models.CriterionModifierIsNull: + f.whereClauses = append(f.whereClauses, makeClause(fmt.Sprintf("(%s IS NULL OR TRIM(%[1]s) = '' OR %s IS NULL OR TRIM(%[2]s) = '')", pathColumn, basenameColumn))) + f.whereClauses = append(f.whereClauses, makeClause(fmt.Sprintf("(%s IS NULL OR TRIM(%[1]s) = '')", folderPathColumn))) + case models.CriterionModifierNotNull: + clause := makeClause(fmt.Sprintf("(%s IS NOT NULL AND TRIM(%[1]s) != '' AND %s IS NOT NULL AND TRIM(%[2]s) != '')", pathColumn, basenameColumn)) + clause2 := makeClause(fmt.Sprintf("(%s IS NOT NULL AND TRIM(%[1]s) != '')", folderPathColumn)) + f.whereClauses = append(f.whereClauses, orClauses(clause, clause2)) + default: + panic("unsupported string filter modifier") + } + } + } + } +} + func galleryFileCountCriterionHandler(qb *GalleryStore, fileCount *models.IntCriterionInput) criterionHandlerFunc { h := countCriterionHandlerBuilder{ primaryTable: galleryTable, @@ -962,10 +1028,15 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F query.addJoins( join{ table: folderTable, - onClause: "files.parent_folder_id = folders.id", + onClause: "folders.id = galleries.folder_id", + }, + join{ + table: folderTable, + as: "file_folder", + onClause: "files.parent_folder_id = file_folder.id", }, ) - query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction) + query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, file_folder.path %[1]s, files.basename %[1]s", direction) case "file_mod_time": sort = "mod_time" addFileTable() diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 985f6671c..c6f367485 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1325,12 +1325,15 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF sort = "frame_rate" addVideoFileTable() query.sortAndPagination += getSort(sort, direction, videoFileTable) - case "size": + case "filesize": addFileTable() query.sortAndPagination += getSort(sort, direction, fileTable) case "duration": addVideoFileTable() query.sortAndPagination += getSort(sort, direction, videoFileTable) + case "interactive", "interactive_speed": + addVideoFileTable() + query.sortAndPagination += getSort(sort, direction, videoFileTable) default: query.sortAndPagination += getSort(sort, direction, "scenes") } diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index e32dd78ea..d74d73f14 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -3477,7 +3477,7 @@ func TestSceneQuerySorting(t *testing.T) { }, { "file size", - "size", + "filesize", models.SortDirectionEnumDesc, -1, -1, diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index f23402986..44920903e 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -105,7 +105,7 @@ func getCountSort(primaryTable, joinTable, primaryFK, direction string) string { return fmt.Sprintf(" ORDER BY (SELECT COUNT(*) FROM %s WHERE %s = %s.id) %s", joinTable, primaryFK, primaryTable, getSortDirection(direction)) } -func getSearchBinding(columns []string, q string, not bool) (string, []interface{}) { +func getStringSearchClause(columns []string, q string, not bool) sqlClause { var likeClauses []string var args []interface{} @@ -137,7 +137,7 @@ func getSearchBinding(columns []string, q string, not bool) (string, []interface } likes := strings.Join(likeClauses, binaryType) - return "(" + likes + ")", args + return makeClause("("+likes+")", args...) } func getInBinding(length int) string {