From d25510fdd761c2618289fa406991799a6e29455e Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Thu, 16 Dec 2021 13:28:44 +1100 Subject: [PATCH] Selective clean (#2125) * Add backend support for selective clean * Add selective clean button and dialog --- graphql/schema/types/metadata.graphql | 2 + pkg/gallery/filter.go | 40 ++++ pkg/image/filter.go | 40 ++++ pkg/manager/task_clean.go | 24 ++- pkg/match/path.go | 103 +---------- pkg/scene/filter.go | 40 ++++ .../components/Changelog/versions/v0120.md | 3 +- .../Settings/Tasks/DataManagementTasks.tsx | 172 ++++++++++++++---- ui/v2.5/src/locales/en-GB.json | 1 + 9 files changed, 283 insertions(+), 142 deletions(-) create mode 100644 pkg/gallery/filter.go create mode 100644 pkg/image/filter.go create mode 100644 pkg/scene/filter.go diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index df2b306b8..faf4d63bc 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -106,6 +106,8 @@ type ScanMetadataOptions { } input CleanMetadataInput { + paths: [String!] + """Do a dry run. Don't delete any files""" dryRun: Boolean! } diff --git a/pkg/gallery/filter.go b/pkg/gallery/filter.go new file mode 100644 index 000000000..ce4bd71d0 --- /dev/null +++ b/pkg/gallery/filter.go @@ -0,0 +1,40 @@ +package gallery + +import ( + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/models" +) + +func PathsFilter(paths []string) *models.GalleryFilterType { + if paths == nil { + return nil + } + + sep := string(filepath.Separator) + + var ret *models.GalleryFilterType + var or *models.GalleryFilterType + for _, p := range paths { + newOr := &models.GalleryFilterType{} + if or != nil { + or.Or = newOr + } else { + ret = newOr + } + + or = newOr + + if !strings.HasSuffix(p, sep) { + p += sep + } + + or.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierEquals, + Value: p + "%", + } + } + + return ret +} diff --git a/pkg/image/filter.go b/pkg/image/filter.go new file mode 100644 index 000000000..c36a156af --- /dev/null +++ b/pkg/image/filter.go @@ -0,0 +1,40 @@ +package image + +import ( + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/models" +) + +func PathsFilter(paths []string) *models.ImageFilterType { + if paths == nil { + return nil + } + + sep := string(filepath.Separator) + + var ret *models.ImageFilterType + var or *models.ImageFilterType + for _, p := range paths { + newOr := &models.ImageFilterType{} + if or != nil { + or.Or = newOr + } else { + ret = newOr + } + + or = newOr + + if !strings.HasSuffix(p, sep) { + p += sep + } + + or.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierEquals, + Value: p + "%", + } + } + + return ret +} diff --git a/pkg/manager/task_clean.go b/pkg/manager/task_clean.go index 45d6e4590..213256541 100644 --- a/pkg/manager/task_clean.go +++ b/pkg/manager/task_clean.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" @@ -66,28 +67,35 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) { } func (j *cleanJob) getCount(r models.ReaderRepository) (int, error) { - sceneCount, err := r.Scene().Count() + sceneFilter := scene.PathsFilter(j.input.Paths) + sceneResult, err := r.Scene().Query(models.SceneQueryOptions{ + QueryOptions: models.QueryOptions{ + Count: true, + }, + SceneFilter: sceneFilter, + }) if err != nil { return 0, err } - imageCount, err := r.Image().Count() + imageCount, err := r.Image().QueryCount(image.PathsFilter(j.input.Paths), nil) if err != nil { return 0, err } - galleryCount, err := r.Gallery().Count() + galleryCount, err := r.Gallery().QueryCount(gallery.PathsFilter(j.input.Paths), nil) if err != nil { return 0, err } - return sceneCount + imageCount + galleryCount, nil + return sceneResult.Count + imageCount + galleryCount, nil } func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb models.SceneReader) error { batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) + sceneFilter := scene.PathsFilter(j.input.Paths) sort := "path" findFilter.Sort = &sort @@ -99,7 +107,7 @@ func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb return nil } - scenes, err := scene.Query(qb, nil, findFilter) + scenes, err := scene.Query(qb, sceneFilter, findFilter) if err != nil { return fmt.Errorf("error querying for scenes: %w", err) } @@ -150,6 +158,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress, batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) + galleryFilter := gallery.PathsFilter(j.input.Paths) sort := "path" findFilter.Sort = &sort @@ -161,7 +170,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress, return nil } - galleries, _, err := qb.Query(nil, findFilter) + galleries, _, err := qb.Query(galleryFilter, findFilter) if err != nil { return fmt.Errorf("error querying for galleries: %w", err) } @@ -210,6 +219,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb batchSize := 1000 findFilter := models.BatchFindFilter(batchSize) + imageFilter := image.PathsFilter(j.input.Paths) // performance consideration: order by path since default ordering by // title is slow @@ -224,7 +234,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb return nil } - images, err := image.Query(qb, nil, findFilter) + images, err := image.Query(qb, imageFilter, findFilter) if err != nil { return fmt.Errorf("error querying for images: %w", err) } diff --git a/pkg/match/path.go b/pkg/match/path.go index 2de78e716..9dd4bdec0 100644 --- a/pkg/match/path.go +++ b/pkg/match/path.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" @@ -175,38 +176,6 @@ func PathToTags(path string, tagReader models.TagReader) ([]*models.Tag, error) return ret, nil } -func scenePathsFilter(paths []string) *models.SceneFilterType { - if paths == nil { - return nil - } - - sep := string(filepath.Separator) - - var ret *models.SceneFilterType - var or *models.SceneFilterType - for _, p := range paths { - newOr := &models.SceneFilterType{} - if or != nil { - or.Or = newOr - } else { - ret = newOr - } - - or = newOr - - if !strings.HasSuffix(p, sep) { - p += sep - } - - or.Path = &models.StringCriterionInput{ - Modifier: models.CriterionModifierEquals, - Value: p + "%", - } - } - - return ret -} - func PathToScenes(name string, paths []string, sceneReader models.SceneReader) ([]*models.Scene, error) { regex := getPathQueryRegex(name) organized := false @@ -218,7 +187,7 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) ( Organized: &organized, } - filter.And = scenePathsFilter(paths) + filter.And = scene.PathsFilter(paths) pp := models.PerPageAll scenes, err := scene.Query(sceneReader, &filter, &models.FindFilterType{ @@ -239,38 +208,6 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) ( return ret, nil } -func imagePathsFilter(paths []string) *models.ImageFilterType { - if paths == nil { - return nil - } - - sep := string(filepath.Separator) - - var ret *models.ImageFilterType - var or *models.ImageFilterType - for _, p := range paths { - newOr := &models.ImageFilterType{} - if or != nil { - or.Or = newOr - } else { - ret = newOr - } - - or = newOr - - if !strings.HasSuffix(p, sep) { - p += sep - } - - or.Path = &models.StringCriterionInput{ - Modifier: models.CriterionModifierEquals, - Value: p + "%", - } - } - - return ret -} - func PathToImages(name string, paths []string, imageReader models.ImageReader) ([]*models.Image, error) { regex := getPathQueryRegex(name) organized := false @@ -282,7 +219,7 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) ( Organized: &organized, } - filter.And = imagePathsFilter(paths) + filter.And = image.PathsFilter(paths) pp := models.PerPageAll images, err := image.Query(imageReader, &filter, &models.FindFilterType{ @@ -303,38 +240,6 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) ( return ret, nil } -func galleryPathsFilter(paths []string) *models.GalleryFilterType { - if paths == nil { - return nil - } - - sep := string(filepath.Separator) - - var ret *models.GalleryFilterType - var or *models.GalleryFilterType - for _, p := range paths { - newOr := &models.GalleryFilterType{} - if or != nil { - or.Or = newOr - } else { - ret = newOr - } - - or = newOr - - if !strings.HasSuffix(p, sep) { - p += sep - } - - or.Path = &models.StringCriterionInput{ - Modifier: models.CriterionModifierEquals, - Value: p + "%", - } - } - - return ret -} - func PathToGalleries(name string, paths []string, galleryReader models.GalleryReader) ([]*models.Gallery, error) { regex := getPathQueryRegex(name) organized := false @@ -346,7 +251,7 @@ func PathToGalleries(name string, paths []string, galleryReader models.GalleryRe Organized: &organized, } - filter.And = galleryPathsFilter(paths) + filter.And = gallery.PathsFilter(paths) pp := models.PerPageAll gallerys, _, err := galleryReader.Query(&filter, &models.FindFilterType{ diff --git a/pkg/scene/filter.go b/pkg/scene/filter.go new file mode 100644 index 000000000..0cc875b27 --- /dev/null +++ b/pkg/scene/filter.go @@ -0,0 +1,40 @@ +package scene + +import ( + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/models" +) + +func PathsFilter(paths []string) *models.SceneFilterType { + if paths == nil { + return nil + } + + sep := string(filepath.Separator) + + var ret *models.SceneFilterType + var or *models.SceneFilterType + for _, p := range paths { + newOr := &models.SceneFilterType{} + if or != nil { + or.Or = newOr + } else { + ret = newOr + } + + or = newOr + + if !strings.HasSuffix(p, sep) { + p += sep + } + + or.Path = &models.StringCriterionInput{ + Modifier: models.CriterionModifierEquals, + Value: p + "%", + } + } + + return ret +} diff --git a/ui/v2.5/src/components/Changelog/versions/v0120.md b/ui/v2.5/src/components/Changelog/versions/v0120.md index f9f4348cc..35cdb9c2a 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0120.md +++ b/ui/v2.5/src/components/Changelog/versions/v0120.md @@ -1,8 +1,9 @@ ### ✨ New Features +* Added selective clean task. ([#2125](https://github.com/stashapp/stash/pull/2125)) * Show heatmaps and median stroke speed for interactive scenes on the scenes page. ([#2096](https://github.com/stashapp/stash/pull/2096)) * Save task options when scanning, generating and auto-tagging. ([#1949](https://github.com/stashapp/stash/pull/1949), [#2061](https://github.com/stashapp/stash/pull/2061)) * Changed query string parsing behaviour to require all words by default, with the option to `or` keywords and exclude keywords. See the `Browsing` section of the manual for details. ([#1982](https://github.com/stashapp/stash/pull/1982)) -* Add forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973)) +* Added forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973)) ### 🎨 Improvements * Added keyboard shortcuts to hide scene page sidebar and scene scrubber. ([#2099](https://github.com/stashapp/stash/pull/2099)) diff --git a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx index 2aa9908bd..684aec6d3 100644 --- a/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx @@ -1,6 +1,6 @@ import React, { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { Button, Form } from "react-bootstrap"; +import { Button, Col, Form, Row } from "react-bootstrap"; import { mutateMigrateHashNaming, mutateMetadataExport, @@ -17,6 +17,104 @@ import { SettingSection } from "../SettingSection"; import { BooleanSetting, Setting } from "../Inputs"; import { ManualLink } from "src/components/Help/Manual"; import { Icon } from "src/components/Shared"; +import { ConfigurationContext } from "src/hooks/Config"; +import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect"; + +interface ICleanDialog { + pathSelection?: boolean; + dryRun: boolean; + onClose: (paths?: string[]) => void; +} + +const CleanDialog: React.FC = ({ + pathSelection = false, + dryRun, + onClose, +}) => { + const intl = useIntl(); + const { configuration } = React.useContext(ConfigurationContext); + + const libraryPaths = configuration?.general.stashes.map((s) => s.path); + + const [paths, setPaths] = useState([]); + const [currentDirectory, setCurrentDirectory] = useState(""); + + function removePath(p: string) { + setPaths(paths.filter((path) => path !== p)); + } + + function addPath(p: string) { + if (p && !paths.includes(p)) { + setPaths(paths.concat(p)); + } + } + + let msg; + if (dryRun) { + msg = ( +

{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}

+ ); + } else { + msg = ( +

{intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}

+ ); + } + + return ( + onClose(paths), + }} + cancel={{ onClick: () => onClose() }} + > +
+
+ {paths.map((p) => ( + + + {p} + + + + + + ))} + + {pathSelection ? ( + setCurrentDirectory(v)} + defaultDirectories={libraryPaths} + appendButton={ + + } + /> + ) : undefined} +
+ + {msg} +
+ + ); +}; interface ICleanOptions { options: GQL.CleanMetadataInput; @@ -56,6 +154,7 @@ export const DataManagementTasks: React.FC = ({ importAlert: false, import: false, clean: false, + cleanAlert: false, }); const [cleanOptions, setCleanOptions] = useState({ @@ -110,39 +209,12 @@ export const DataManagementTasks: React.FC = ({ return setDialogOpen({ import: false })} />; } - function renderCleanDialog() { - let msg; - if (cleanOptions.dryRun) { - msg = ( -

{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}

- ); - } else { - msg = ( -

- {intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })} -

- ); - } - - return ( - setDialogOpen({ clean: false }) }} - > - {msg} - - ); - } - - async function onClean() { + async function onClean(paths?: string[]) { try { - await mutateMetadataClean(cleanOptions); + await mutateMetadataClean({ + ...cleanOptions, + paths, + }); Toast.success({ content: intl.formatMessage( @@ -212,7 +284,30 @@ export const DataManagementTasks: React.FC = ({ {renderImportAlert()} {renderImportDialog()} - {renderCleanDialog()} + {dialogOpen.cleanAlert || dialogOpen.clean ? ( + { + // undefined means cancelled + if (p !== undefined) { + if (dialogOpen.cleanAlert) { + // don't provide paths + onClean(); + } else { + onClean(p); + } + } + + setDialogOpen({ + clean: false, + cleanAlert: false, + }); + }} + /> + ) : ( + dialogOpen.clean + )}
@@ -230,10 +325,17 @@ export const DataManagementTasks: React.FC = ({ +