Rooba
/
stash
mirror of https://github.com/stashapp/stash.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Selective clean (#2125) 

* Add backend support for selective clean
* Add selective clean button and dialog
This commit is contained in:
WithoutPants 2021-12-16 13:28:44 +11:00 committed by GitHub
parent d94e4f9a5b
commit d25510fdd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 283 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
graphql/schema/types/metadata.graphql
View File

@ -106,6 +106,8 @@ type ScanMetadataOptions {
}
input CleanMetadataInput {
paths: [String!]
"""Do a dry run. Don't delete any files"""
dryRun: Boolean!
}

40
pkg/gallery/filter.go Normal file
View File

@ -0,0 +1,40 @@
package gallery
import (
"path/filepath"
"strings"
"github.com/stashapp/stash/pkg/models"
)
func PathsFilter(paths []string) *models.GalleryFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.GalleryFilterType
var or *models.GalleryFilterType
for _, p := range paths {
newOr := &models.GalleryFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}

40
pkg/image/filter.go Normal file
View File

@ -0,0 +1,40 @@
package image
import (
"path/filepath"
"strings"
"github.com/stashapp/stash/pkg/models"
)
func PathsFilter(paths []string) *models.ImageFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.ImageFilterType
var or *models.ImageFilterType
for _, p := range paths {
newOr := &models.ImageFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}

24
pkg/manager/task_clean.go
View File

@ -6,6 +6,7 @@ import (
"path/filepath"
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/job"
"github.com/stashapp/stash/pkg/logger"
@ -66,28 +67,35 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) {
}
func (j *cleanJob) getCount(r models.ReaderRepository) (int, error) {
sceneCount, err := r.Scene().Count()
sceneFilter := scene.PathsFilter(j.input.Paths)
sceneResult, err := r.Scene().Query(models.SceneQueryOptions{
QueryOptions: models.QueryOptions{
Count: true,
},
SceneFilter: sceneFilter,
})
if err != nil {
return 0, err
}
imageCount, err := r.Image().Count()
imageCount, err := r.Image().QueryCount(image.PathsFilter(j.input.Paths), nil)
if err != nil {
return 0, err
}
galleryCount, err := r.Gallery().Count()
galleryCount, err := r.Gallery().QueryCount(gallery.PathsFilter(j.input.Paths), nil)
if err != nil {
return 0, err
}
return sceneCount + imageCount + galleryCount, nil
return sceneResult.Count + imageCount + galleryCount, nil
}
func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb models.SceneReader) error {
batchSize := 1000
findFilter := models.BatchFindFilter(batchSize)
sceneFilter := scene.PathsFilter(j.input.Paths)
sort := "path"
findFilter.Sort = &sort
@ -99,7 +107,7 @@ func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb
return nil
}
scenes, err := scene.Query(qb, nil, findFilter)
scenes, err := scene.Query(qb, sceneFilter, findFilter)
if err != nil {
return fmt.Errorf("error querying for scenes: %w", err)
}
@ -150,6 +158,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress,
batchSize := 1000
findFilter := models.BatchFindFilter(batchSize)
galleryFilter := gallery.PathsFilter(j.input.Paths)
sort := "path"
findFilter.Sort = &sort
@ -161,7 +170,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress,
return nil
}
galleries, _, err := qb.Query(nil, findFilter)
galleries, _, err := qb.Query(galleryFilter, findFilter)
if err != nil {
return fmt.Errorf("error querying for galleries: %w", err)
}
@ -210,6 +219,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb
batchSize := 1000
findFilter := models.BatchFindFilter(batchSize)
imageFilter := image.PathsFilter(j.input.Paths)
// performance consideration: order by path since default ordering by
// title is slow
@ -224,7 +234,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb
return nil
}
images, err := image.Query(qb, nil, findFilter)
images, err := image.Query(qb, imageFilter, findFilter)
if err != nil {
return fmt.Errorf("error querying for images: %w", err)
}

103
pkg/match/path.go
View File

@ -6,6 +6,7 @@ import (
"regexp"
"strings"
"github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
@ -175,38 +176,6 @@ func PathToTags(path string, tagReader models.TagReader) ([]*models.Tag, error)
return ret, nil
}
func scenePathsFilter(paths []string) *models.SceneFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.SceneFilterType
var or *models.SceneFilterType
for _, p := range paths {
newOr := &models.SceneFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}
func PathToScenes(name string, paths []string, sceneReader models.SceneReader) ([]*models.Scene, error) {
regex := getPathQueryRegex(name)
organized := false
@ -218,7 +187,7 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) (
Organized: &organized,
}
filter.And = scenePathsFilter(paths)
filter.And = scene.PathsFilter(paths)
pp := models.PerPageAll
scenes, err := scene.Query(sceneReader, &filter, &models.FindFilterType{
@ -239,38 +208,6 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) (
return ret, nil
}
func imagePathsFilter(paths []string) *models.ImageFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.ImageFilterType
var or *models.ImageFilterType
for _, p := range paths {
newOr := &models.ImageFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}
func PathToImages(name string, paths []string, imageReader models.ImageReader) ([]*models.Image, error) {
regex := getPathQueryRegex(name)
organized := false
@ -282,7 +219,7 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) (
Organized: &organized,
}
filter.And = imagePathsFilter(paths)
filter.And = image.PathsFilter(paths)
pp := models.PerPageAll
images, err := image.Query(imageReader, &filter, &models.FindFilterType{
@ -303,38 +240,6 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) (
return ret, nil
}
func galleryPathsFilter(paths []string) *models.GalleryFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.GalleryFilterType
var or *models.GalleryFilterType
for _, p := range paths {
newOr := &models.GalleryFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}
func PathToGalleries(name string, paths []string, galleryReader models.GalleryReader) ([]*models.Gallery, error) {
regex := getPathQueryRegex(name)
organized := false
@ -346,7 +251,7 @@ func PathToGalleries(name string, paths []string, galleryReader models.GalleryRe
Organized: &organized,
}
filter.And = galleryPathsFilter(paths)
filter.And = gallery.PathsFilter(paths)
pp := models.PerPageAll
gallerys, _, err := galleryReader.Query(&filter, &models.FindFilterType{

40
pkg/scene/filter.go Normal file
View File

@ -0,0 +1,40 @@
package scene
import (
"path/filepath"
"strings"
"github.com/stashapp/stash/pkg/models"
)
func PathsFilter(paths []string) *models.SceneFilterType {
if paths == nil {
return nil
}
sep := string(filepath.Separator)
var ret *models.SceneFilterType
var or *models.SceneFilterType
for _, p := range paths {
newOr := &models.SceneFilterType{}
if or != nil {
or.Or = newOr
} else {
ret = newOr
}
or = newOr
if !strings.HasSuffix(p, sep) {
p += sep
}
or.Path = &models.StringCriterionInput{
Modifier: models.CriterionModifierEquals,
Value: p + "%",
}
}
return ret
}

3
ui/v2.5/src/components/Changelog/versions/v0120.md
View File

@ -1,8 +1,9 @@
### ✨ New Features
* Added selective clean task. ([#2125](https://github.com/stashapp/stash/pull/2125))
* Show heatmaps and median stroke speed for interactive scenes on the scenes page. ([#2096](https://github.com/stashapp/stash/pull/2096))
* Save task options when scanning, generating and auto-tagging. ([#1949](https://github.com/stashapp/stash/pull/1949), [#2061](https://github.com/stashapp/stash/pull/2061))
* Changed query string parsing behaviour to require all words by default, with the option to `or` keywords and exclude keywords. See the `Browsing` section of the manual for details. ([#1982](https://github.com/stashapp/stash/pull/1982))
* Add forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973))
* Added forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973))
### 🎨 Improvements
* Added keyboard shortcuts to hide scene page sidebar and scene scrubber. ([#2099](https://github.com/stashapp/stash/pull/2099))

172
ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx
View File

@ -1,6 +1,6 @@
import React, { useState } from "react";
import { FormattedMessage, useIntl } from "react-intl";
import { Button, Form } from "react-bootstrap";
import { Button, Col, Form, Row } from "react-bootstrap";
import {
mutateMigrateHashNaming,
mutateMetadataExport,
@ -17,6 +17,104 @@ import { SettingSection } from "../SettingSection";
import { BooleanSetting, Setting } from "../Inputs";
import { ManualLink } from "src/components/Help/Manual";
import { Icon } from "src/components/Shared";
import { ConfigurationContext } from "src/hooks/Config";
import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect";
interface ICleanDialog {
pathSelection?: boolean;
dryRun: boolean;
onClose: (paths?: string[]) => void;
}
const CleanDialog: React.FC<ICleanDialog> = ({
pathSelection = false,
dryRun,
onClose,
}) => {
const intl = useIntl();
const { configuration } = React.useContext(ConfigurationContext);
const libraryPaths = configuration?.general.stashes.map((s) => s.path);
const [paths, setPaths] = useState<string[]>([]);
const [currentDirectory, setCurrentDirectory] = useState<string>("");
function removePath(p: string) {
setPaths(paths.filter((path) => path !== p));
}
function addPath(p: string) {
if (p && !paths.includes(p)) {
setPaths(paths.concat(p));
}
}
let msg;
if (dryRun) {
msg = (
<p>{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}</p>
);
} else {
msg = (
<p>{intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}</p>
);
}
return (
<Modal
show
icon="trash-alt"
disabled={pathSelection && paths.length === 0}
accept={{
text: intl.formatMessage({ id: "actions.clean" }),
variant: "danger",
onClick: () => onClose(paths),
}}
cancel={{ onClick: () => onClose() }}
>
<div className="dialog-container">
<div className="mb-3">
{paths.map((p) => (
<Row className="align-items-center mb-1" key={p}>
<Form.Label column xs={10}>
{p}
</Form.Label>
<Col xs={2} className="d-flex justify-content-end">
<Button
className="ml-auto"
size="sm"
variant="danger"
title={intl.formatMessage({ id: "actions.delete" })}
onClick={() => removePath(p)}
>
<Icon icon="minus" />
</Button>
</Col>
</Row>
))}
{pathSelection ? (
<FolderSelect
currentDirectory={currentDirectory}
setCurrentDirectory={(v) => setCurrentDirectory(v)}
defaultDirectories={libraryPaths}
appendButton={
<Button
variant="secondary"
onClick={() => addPath(currentDirectory)}
>
<Icon icon="plus" />
</Button>
}
/>
) : undefined}
</div>
{msg}
</div>
</Modal>
);
};
interface ICleanOptions {
options: GQL.CleanMetadataInput;
@ -56,6 +154,7 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
importAlert: false,
import: false,
clean: false,
cleanAlert: false,
});
const [cleanOptions, setCleanOptions] = useState<GQL.CleanMetadataInput>({
@ -110,39 +209,12 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
return <ImportDialog onClose={() => setDialogOpen({ import: false })} />;
}
function renderCleanDialog() {
let msg;
if (cleanOptions.dryRun) {
msg = (
<p>{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}</p>
);
} else {
msg = (
<p>
{intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}
</p>
);
}
return (
<Modal
show={dialogOpen.clean}
icon="trash-alt"
accept={{
text: intl.formatMessage({ id: "actions.clean" }),
variant: "danger",
onClick: onClean,
}}
cancel={{ onClick: () => setDialogOpen({ clean: false }) }}
>
{msg}
</Modal>
);
}
async function onClean() {
async function onClean(paths?: string[]) {
try {
await mutateMetadataClean(cleanOptions);
await mutateMetadataClean({
...cleanOptions,
paths,
});
Toast.success({
content: intl.formatMessage(
@ -212,7 +284,30 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
<Form.Group>
{renderImportAlert()}
{renderImportDialog()}
{renderCleanDialog()}
{dialogOpen.cleanAlert || dialogOpen.clean ? (
<CleanDialog
dryRun={cleanOptions.dryRun}
pathSelection={dialogOpen.clean}
onClose={(p) => {
// undefined means cancelled
if (p !== undefined) {
if (dialogOpen.cleanAlert) {
// don't provide paths
onClean();
} else {
onClean(p);
}
}
setDialogOpen({
clean: false,
cleanAlert: false,
});
}}
/>
) : (
dialogOpen.clean
)}
<SettingSection headingID="config.tasks.maintenance">
<div className="setting-group">
@ -230,10 +325,17 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
<Button
variant="danger"
type="submit"
onClick={() => setDialogOpen({ clean: true })}
onClick={() => setDialogOpen({ cleanAlert: true })}
>
<FormattedMessage id="actions.clean" />…
</Button>
<Button
variant="danger"
type="submit"
onClick={() => setDialogOpen({ clean: true })}
>
<FormattedMessage id="actions.selective_clean" />…
</Button>
</Setting>
<CleanOptions
options={cleanOptions}

1
ui/v2.5/src/locales/en-GB.json
View File

@ -79,6 +79,7 @@
"select_folders": "Select folders",
"select_none": "Select None",
"selective_auto_tag": "Selective Auto Tag",
"selective_clean": "Selective Clean",
"selective_scan": "Selective Scan",
"set_as_default": "Set as default",
"set_back_image": "Back image…",