From 09c724b8d5bdb4979bc96f995ab5163c93650d05 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Wed, 22 Mar 2023 07:57:26 +1100 Subject: [PATCH] Add move files external interface (#3557) * Add moveFiles graphql mutation * Move library resolution code into config * Implement file moving * Log if old file not removed in SafeMove * Ensure extensions are consistent * Don't allow overwriting existing files --- graphql/schema/schema.graphql | 8 ++ graphql/schema/types/file.graphql | 14 +- internal/api/resolver_mutation_file.go | 129 ++++++++++++++++++ internal/manager/config/config.go | 17 +-- internal/manager/config/stash_config.go | 40 ++++++ internal/manager/manager_tasks.go | 4 +- internal/manager/task_clean.go | 22 +-- internal/manager/task_scan.go | 4 +- pkg/file/delete.go | 22 +-- pkg/file/folder.go | 50 ++++++- pkg/file/move.go | 172 ++++++++++++++++++++++++ pkg/fsutil/file.go | 4 +- 12 files changed, 436 insertions(+), 50 deletions(-) create mode 100644 internal/manager/config/stash_config.go create mode 100644 pkg/file/move.go diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 87b0a916f..112f8aba9 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -245,6 +245,14 @@ type Mutation { tagsDestroy(ids: [ID!]!): Boolean! tagsMerge(input: TagsMergeInput!): Tag + """Moves the given files to the given destination. Returns true if successful. + Either the destination_folder or destination_folder_id must be provided. If both are provided, the destination_folder_id takes precedence. + Destination folder must be a subfolder of one of the stash library paths. + If provided, destination_basename must be a valid filename with an extension that + matches one of the media extensions. + Creates folder hierarchy if needed. + """ + moveFiles(input: MoveFilesInput!): Boolean! deleteFiles(ids: [ID!]!): Boolean! # Saved filters diff --git a/graphql/schema/types/file.graphql b/graphql/schema/types/file.graphql index 2493b622f..09b733c39 100644 --- a/graphql/schema/types/file.graphql +++ b/graphql/schema/types/file.graphql @@ -94,4 +94,16 @@ type GalleryFile implements BaseFile { created_at: Time! updated_at: Time! -} \ No newline at end of file +} + +input MoveFilesInput { + ids: [ID!]! + """valid for single or multiple file ids""" + destination_folder: String + + """valid for single or multiple file ids""" + destination_folder_id: ID + + """valid only for single file id. If empty, existing basename is used""" + destination_basename: String +} diff --git a/internal/api/resolver_mutation_file.go b/internal/api/resolver_mutation_file.go index 1a9fd66a0..4bc056e05 100644 --- a/internal/api/resolver_mutation_file.go +++ b/internal/api/resolver_mutation_file.go @@ -3,11 +3,140 @@ package api import ( "context" "fmt" + "strconv" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) +func (r *mutationResolver) MoveFiles(ctx context.Context, input MoveFilesInput) (bool, error) { + if err := r.withTxn(ctx, func(ctx context.Context) error { + qb := r.repository.File + mover := file.NewMover(qb) + mover.RegisterHooks(ctx, r.txnManager) + + var ( + folder *file.Folder + basename string + ) + + fileIDs, err := stringslice.StringSliceToIntSlice(input.Ids) + if err != nil { + return fmt.Errorf("converting file ids: %w", err) + } + + switch { + case input.DestinationFolderID != nil: + var err error + + folderID, err := strconv.Atoi(*input.DestinationFolderID) + if err != nil { + return fmt.Errorf("invalid folder id %s: %w", *input.DestinationFolderID, err) + } + + folder, err = r.repository.Folder.Find(ctx, file.FolderID(folderID)) + if err != nil { + return fmt.Errorf("finding destination folder: %w", err) + } + + if folder == nil { + return fmt.Errorf("folder with id %d not found", input.DestinationFolderID) + } + case input.DestinationFolder != nil: + folderPath := *input.DestinationFolder + + // ensure folder path is within the library + if err := r.validateFolderPath(folderPath); err != nil { + return err + } + + // get or create folder hierarchy + var err error + folder, err = file.GetOrCreateFolderHierarchy(ctx, r.repository.Folder, folderPath) + if err != nil { + return fmt.Errorf("getting or creating folder hierarchy: %w", err) + } + default: + return fmt.Errorf("must specify destination folder or path") + } + + if input.DestinationBasename != nil { + // ensure only one file was supplied + if len(input.Ids) != 1 { + return fmt.Errorf("must specify one file when providing destination path") + } + + basename = *input.DestinationBasename + } + + // create the folder hierarchy in the filesystem if needed + if err := mover.CreateFolderHierarchy(folder.Path); err != nil { + return fmt.Errorf("creating folder hierarchy %s in filesystem: %w", folder.Path, err) + } + + for _, fileIDInt := range fileIDs { + fileID := file.ID(fileIDInt) + f, err := qb.Find(ctx, fileID) + if err != nil { + return fmt.Errorf("finding file %d: %w", fileID, err) + } + + // ensure that the file extension matches the existing file type + if basename != "" { + if err := r.validateFileExtension(f[0].Base().Basename, basename); err != nil { + return err + } + } + + if err := mover.Move(ctx, f[0], folder, basename); err != nil { + return err + } + } + + return nil + }); err != nil { + return false, err + } + + return true, nil +} + +func (r *mutationResolver) validateFolderPath(folderPath string) error { + paths := manager.GetInstance().Config.GetStashPaths() + if l := paths.GetStashFromDirPath(folderPath); l == nil { + return fmt.Errorf("folder path %s must be within a stash library path", folderPath) + } + + return nil +} + +func (r *mutationResolver) validateFileExtension(oldBasename, newBasename string) error { + c := manager.GetInstance().Config + if err := r.validateFileExtensionList(c.GetVideoExtensions(), oldBasename, newBasename); err != nil { + return err + } + + if err := r.validateFileExtensionList(c.GetImageExtensions(), oldBasename, newBasename); err != nil { + return err + } + + if err := r.validateFileExtensionList(c.GetGalleryExtensions(), oldBasename, newBasename); err != nil { + return err + } + + return nil +} + +func (r *mutationResolver) validateFileExtensionList(exts []string, oldBasename, newBasename string) error { + if fsutil.MatchExtension(oldBasename, exts) && !fsutil.MatchExtension(newBasename, exts) { + return fmt.Errorf("file extension for %s is inconsistent with old filename %s", newBasename, oldBasename) + } + + return nil +} + func (r *mutationResolver) DeleteFiles(ctx context.Context, ids []string) (ret bool, err error) { fileIDs, err := stringslice.StringSliceToIntSlice(ids) if err != nil { diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index d5c6b7ac6..4b2ba7921 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -504,27 +504,14 @@ func (i *Instance) getStringMapString(key string) map[string]string { return ret } -type StashConfig struct { - Path string `json:"path"` - ExcludeVideo bool `json:"excludeVideo"` - ExcludeImage bool `json:"excludeImage"` -} - -// Stash configuration details -type StashConfigInput struct { - Path string `json:"path"` - ExcludeVideo bool `json:"excludeVideo"` - ExcludeImage bool `json:"excludeImage"` -} - // GetStathPaths returns the configured stash library paths. // Works opposite to the usual case - it will return the override // value only if the main value is not set. -func (i *Instance) GetStashPaths() []*StashConfig { +func (i *Instance) GetStashPaths() StashConfigs { i.RLock() defer i.RUnlock() - var ret []*StashConfig + var ret StashConfigs v := i.main if !v.IsSet(Stash) { diff --git a/internal/manager/config/stash_config.go b/internal/manager/config/stash_config.go new file mode 100644 index 000000000..4a2cc7d60 --- /dev/null +++ b/internal/manager/config/stash_config.go @@ -0,0 +1,40 @@ +package config + +import ( + "path/filepath" + + "github.com/stashapp/stash/pkg/fsutil" +) + +// Stash configuration details +type StashConfigInput struct { + Path string `json:"path"` + ExcludeVideo bool `json:"excludeVideo"` + ExcludeImage bool `json:"excludeImage"` +} + +type StashConfig struct { + Path string `json:"path"` + ExcludeVideo bool `json:"excludeVideo"` + ExcludeImage bool `json:"excludeImage"` +} + +type StashConfigs []*StashConfig + +func (s StashConfigs) GetStashFromPath(path string) *StashConfig { + for _, f := range s { + if fsutil.IsPathInDir(f.Path, filepath.Dir(path)) { + return f + } + } + return nil +} + +func (s StashConfigs) GetStashFromDirPath(dirPath string) *StashConfig { + for _, f := range s { + if fsutil.IsPathInDir(f.Path, dirPath) { + return f + } + } + return nil +} diff --git a/internal/manager/manager_tasks.go b/internal/manager/manager_tasks.go index 110e7eb10..10bcacab0 100644 --- a/internal/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -37,9 +37,9 @@ func getScanPaths(inputPaths []string) []*config.StashConfig { return stashPaths } - var ret []*config.StashConfig + var ret config.StashConfigs for _, p := range inputPaths { - s := getStashFromDirPath(stashPaths, p) + s := stashPaths.GetStashFromDirPath(p) if s == nil { logger.Warnf("%s is not in the configured stash paths", p) continue diff --git a/internal/manager/task_clean.go b/internal/manager/task_clean.go index f6d7304f8..b90f11be8 100644 --- a/internal/manager/task_clean.go +++ b/internal/manager/task_clean.go @@ -164,9 +164,9 @@ func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) if info.IsDir() { fileOrFolder = "Folder" - stash = getStashFromDirPath(f.stashPaths, path) + stash = f.stashPaths.GetStashFromDirPath(path) } else { - stash = getStashFromPath(f.stashPaths, path) + stash = f.stashPaths.GetStashFromPath(path) } if stash == nil { @@ -449,21 +449,3 @@ func (h *cleanHandler) handleRelatedImages(ctx context.Context, fileDeleter *fil return nil } - -func getStashFromPath(stashes []*config.StashConfig, pathToCheck string) *config.StashConfig { - for _, f := range stashes { - if fsutil.IsPathInDir(f.Path, filepath.Dir(pathToCheck)) { - return f - } - } - return nil -} - -func getStashFromDirPath(stashes []*config.StashConfig, pathToCheck string) *config.StashConfig { - for _, f := range stashes { - if fsutil.IsPathInDir(f.Path, pathToCheck) { - return f - } - } - return nil -} diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index 6b24af27b..fa31af610 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -226,7 +226,7 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool { type scanFilter struct { extensionConfig - stashPaths []*config.StashConfig + stashPaths config.StashConfigs generatedPath string videoExcludeRegex []*regexp.Regexp imageExcludeRegex []*regexp.Regexp @@ -278,7 +278,7 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) return false } - s := getStashFromDirPath(f.stashPaths, path) + s := f.stashPaths.GetStashFromDirPath(path) if s == nil { logger.Debugf("Skipping %s as it is not in the stash library", path) diff --git a/pkg/file/delete.go b/pkg/file/delete.go index 4c7a621b6..9ee27c176 100644 --- a/pkg/file/delete.go +++ b/pkg/file/delete.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/txn" ) @@ -15,10 +16,10 @@ const deleteFileSuffix = ".delete" // RenamerRemover provides access to the Rename and Remove functions. type RenamerRemover interface { - Rename(oldpath, newpath string) error + Renamer Remove(name string) error RemoveAll(path string) error - Stat(name string) (fs.FileInfo, error) + Statter } type renamerRemoverImpl struct { @@ -44,6 +45,16 @@ func (r renamerRemoverImpl) Stat(path string) (fs.FileInfo, error) { return r.StatFn(path) } +func newRenamerRemoverImpl() renamerRemoverImpl { + return renamerRemoverImpl{ + // use fsutil.SafeMove to support cross-device moves + RenameFn: fsutil.SafeMove, + RemoveFn: os.Remove, + RemoveAllFn: os.RemoveAll, + StatFn: os.Stat, + } +} + // Deleter is used to safely delete files and directories from the filesystem. // During a transaction, files and directories are marked for deletion using // the Files and Dirs methods. This will rename the files/directories to be @@ -59,12 +70,7 @@ type Deleter struct { func NewDeleter() *Deleter { return &Deleter{ - RenamerRemover: renamerRemoverImpl{ - RenameFn: os.Rename, - RemoveFn: os.Remove, - RemoveAllFn: os.RemoveAll, - StatFn: os.Stat, - }, + RenamerRemover: newRenamerRemoverImpl(), } } diff --git a/pkg/file/folder.go b/pkg/file/folder.go index 2eb4edd12..75d5716e3 100644 --- a/pkg/file/folder.go +++ b/pkg/file/folder.go @@ -2,7 +2,9 @@ package file import ( "context" + "fmt" "io/fs" + "path/filepath" "strconv" "time" ) @@ -30,9 +32,14 @@ func (f *Folder) Info(fs FS) (fs.FileInfo, error) { return f.info(fs, f.Path) } +// FolderPathFinder finds Folders by their path. +type FolderPathFinder interface { + FindByPath(ctx context.Context, path string) (*Folder, error) +} + // FolderGetter provides methods to find Folders. type FolderGetter interface { - FindByPath(ctx context.Context, path string) (*Folder, error) + FolderPathFinder FindByZipFileID(ctx context.Context, zipFileID ID) ([]*Folder, error) FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error) FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error) @@ -47,6 +54,11 @@ type FolderCreator interface { Create(ctx context.Context, f *Folder) error } +type FolderFinderCreator interface { + FolderPathFinder + FolderCreator +} + // FolderUpdater provides methods to update Folders. type FolderUpdater interface { Update(ctx context.Context, f *Folder) error @@ -69,3 +81,39 @@ type FolderStore interface { FolderUpdater FolderDestroyer } + +// GetOrCreateFolderHierarchy gets the folder for the given path, or creates a folder hierarchy for the given path if one if no existing folder is found. +// Does not create any folders in the file system +func GetOrCreateFolderHierarchy(ctx context.Context, fc FolderFinderCreator, path string) (*Folder, error) { + // get or create folder hierarchy + folder, err := fc.FindByPath(ctx, path) + if err != nil { + return nil, err + } + + if folder == nil { + parentPath := filepath.Dir(path) + parent, err := GetOrCreateFolderHierarchy(ctx, fc, parentPath) + if err != nil { + return nil, err + } + + now := time.Now() + + folder = &Folder{ + Path: path, + ParentFolderID: &parent.ID, + DirEntry: DirEntry{ + // leave mod time empty for now - it will be updated when the folder is scanned + }, + CreatedAt: now, + UpdatedAt: now, + } + + if err = fc.Create(ctx, folder); err != nil { + return nil, fmt.Errorf("creating folder %s: %w", path, err) + } + } + + return folder, nil +} diff --git a/pkg/file/move.go b/pkg/file/move.go new file mode 100644 index 000000000..f965489ef --- /dev/null +++ b/pkg/file/move.go @@ -0,0 +1,172 @@ +package file + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/txn" +) + +type Renamer interface { + Rename(oldpath, newpath string) error +} + +type Statter interface { + Stat(name string) (fs.FileInfo, error) +} + +type DirMakerStatRenamer interface { + Statter + Renamer + Mkdir(name string, perm os.FileMode) error + Remove(name string) error +} + +type folderCreatorStatRenamerImpl struct { + renamerRemoverImpl + mkDirFn func(name string, perm os.FileMode) error +} + +func (r folderCreatorStatRenamerImpl) Mkdir(name string, perm os.FileMode) error { + return r.mkDirFn(name, perm) +} + +type Mover struct { + Renamer DirMakerStatRenamer + Updater Updater + + moved map[string]string + foldersCreated []string +} + +func NewMover(u Updater) *Mover { + return &Mover{ + Updater: u, + Renamer: &folderCreatorStatRenamerImpl{ + renamerRemoverImpl: newRenamerRemoverImpl(), + mkDirFn: os.Mkdir, + }, + } +} + +// Move moves the file to the given folder and basename. If basename is empty, then the existing basename is used. +// Assumes that the parent folder exists in the filesystem. +func (m *Mover) Move(ctx context.Context, f File, folder *Folder, basename string) error { + fBase := f.Base() + + // don't allow moving files in zip files + if fBase.ZipFileID != nil { + return fmt.Errorf("cannot move file %s in zip file", f.Base().Path) + } + + if basename == "" { + basename = fBase.Basename + } + + // modify the database first + + oldPath := fBase.Path + + if folder.ID == fBase.ParentFolderID && (basename == "" || basename == fBase.Basename) { + // nothing to do + return nil + } + + // ensure that the new path doesn't already exist + newPath := filepath.Join(folder.Path, basename) + if _, err := m.Renamer.Stat(newPath); !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("file %s already exists", newPath) + } + + fBase.ParentFolderID = folder.ID + fBase.Basename = basename + fBase.UpdatedAt = time.Now() + // leave ModTime as is. It may or may not be changed by this operation + + if err := m.Updater.Update(ctx, f); err != nil { + return fmt.Errorf("updating file %s: %w", oldPath, err) + } + + // then move the file + return m.moveFile(oldPath, newPath) +} + +func (m *Mover) CreateFolderHierarchy(path string) error { + info, err := m.Renamer.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // create the parent folder + parentPath := filepath.Dir(path) + if err := m.CreateFolderHierarchy(parentPath); err != nil { + return err + } + + // create the folder + if err := m.Renamer.Mkdir(path, 0755); err != nil { + return fmt.Errorf("creating folder %s: %w", path, err) + } + + m.foldersCreated = append(m.foldersCreated, path) + } else { + return fmt.Errorf("getting info for %s: %w", path, err) + } + } else { + if !info.IsDir() { + return fmt.Errorf("%s is not a directory", path) + } + } + + return nil +} + +func (m *Mover) moveFile(oldPath, newPath string) error { + if err := m.Renamer.Rename(oldPath, newPath); err != nil { + return fmt.Errorf("renaming file %s to %s: %w", oldPath, newPath, err) + } + + if m.moved == nil { + m.moved = make(map[string]string) + } + + m.moved[newPath] = oldPath + + return nil +} + +func (m *Mover) RegisterHooks(ctx context.Context, mgr txn.Manager) { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { + m.commit() + }) + + txn.AddPostRollbackHook(ctx, func(ctx context.Context) { + m.rollback() + }) +} + +func (m *Mover) commit() { + m.moved = nil + m.foldersCreated = nil +} + +func (m *Mover) rollback() { + // move files back to their original location + for newPath, oldPath := range m.moved { + if err := m.Renamer.Rename(newPath, oldPath); err != nil { + logger.Errorf("error moving file %s back to %s: %s", newPath, oldPath, err.Error()) + } + } + + // remove folders created in reverse order + for i := len(m.foldersCreated) - 1; i >= 0; i-- { + folder := m.foldersCreated[i] + if err := m.Renamer.Remove(folder); err != nil { + logger.Errorf("error removing folder %s: %s", folder, err.Error()) + } + } +} diff --git a/pkg/fsutil/file.go b/pkg/fsutil/file.go index 60a50d74f..7d91679fe 100644 --- a/pkg/fsutil/file.go +++ b/pkg/fsutil/file.go @@ -7,6 +7,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/stashapp/stash/pkg/logger" ) // SafeMove attempts to move the file with path src to dest using os.Rename. If this fails, then it copies src to dest, then deletes src. @@ -38,7 +40,7 @@ func SafeMove(src, dst string) error { err = os.Remove(src) if err != nil { - return err + logger.Errorf("error removing old file %s during SafeMove: %v", src, err) } }