Add move files external interface (#3557)

* Add moveFiles graphql mutation
* Move library resolution code into config
* Implement file moving
* Log if old file not removed in SafeMove
* Ensure extensions are consistent
* Don't allow overwriting existing files

graphql/schema/schema.graphql

@@ -245,6 +245,14 @@ type Mutation {
   tagsDestroy(ids: [ID!]!): Boolean!
   tagsMerge(input: TagsMergeInput!): Tag
 
+  """Moves the given files to the given destination. Returns true if successful.
+  Either the destination_folder or destination_folder_id must be provided. If both are provided, the destination_folder_id takes precedence.
+  Destination folder must be a subfolder of one of the stash library paths.
+  If provided, destination_basename must be a valid filename with an extension that
+  matches one of the media extensions.
+  Creates folder hierarchy if needed.
+  """
+  moveFiles(input: MoveFilesInput!): Boolean!
   deleteFiles(ids: [ID!]!): Boolean!
 
   # Saved filters

graphql/schema/types/file.graphql

@@ -94,4 +94,16 @@ type GalleryFile implements BaseFile {
 
     created_at: Time!
     updated_at: Time!
-}
+}
+
+input MoveFilesInput {
+    ids: [ID!]!
+    """valid for single or multiple file ids"""
+    destination_folder: String
+
+    """valid for single or multiple file ids"""
+    destination_folder_id: ID
+
+    """valid only for single file id. If empty, existing basename is used"""
+    destination_basename: String
+}

internal/api/resolver_mutation_file.go

@@ -3,11 +3,140 @@ package api
 import (
 	"context"
 	"fmt"
+	"strconv"
 
+	"github.com/stashapp/stash/internal/manager"
 	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/sliceutil/stringslice"
 )
 
+func (r *mutationResolver) MoveFiles(ctx context.Context, input MoveFilesInput) (bool, error) {
+	if err := r.withTxn(ctx, func(ctx context.Context) error {
+		qb := r.repository.File
+		mover := file.NewMover(qb)
+		mover.RegisterHooks(ctx, r.txnManager)
+
+		var (
+			folder   *file.Folder
+			basename string
+		)
+
+		fileIDs, err := stringslice.StringSliceToIntSlice(input.Ids)
+		if err != nil {
+			return fmt.Errorf("converting file ids: %w", err)
+		}
+
+		switch {
+		case input.DestinationFolderID != nil:
+			var err error
+
+			folderID, err := strconv.Atoi(*input.DestinationFolderID)
+			if err != nil {
+				return fmt.Errorf("invalid folder id %s: %w", *input.DestinationFolderID, err)
+			}
+
+			folder, err = r.repository.Folder.Find(ctx, file.FolderID(folderID))
+			if err != nil {
+				return fmt.Errorf("finding destination folder: %w", err)
+			}
+
+			if folder == nil {
+				return fmt.Errorf("folder with id %d not found", input.DestinationFolderID)
+			}
+		case input.DestinationFolder != nil:
+			folderPath := *input.DestinationFolder
+
+			// ensure folder path is within the library
+			if err := r.validateFolderPath(folderPath); err != nil {
+				return err
+			}
+
+			// get or create folder hierarchy
+			var err error
+			folder, err = file.GetOrCreateFolderHierarchy(ctx, r.repository.Folder, folderPath)
+			if err != nil {
+				return fmt.Errorf("getting or creating folder hierarchy: %w", err)
+			}
+		default:
+			return fmt.Errorf("must specify destination folder or path")
+		}
+
+		if input.DestinationBasename != nil {
+			// ensure only one file was supplied
+			if len(input.Ids) != 1 {
+				return fmt.Errorf("must specify one file when providing destination path")
+			}
+
+			basename = *input.DestinationBasename
+		}
+
+		// create the folder hierarchy in the filesystem if needed
+		if err := mover.CreateFolderHierarchy(folder.Path); err != nil {
+			return fmt.Errorf("creating folder hierarchy %s in filesystem: %w", folder.Path, err)
+		}
+
+		for _, fileIDInt := range fileIDs {
+			fileID := file.ID(fileIDInt)
+			f, err := qb.Find(ctx, fileID)
+			if err != nil {
+				return fmt.Errorf("finding file %d: %w", fileID, err)
+			}
+
+			// ensure that the file extension matches the existing file type
+			if basename != "" {
+				if err := r.validateFileExtension(f[0].Base().Basename, basename); err != nil {
+					return err
+				}
+			}
+
+			if err := mover.Move(ctx, f[0], folder, basename); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (r *mutationResolver) validateFolderPath(folderPath string) error {
+	paths := manager.GetInstance().Config.GetStashPaths()
+	if l := paths.GetStashFromDirPath(folderPath); l == nil {
+		return fmt.Errorf("folder path %s must be within a stash library path", folderPath)
+	}
+
+	return nil
+}
+
+func (r *mutationResolver) validateFileExtension(oldBasename, newBasename string) error {
+	c := manager.GetInstance().Config
+	if err := r.validateFileExtensionList(c.GetVideoExtensions(), oldBasename, newBasename); err != nil {
+		return err
+	}
+
+	if err := r.validateFileExtensionList(c.GetImageExtensions(), oldBasename, newBasename); err != nil {
+		return err
+	}
+
+	if err := r.validateFileExtensionList(c.GetGalleryExtensions(), oldBasename, newBasename); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (r *mutationResolver) validateFileExtensionList(exts []string, oldBasename, newBasename string) error {
+	if fsutil.MatchExtension(oldBasename, exts) && !fsutil.MatchExtension(newBasename, exts) {
+		return fmt.Errorf("file extension for %s is inconsistent with old filename %s", newBasename, oldBasename)
+	}
+
+	return nil
+}
+
 func (r *mutationResolver) DeleteFiles(ctx context.Context, ids []string) (ret bool, err error) {
 	fileIDs, err := stringslice.StringSliceToIntSlice(ids)
 	if err != nil {

internal/manager/config/config.go

@@ -504,27 +504,14 @@ func (i *Instance) getStringMapString(key string) map[string]string {
 	return ret
 }
 
-type StashConfig struct {
-	Path         string `json:"path"`
-	ExcludeVideo bool   `json:"excludeVideo"`
-	ExcludeImage bool   `json:"excludeImage"`
-}
-
-// Stash configuration details
-type StashConfigInput struct {
-	Path         string `json:"path"`
-	ExcludeVideo bool   `json:"excludeVideo"`
-	ExcludeImage bool   `json:"excludeImage"`
-}
-
 // GetStathPaths returns the configured stash library paths.
 // Works opposite to the usual case - it will return the override
 // value only if the main value is not set.
-func (i *Instance) GetStashPaths() []*StashConfig {
+func (i *Instance) GetStashPaths() StashConfigs {
 	i.RLock()
 	defer i.RUnlock()
 
-	var ret []*StashConfig
+	var ret StashConfigs
 
 	v := i.main
 	if !v.IsSet(Stash) {

internal/manager/config/stash_config.go

@@ -0,0 +1,40 @@
+package config
+
+import (
+	"path/filepath"
+
+	"github.com/stashapp/stash/pkg/fsutil"
+)
+
+// Stash configuration details
+type StashConfigInput struct {
+	Path         string `json:"path"`
+	ExcludeVideo bool   `json:"excludeVideo"`
+	ExcludeImage bool   `json:"excludeImage"`
+}
+
+type StashConfig struct {
+	Path         string `json:"path"`
+	ExcludeVideo bool   `json:"excludeVideo"`
+	ExcludeImage bool   `json:"excludeImage"`
+}
+
+type StashConfigs []*StashConfig
+
+func (s StashConfigs) GetStashFromPath(path string) *StashConfig {
+	for _, f := range s {
+		if fsutil.IsPathInDir(f.Path, filepath.Dir(path)) {
+			return f
+		}
+	}
+	return nil
+}
+
+func (s StashConfigs) GetStashFromDirPath(dirPath string) *StashConfig {
+	for _, f := range s {
+		if fsutil.IsPathInDir(f.Path, dirPath) {
+			return f
+		}
+	}
+	return nil
+}

internal/manager/manager_tasks.go

@@ -37,9 +37,9 @@ func getScanPaths(inputPaths []string) []*config.StashConfig {
 		return stashPaths
 	}
 
-	var ret []*config.StashConfig
+	var ret config.StashConfigs
 	for _, p := range inputPaths {
-		s := getStashFromDirPath(stashPaths, p)
+		s := stashPaths.GetStashFromDirPath(p)
 		if s == nil {
 			logger.Warnf("%s is not in the configured stash paths", p)
 			continue

internal/manager/task_clean.go

@@ -164,9 +164,9 @@ func (f *cleanFilter) Accept(ctx context.Context, path string, info fs.FileInfo)
 
 	if info.IsDir() {
 		fileOrFolder = "Folder"
-		stash = getStashFromDirPath(f.stashPaths, path)
+		stash = f.stashPaths.GetStashFromDirPath(path)
 	} else {
-		stash = getStashFromPath(f.stashPaths, path)
+		stash = f.stashPaths.GetStashFromPath(path)
 	}
 
 	if stash == nil {
@@ -449,21 +449,3 @@ func (h *cleanHandler) handleRelatedImages(ctx context.Context, fileDeleter *fil
 
 	return nil
 }
-
-func getStashFromPath(stashes []*config.StashConfig, pathToCheck string) *config.StashConfig {
-	for _, f := range stashes {
-		if fsutil.IsPathInDir(f.Path, filepath.Dir(pathToCheck)) {
-			return f
-		}
-	}
-	return nil
-}
-
-func getStashFromDirPath(stashes []*config.StashConfig, pathToCheck string) *config.StashConfig {
-	for _, f := range stashes {
-		if fsutil.IsPathInDir(f.Path, pathToCheck) {
-			return f
-		}
-	}
-	return nil
-}

internal/manager/task_scan.go

@@ -226,7 +226,7 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool {
 
 type scanFilter struct {
 	extensionConfig
-	stashPaths        []*config.StashConfig
+	stashPaths        config.StashConfigs
 	generatedPath     string
 	videoExcludeRegex []*regexp.Regexp
 	imageExcludeRegex []*regexp.Regexp
@@ -278,7 +278,7 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo)
 		return false
 	}
 
-	s := getStashFromDirPath(f.stashPaths, path)
+	s := f.stashPaths.GetStashFromDirPath(path)
 
 	if s == nil {
 		logger.Debugf("Skipping %s as it is not in the stash library", path)

pkg/file/delete.go

@@ -7,6 +7,7 @@ import (
 	"io/fs"
 	"os"
 
+	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/txn"
 )
@@ -15,10 +16,10 @@ const deleteFileSuffix = ".delete"
 
 // RenamerRemover provides access to the Rename and Remove functions.
 type RenamerRemover interface {
-	Rename(oldpath, newpath string) error
+	Renamer
 	Remove(name string) error
 	RemoveAll(path string) error
-	Stat(name string) (fs.FileInfo, error)
+	Statter
 }
 
 type renamerRemoverImpl struct {
@@ -44,6 +45,16 @@ func (r renamerRemoverImpl) Stat(path string) (fs.FileInfo, error) {
 	return r.StatFn(path)
 }
 
+func newRenamerRemoverImpl() renamerRemoverImpl {
+	return renamerRemoverImpl{
+		// use fsutil.SafeMove to support cross-device moves
+		RenameFn:    fsutil.SafeMove,
+		RemoveFn:    os.Remove,
+		RemoveAllFn: os.RemoveAll,
+		StatFn:      os.Stat,
+	}
+}
+
 // Deleter is used to safely delete files and directories from the filesystem.
 // During a transaction, files and directories are marked for deletion using
 // the Files and Dirs methods. This will rename the files/directories to be
@@ -59,12 +70,7 @@ type Deleter struct {
 
 func NewDeleter() *Deleter {
 	return &Deleter{
-		RenamerRemover: renamerRemoverImpl{
-			RenameFn:    os.Rename,
-			RemoveFn:    os.Remove,
-			RemoveAllFn: os.RemoveAll,
-			StatFn:      os.Stat,
-		},
+		RenamerRemover: newRenamerRemoverImpl(),
 	}
 }
 

pkg/file/folder.go

@@ -2,7 +2,9 @@ package file
 
 import (
 	"context"
+	"fmt"
 	"io/fs"
+	"path/filepath"
 	"strconv"
 	"time"
 )
@@ -30,9 +32,14 @@ func (f *Folder) Info(fs FS) (fs.FileInfo, error) {
 	return f.info(fs, f.Path)
 }
 
+// FolderPathFinder finds Folders by their path.
+type FolderPathFinder interface {
+	FindByPath(ctx context.Context, path string) (*Folder, error)
+}
+
 // FolderGetter provides methods to find Folders.
 type FolderGetter interface {
-	FindByPath(ctx context.Context, path string) (*Folder, error)
+	FolderPathFinder
 	FindByZipFileID(ctx context.Context, zipFileID ID) ([]*Folder, error)
 	FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error)
 	FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error)
@@ -47,6 +54,11 @@ type FolderCreator interface {
 	Create(ctx context.Context, f *Folder) error
 }
 
+type FolderFinderCreator interface {
+	FolderPathFinder
+	FolderCreator
+}
+
 // FolderUpdater provides methods to update Folders.
 type FolderUpdater interface {
 	Update(ctx context.Context, f *Folder) error
@@ -69,3 +81,39 @@ type FolderStore interface {
 	FolderUpdater
 	FolderDestroyer
 }
+
+// GetOrCreateFolderHierarchy gets the folder for the given path, or creates a folder hierarchy for the given path if one if no existing folder is found.
+// Does not create any folders in the file system
+func GetOrCreateFolderHierarchy(ctx context.Context, fc FolderFinderCreator, path string) (*Folder, error) {
+	// get or create folder hierarchy
+	folder, err := fc.FindByPath(ctx, path)
+	if err != nil {
+		return nil, err
+	}
+
+	if folder == nil {
+		parentPath := filepath.Dir(path)
+		parent, err := GetOrCreateFolderHierarchy(ctx, fc, parentPath)
+		if err != nil {
+			return nil, err
+		}
+
+		now := time.Now()
+
+		folder = &Folder{
+			Path:           path,
+			ParentFolderID: &parent.ID,
+			DirEntry:       DirEntry{
+				// leave mod time empty for now - it will be updated when the folder is scanned
+			},
+			CreatedAt: now,
+			UpdatedAt: now,
+		}
+
+		if err = fc.Create(ctx, folder); err != nil {
+			return nil, fmt.Errorf("creating folder %s: %w", path, err)
+		}
+	}
+
+	return folder, nil
+}

pkg/file/move.go

@@ -0,0 +1,172 @@
+package file
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/txn"
+)
+
+type Renamer interface {
+	Rename(oldpath, newpath string) error
+}
+
+type Statter interface {
+	Stat(name string) (fs.FileInfo, error)
+}
+
+type DirMakerStatRenamer interface {
+	Statter
+	Renamer
+	Mkdir(name string, perm os.FileMode) error
+	Remove(name string) error
+}
+
+type folderCreatorStatRenamerImpl struct {
+	renamerRemoverImpl
+	mkDirFn func(name string, perm os.FileMode) error
+}
+
+func (r folderCreatorStatRenamerImpl) Mkdir(name string, perm os.FileMode) error {
+	return r.mkDirFn(name, perm)
+}
+
+type Mover struct {
+	Renamer DirMakerStatRenamer
+	Updater Updater
+
+	moved          map[string]string
+	foldersCreated []string
+}
+
+func NewMover(u Updater) *Mover {
+	return &Mover{
+		Updater: u,
+		Renamer: &folderCreatorStatRenamerImpl{
+			renamerRemoverImpl: newRenamerRemoverImpl(),
+			mkDirFn:            os.Mkdir,
+		},
+	}
+}
+
+// Move moves the file to the given folder and basename. If basename is empty, then the existing basename is used.
+// Assumes that the parent folder exists in the filesystem.
+func (m *Mover) Move(ctx context.Context, f File, folder *Folder, basename string) error {
+	fBase := f.Base()
+
+	// don't allow moving files in zip files
+	if fBase.ZipFileID != nil {
+		return fmt.Errorf("cannot move file %s in zip file", f.Base().Path)
+	}
+
+	if basename == "" {
+		basename = fBase.Basename
+	}
+
+	// modify the database first
+
+	oldPath := fBase.Path
+
+	if folder.ID == fBase.ParentFolderID && (basename == "" || basename == fBase.Basename) {
+		// nothing to do
+		return nil
+	}
+
+	// ensure that the new path doesn't already exist
+	newPath := filepath.Join(folder.Path, basename)
+	if _, err := m.Renamer.Stat(newPath); !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("file %s already exists", newPath)
+	}
+
+	fBase.ParentFolderID = folder.ID
+	fBase.Basename = basename
+	fBase.UpdatedAt = time.Now()
+	// leave ModTime as is. It may or may not be changed by this operation
+
+	if err := m.Updater.Update(ctx, f); err != nil {
+		return fmt.Errorf("updating file %s: %w", oldPath, err)
+	}
+
+	// then move the file
+	return m.moveFile(oldPath, newPath)
+}
+
+func (m *Mover) CreateFolderHierarchy(path string) error {
+	info, err := m.Renamer.Stat(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			// create the parent folder
+			parentPath := filepath.Dir(path)
+			if err := m.CreateFolderHierarchy(parentPath); err != nil {
+				return err
+			}
+
+			// create the folder
+			if err := m.Renamer.Mkdir(path, 0755); err != nil {
+				return fmt.Errorf("creating folder %s: %w", path, err)
+			}
+
+			m.foldersCreated = append(m.foldersCreated, path)
+		} else {
+			return fmt.Errorf("getting info for %s: %w", path, err)
+		}
+	} else {
+		if !info.IsDir() {
+			return fmt.Errorf("%s is not a directory", path)
+		}
+	}
+
+	return nil
+}
+
+func (m *Mover) moveFile(oldPath, newPath string) error {
+	if err := m.Renamer.Rename(oldPath, newPath); err != nil {
+		return fmt.Errorf("renaming file %s to %s: %w", oldPath, newPath, err)
+	}
+
+	if m.moved == nil {
+		m.moved = make(map[string]string)
+	}
+
+	m.moved[newPath] = oldPath
+
+	return nil
+}
+
+func (m *Mover) RegisterHooks(ctx context.Context, mgr txn.Manager) {
+	txn.AddPostCommitHook(ctx, func(ctx context.Context) {
+		m.commit()
+	})
+
+	txn.AddPostRollbackHook(ctx, func(ctx context.Context) {
+		m.rollback()
+	})
+}
+
+func (m *Mover) commit() {
+	m.moved = nil
+	m.foldersCreated = nil
+}
+
+func (m *Mover) rollback() {
+	// move files back to their original location
+	for newPath, oldPath := range m.moved {
+		if err := m.Renamer.Rename(newPath, oldPath); err != nil {
+			logger.Errorf("error moving file %s back to %s: %s", newPath, oldPath, err.Error())
+		}
+	}
+
+	// remove folders created in reverse order
+	for i := len(m.foldersCreated) - 1; i >= 0; i-- {
+		folder := m.foldersCreated[i]
+		if err := m.Renamer.Remove(folder); err != nil {
+			logger.Errorf("error removing folder %s: %s", folder, err.Error())
+		}
+	}
+}

pkg/fsutil/file.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/stashapp/stash/pkg/logger"
 )
 
 // SafeMove attempts to move the file with path src to dest using os.Rename. If this fails, then it copies src to dest, then deletes src.
@@ -38,7 +40,7 @@ func SafeMove(src, dst string) error {
 
 		err = os.Remove(src)
 		if err != nil {
-			return err
+			logger.Errorf("error removing old file %s during SafeMove: %v", src, err)
 		}
 	}