From cfc8222b9a47c1c6e763cf1e06f808fdb48d3eef Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Wed, 7 Sep 2022 16:50:15 +1000 Subject: [PATCH] [Files Refactor] Cleanup (#2893) * Clean up notes for develop merge * Remove commented code * Lint --- internal/desktop/desktop_platform_windows.go | 8 +- internal/manager/config/config.go | 3 +- internal/manager/task_scan_gallery.go | 160 --------- internal/manager/task_scan_image.go | 179 ---------- internal/manager/task_scan_scene.go | 116 ------ pkg/gallery/scan.go | 221 ------------ pkg/image/scan.go | 176 --------- pkg/scene/scan.go | 355 ------------------- pkg/utils/strings_test.go | 12 + ui/v2.5/src/docs/en/Changelog/v0170.md | 8 +- ui/v2.5/src/docs/en/MigrationNotes/32.md | 4 - ui/v2.5/src/docs/en/ReleaseNotes/20220826.md | 1 - ui/v2.5/src/docs/en/ReleaseNotes/index.ts | 7 +- ui/v2.5/src/docs/en/ReleaseNotes/v0170.md | 9 - 14 files changed, 19 insertions(+), 1240 deletions(-) delete mode 100644 internal/manager/task_scan_gallery.go delete mode 100644 internal/manager/task_scan_image.go delete mode 100644 internal/manager/task_scan_scene.go create mode 100644 pkg/utils/strings_test.go delete mode 100644 ui/v2.5/src/docs/en/ReleaseNotes/20220826.md diff --git a/internal/desktop/desktop_platform_windows.go b/internal/desktop/desktop_platform_windows.go index 1ff3786a3..ecb4060e6 100644 --- a/internal/desktop/desktop_platform_windows.go +++ b/internal/desktop/desktop_platform_windows.go @@ -28,10 +28,10 @@ func isService() bool { } // Detect if windows golang executable file is running via double click or from cmd/shell terminator -// https://stackoverflow.com/questions/8610489/distinguish-if-program-runs-by-clicking-on-the-icon-typing-its-name-in-the-cons?rq=1 -// https://github.com/shirou/w32/blob/master/kernel32.go -// https://github.com/kbinani/win/blob/master/kernel32.go#L3268 -// win.GetConsoleProcessList(new(uint32), win.DWORD(2)) +// https://stackoverflow.com/questions/8610489/distinguish-if-program-runs-by-clicking-on-the-icon-typing-its-name-in-the-cons?rq=1 +// https://github.com/shirou/w32/blob/master/kernel32.go +// https://github.com/kbinani/win/blob/master/kernel32.go#L3268 +// win.GetConsoleProcessList(new(uint32), win.DWORD(2)) // from https://gist.github.com/yougg/213250cc04a52e2b853590b06f49d865 func isDoubleClickLaunched() bool { lp := kernel32.NewProc("GetConsoleProcessList") diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index 90fa2e742..3be7be32a 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -311,8 +311,7 @@ func (i *Instance) GetNotificationsEnabled() bool { // GetShowOneTimeMovedNotification shows whether a small notification to inform the user that Stash // will no longer show a terminal window, and instead will be available in the tray, should be shown. -// -// It is true when an existing system is started after upgrading, and set to false forever after it is shown. +// It is true when an existing system is started after upgrading, and set to false forever after it is shown. func (i *Instance) GetShowOneTimeMovedNotification() bool { return i.getBool(ShowOneTimeMovedNotification) } diff --git a/internal/manager/task_scan_gallery.go b/internal/manager/task_scan_gallery.go deleted file mode 100644 index 542bbdc7f..000000000 --- a/internal/manager/task_scan_gallery.go +++ /dev/null @@ -1,160 +0,0 @@ -package manager - -// func (t *ScanTask) scanGallery(ctx context.Context) { -// var g *models.Gallery -// path := t.file.Path() -// images := 0 -// scanImages := false - -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// var err error -// g, err = t.TxnManager.Gallery.FindByPath(ctx, path) - -// if g != nil && err == nil { -// images, err = t.TxnManager.Image.CountByGalleryID(ctx, g.ID) -// if err != nil { -// return fmt.Errorf("error getting images for zip gallery %s: %s", path, err.Error()) -// } -// } - -// return err -// }); err != nil { -// logger.Error(err.Error()) -// return -// } - -// scanner := gallery.Scanner{ -// Scanner: gallery.FileScanner(&file.FSHasher{}), -// ImageExtensions: instance.Config.GetImageExtensions(), -// StripFileExtension: t.StripFileExtension, -// CaseSensitiveFs: t.CaseSensitiveFs, -// CreatorUpdater: t.TxnManager.Gallery, -// Paths: instance.Paths, -// PluginCache: instance.PluginCache, -// MutexManager: t.mutexManager, -// } - -// var err error -// if g != nil { -// g, scanImages, err = scanner.ScanExisting(ctx, g, t.file) -// if err != nil { -// logger.Error(err.Error()) -// return -// } - -// // scan the zip files if the gallery has no images -// scanImages = scanImages || images == 0 -// } else { -// g, scanImages, err = scanner.ScanNew(ctx, t.file) -// if err != nil { -// logger.Error(err.Error()) -// } -// } - -// if g != nil { -// if scanImages { -// t.scanZipImages(ctx, g) -// } else { -// // in case thumbnails have been deleted, regenerate them -// t.regenerateZipImages(ctx, g) -// } -// } -// } - -// associates a gallery to a scene with the same basename -// func (t *ScanTask) associateGallery(ctx context.Context, wg *sizedwaitgroup.SizedWaitGroup) { -// path := t.file.Path() -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// r := t.TxnManager -// qb := r.Gallery -// sqb := r.Scene -// g, err := qb.FindByPath(ctx, path) -// if err != nil { -// return err -// } - -// if g == nil { -// // associate is run after scan is finished -// // should only happen if gallery is a directory or an io error occurs during hashing -// logger.Warnf("associate: gallery %s not found in DB", path) -// return nil -// } - -// basename := strings.TrimSuffix(path, filepath.Ext(path)) -// var relatedFiles []string -// vExt := config.GetInstance().GetVideoExtensions() -// // make a list of media files that can be related to the gallery -// for _, ext := range vExt { -// related := basename + "." + ext -// // exclude gallery extensions from the related files -// if !isGallery(related) { -// relatedFiles = append(relatedFiles, related) -// } -// } -// for _, scenePath := range relatedFiles { -// scene, _ := sqb.FindByPath(ctx, scenePath) -// // found related Scene -// if scene != nil { -// sceneGalleries, _ := sqb.FindByGalleryID(ctx, g.ID) // check if gallery is already associated to the scene -// isAssoc := false -// for _, sg := range sceneGalleries { -// if scene.ID == sg.ID { -// isAssoc = true -// break -// } -// } -// if !isAssoc { -// logger.Infof("associate: Gallery %s is related to scene: %d", path, scene.ID) -// if _, err := sqb.UpdatePartial(ctx, scene.ID, models.ScenePartial{ -// GalleryIDs: &models.UpdateIDs{ -// IDs: []int{g.ID}, -// Mode: models.RelationshipUpdateModeAdd, -// }, -// }); err != nil { -// return err -// } -// } -// } -// } -// return nil -// }); err != nil { -// logger.Error(err.Error()) -// } -// wg.Done() -// } - -// func (t *ScanTask) scanZipImages(ctx context.Context, zipGallery *models.Gallery) { -// err := walkGalleryZip(*zipGallery.Path, func(f *zip.File) error { -// // copy this task and change the filename -// subTask := *t - -// // filepath is the zip file and the internal file name, separated by a null byte -// subTask.file = file.ZipFile(*zipGallery.Path, f) -// subTask.zipGallery = zipGallery - -// // run the subtask and wait for it to complete -// subTask.Start(ctx) -// return nil -// }) -// if err != nil { -// logger.Warnf("failed to scan zip file images for %s: %s", *zipGallery.Path, err.Error()) -// } -// } - -// func (t *ScanTask) regenerateZipImages(ctx context.Context, zipGallery *models.Gallery) { -// var images []*models.Image -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// iqb := t.TxnManager.Image - -// var err error -// images, err = iqb.FindByGalleryID(ctx, zipGallery.ID) -// return err -// }); err != nil { -// logger.Warnf("failed to find gallery images: %s", err.Error()) -// return -// } - -// for _, img := range images { -// t.generateThumbnail(img) -// } -// } diff --git a/internal/manager/task_scan_image.go b/internal/manager/task_scan_image.go deleted file mode 100644 index b253262bd..000000000 --- a/internal/manager/task_scan_image.go +++ /dev/null @@ -1,179 +0,0 @@ -package manager - -// import ( -// "context" -// "errors" -// "os/exec" -// "path/filepath" -// "time" - -// "github.com/stashapp/stash/internal/manager/config" -// "github.com/stashapp/stash/pkg/file" -// "github.com/stashapp/stash/pkg/fsutil" -// "github.com/stashapp/stash/pkg/gallery" -// "github.com/stashapp/stash/pkg/hash/md5" -// "github.com/stashapp/stash/pkg/image" -// "github.com/stashapp/stash/pkg/logger" -// "github.com/stashapp/stash/pkg/models" -// "github.com/stashapp/stash/pkg/plugin" -// ) - -// func (t *ScanTask) scanImage(ctx context.Context) { -// var i *models.Image -// path := t.file.Path() - -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// var err error -// i, err = t.TxnManager.Image.FindByPath(ctx, path) -// return err -// }); err != nil { -// logger.Error(err.Error()) -// return -// } - -// scanner := image.Scanner{ -// Scanner: image.FileScanner(&file.FSHasher{}), -// StripFileExtension: t.StripFileExtension, -// TxnManager: t.TxnManager, -// CreatorUpdater: t.TxnManager.Image, -// CaseSensitiveFs: t.CaseSensitiveFs, -// Paths: GetInstance().Paths, -// PluginCache: instance.PluginCache, -// MutexManager: t.mutexManager, -// } - -// var err error -// if i != nil { -// i, err = scanner.ScanExisting(ctx, i, t.file) -// if err != nil { -// logger.Error(err.Error()) -// return -// } -// } else { -// i, err = scanner.ScanNew(ctx, t.file) -// if err != nil { -// logger.Error(err.Error()) -// return -// } - -// if i != nil { -// if t.zipGallery != nil { -// // associate with gallery -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// return gallery.AddImage(ctx, t.TxnManager.Gallery, t.zipGallery.ID, i.ID) -// }); err != nil { -// logger.Error(err.Error()) -// return -// } -// } else if config.GetInstance().GetCreateGalleriesFromFolders() { -// // create gallery from folder or associate with existing gallery -// logger.Infof("Associating image %s with folder gallery", i.Path) -// var galleryID int -// var isNewGallery bool -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// var err error -// galleryID, isNewGallery, err = t.associateImageWithFolderGallery(ctx, i.ID, t.TxnManager.Gallery) -// return err -// }); err != nil { -// logger.Error(err.Error()) -// return -// } - -// if isNewGallery { -// GetInstance().PluginCache.ExecutePostHooks(ctx, galleryID, plugin.GalleryCreatePost, nil, nil) -// } -// } -// } -// } - -// if i != nil { -// t.generateThumbnail(i) -// } -// } - -// type GalleryImageAssociator interface { -// FindByPath(ctx context.Context, path string) (*models.Gallery, error) -// Create(ctx context.Context, newGallery *models.Gallery) error -// gallery.ImageUpdater -// } - -// func (t *ScanTask) associateImageWithFolderGallery(ctx context.Context, imageID int, qb GalleryImageAssociator) (galleryID int, isNew bool, err error) { -// // find a gallery with the path specified -// path := filepath.Dir(t.file.Path()) -// var g *models.Gallery -// g, err = qb.FindByPath(ctx, path) -// if err != nil { -// return -// } - -// if g == nil { -// checksum := md5.FromString(path) - -// // create the gallery -// currentTime := time.Now() - -// title := fsutil.GetNameFromPath(path, false) - -// g = &models.Gallery{ -// Checksum: checksum, -// Path: &path, -// CreatedAt: currentTime, -// UpdatedAt: currentTime, -// Title: title, -// } - -// logger.Infof("Creating gallery for folder %s", path) -// err = qb.Create(ctx, g) -// if err != nil { -// return 0, false, err -// } - -// isNew = true -// } - -// // associate image with gallery -// err = gallery.AddImage(ctx, qb, g.ID, imageID) -// galleryID = g.ID -// return -// } - -// func (t *ScanTask) generateThumbnail(i *models.Image) { -// if !t.GenerateThumbnails { -// return -// } - -// thumbPath := GetInstance().Paths.Generated.GetThumbnailPath(i.Checksum, models.DefaultGthumbWidth) -// exists, _ := fsutil.FileExists(thumbPath) -// if exists { -// return -// } - -// config, _, err := image.DecodeSourceImage(i) -// if err != nil { -// logger.Errorf("error reading image %s: %s", i.Path, err.Error()) -// return -// } - -// if config.Height > models.DefaultGthumbWidth || config.Width > models.DefaultGthumbWidth { -// encoder := image.NewThumbnailEncoder(instance.FFMPEG) -// data, err := encoder.GetThumbnail(i, models.DefaultGthumbWidth) - -// if err != nil { -// // don't log for animated images -// if !errors.Is(err, image.ErrNotSupportedForThumbnail) { -// logger.Errorf("error getting thumbnail for image %s: %s", i.Path, err.Error()) - -// var exitErr *exec.ExitError -// if errors.As(err, &exitErr) { -// logger.Errorf("stderr: %s", string(exitErr.Stderr)) -// } -// } -// return -// } - -// err = fsutil.WriteFile(thumbPath, data) -// if err != nil { -// logger.Errorf("error writing thumbnail for image %s: %s", i.Path, err) -// } -// } -// } diff --git a/internal/manager/task_scan_scene.go b/internal/manager/task_scan_scene.go deleted file mode 100644 index e48ed19a8..000000000 --- a/internal/manager/task_scan_scene.go +++ /dev/null @@ -1,116 +0,0 @@ -package manager - -// type sceneScreenshotter struct { -// g *generate.Generator -// } - -// func (ss *sceneScreenshotter) GenerateScreenshot(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error { -// return ss.g.Screenshot(ctx, probeResult.Path, hash, probeResult.Width, probeResult.Duration, generate.ScreenshotOptions{}) -// } - -// func (ss *sceneScreenshotter) GenerateThumbnail(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error { -// return ss.g.Thumbnail(ctx, probeResult.Path, hash, probeResult.Duration, generate.ScreenshotOptions{}) -// } - -// func (t *ScanTask) scanScene(ctx context.Context) *models.Scene { -// logError := func(err error) *models.Scene { -// logger.Error(err.Error()) -// return nil -// } - -// var retScene *models.Scene -// var s *models.Scene - -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// var err error -// s, err = t.TxnManager.Scene.FindByPath(ctx, t.file.Path()) -// return err -// }); err != nil { -// logger.Error(err.Error()) -// return nil -// } - -// g := &generate.Generator{ -// Encoder: instance.FFMPEG, -// LockManager: instance.ReadLockManager, -// ScenePaths: instance.Paths.Scene, -// } - -// scanner := scene.Scanner{ -// Scanner: scene.FileScanner(&file.FSHasher{}, t.fileNamingAlgorithm, t.calculateMD5), -// StripFileExtension: t.StripFileExtension, -// FileNamingAlgorithm: t.fileNamingAlgorithm, -// TxnManager: t.TxnManager, -// CreatorUpdater: t.TxnManager.Scene, -// Paths: GetInstance().Paths, -// CaseSensitiveFs: t.CaseSensitiveFs, -// Screenshotter: &sceneScreenshotter{ -// g: g, -// }, -// VideoFileCreator: &instance.FFProbe, -// PluginCache: instance.PluginCache, -// MutexManager: t.mutexManager, -// UseFileMetadata: t.UseFileMetadata, -// } - -// if s != nil { -// if err := scanner.ScanExisting(ctx, s, t.file); err != nil { -// return logError(err) -// } - -// return nil -// } - -// var err error -// retScene, err = scanner.ScanNew(ctx, t.file) -// if err != nil { -// return logError(err) -// } - -// return retScene -// } - -// associates captions to scene/s with the same basename -// func (t *ScanTask) associateCaptions(ctx context.Context) { -// vExt := config.GetInstance().GetVideoExtensions() -// captionPath := t.file.Path() -// captionLang := scene.GetCaptionsLangFromPath(captionPath) - -// relatedFiles := scene.GenerateCaptionCandidates(captionPath, vExt) -// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error { -// var err error -// sqb := t.TxnManager.Scene - -// for _, scenePath := range relatedFiles { -// s, er := sqb.FindByPath(ctx, scenePath) - -// if er != nil { -// logger.Errorf("Error searching for scene %s: %v", scenePath, er) -// continue -// } -// if s != nil { // found related Scene -// logger.Debugf("Matched captions to scene %s", s.Path) -// captions, er := sqb.GetCaptions(ctx, s.ID) -// if er == nil { -// fileExt := filepath.Ext(captionPath) -// ext := fileExt[1:] -// if !scene.IsLangInCaptions(captionLang, ext, captions) { // only update captions if language code is not present -// newCaption := &models.SceneCaption{ -// LanguageCode: captionLang, -// Filename: filepath.Base(captionPath), -// CaptionType: ext, -// } -// captions = append(captions, newCaption) -// er = sqb.UpdateCaptions(ctx, s.ID, captions) -// if er == nil { -// logger.Debugf("Updated captions for scene %s. Added %s", s.Path, captionLang) -// } -// } -// } -// } -// } -// return err -// }); err != nil { -// logger.Error(err.Error()) -// } -// } diff --git a/pkg/gallery/scan.go b/pkg/gallery/scan.go index 5b63faa87..3908f1cc2 100644 --- a/pkg/gallery/scan.go +++ b/pkg/gallery/scan.go @@ -13,8 +13,6 @@ import ( "github.com/stashapp/stash/pkg/plugin" ) -// const mutexType = "gallery" - type FinderCreatorUpdater interface { Finder Create(ctx context.Context, newGallery *models.Gallery, fileIDs []file.ID) error @@ -133,222 +131,3 @@ func (h *ScanHandler) associateScene(ctx context.Context, existing []*models.Gal return nil } - -// type Scanner struct { -// file.Scanner - -// ImageExtensions []string -// StripFileExtension bool -// CaseSensitiveFs bool -// TxnManager txn.Manager -// CreatorUpdater FinderCreatorUpdater -// Paths *paths.Paths -// PluginCache *plugin.Cache -// MutexManager *utils.MutexManager -// } - -// func FileScanner(hasher file.Hasher) file.Scanner { -// return file.Scanner{ -// Hasher: hasher, -// CalculateMD5: true, -// } -// } - -// func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) { -// scanned, err := scanner.Scanner.ScanExisting(existing, file) -// if err != nil { -// return nil, false, err -// } - -// // we don't currently store sizes for gallery files -// // clear the file size so that we don't incorrectly detect a -// // change -// scanned.New.Size = "" - -// retGallery = existing.(*models.Gallery) - -// path := scanned.New.Path - -// changed := false - -// if scanned.ContentsChanged() { -// retGallery.SetFile(*scanned.New) -// changed = true -// } else if scanned.FileUpdated() { -// logger.Infof("Updated gallery file %s", path) - -// retGallery.SetFile(*scanned.New) -// changed = true -// } - -// if changed { -// scanImages = true -// logger.Infof("%s has been updated: rescanning", path) - -// retGallery.UpdatedAt = time.Now() - -// // we are operating on a checksum now, so grab a mutex on the checksum -// done := make(chan struct{}) -// scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// // free the mutex once transaction is complete -// defer close(done) - -// // ensure no clashes of hashes -// if scanned.New.Checksum != "" && scanned.Old.Checksum != scanned.New.Checksum { -// dupe, _ := scanner.CreatorUpdater.FindByChecksum(ctx, retGallery.Checksum) -// if dupe != nil { -// return fmt.Errorf("MD5 for file %s is the same as that of %s", path, *dupe.Path) -// } -// } - -// return scanner.CreatorUpdater.Update(ctx, retGallery) -// }); err != nil { -// return nil, false, err -// } - -// scanner.PluginCache.ExecutePostHooks(ctx, retGallery.ID, plugin.GalleryUpdatePost, nil, nil) -// } - -// return -// } - -// func (scanner *Scanner) ScanNew(ctx context.Context, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) { -// scanned, err := scanner.Scanner.ScanNew(file) -// if err != nil { -// return nil, false, err -// } - -// path := file.Path() -// checksum := scanned.Checksum -// isNewGallery := false -// isUpdatedGallery := false -// var g *models.Gallery - -// // grab a mutex on the checksum -// done := make(chan struct{}) -// scanner.MutexManager.Claim(mutexType, checksum, done) -// defer close(done) - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// qb := scanner.CreatorUpdater - -// g, _ = qb.FindByChecksum(ctx, checksum) -// if g != nil { -// exists, _ := fsutil.FileExists(*g.Path) -// if !scanner.CaseSensitiveFs { -// // #1426 - if file exists but is a case-insensitive match for the -// // original filename, then treat it as a move -// if exists && strings.EqualFold(path, *g.Path) { -// exists = false -// } -// } - -// if exists { -// logger.Infof("%s already exists. Duplicate of %s ", path, *g.Path) -// } else { -// logger.Infof("%s already exists. Updating path...", path) -// g.Path = &path -// err = qb.Update(ctx, g) -// if err != nil { -// return err -// } - -// isUpdatedGallery = true -// } -// } else if scanner.hasImages(path) { // don't create gallery if it has no images -// currentTime := time.Now() - -// title := fsutil.GetNameFromPath(path, scanner.StripFileExtension) -// g = &models.Gallery{ -// Zip: true, -// Title: title, -// CreatedAt: currentTime, -// UpdatedAt: currentTime, -// } - -// g.SetFile(*scanned) - -// // only warn when creating the gallery -// ok, err := isZipFileUncompressed(path) -// if err == nil && !ok { -// logger.Warnf("%s is using above store (0) level compression.", path) -// } - -// logger.Infof("%s doesn't exist. Creating new item...", path) -// err = qb.Create(ctx, g) -// if err != nil { -// return err -// } - -// scanImages = true -// isNewGallery = true -// } - -// return nil -// }); err != nil { -// return nil, false, err -// } - -// if isNewGallery { -// scanner.PluginCache.ExecutePostHooks(ctx, g.ID, plugin.GalleryCreatePost, nil, nil) -// } else if isUpdatedGallery { -// scanner.PluginCache.ExecutePostHooks(ctx, g.ID, plugin.GalleryUpdatePost, nil, nil) -// } - -// // Also scan images if zip file has been moved (ie updated) as the image paths are no longer valid -// scanImages = isNewGallery || isUpdatedGallery -// retGallery = g - -// return -// } - -// // IsZipFileUnmcompressed returns true if zip file in path is using 0 compression level -// func isZipFileUncompressed(path string) (bool, error) { -// r, err := zip.OpenReader(path) -// if err != nil { -// fmt.Printf("Error reading zip file %s: %s\n", path, err) -// return false, err -// } else { -// defer r.Close() -// for _, f := range r.File { -// if f.FileInfo().IsDir() { // skip dirs, they always get store level compression -// continue -// } -// return f.Method == 0, nil // check compression level of first actual file -// } -// } -// return false, nil -// } - -// func (scanner *Scanner) isImage(pathname string) bool { -// return fsutil.MatchExtension(pathname, scanner.ImageExtensions) -// } - -// func (scanner *Scanner) hasImages(path string) bool { -// readCloser, err := zip.OpenReader(path) -// if err != nil { -// logger.Warnf("Error while walking gallery zip: %v", err) -// return false -// } -// defer readCloser.Close() - -// for _, file := range readCloser.File { -// if file.FileInfo().IsDir() { -// continue -// } - -// if strings.Contains(file.Name, "__MACOSX") { -// continue -// } - -// if !scanner.isImage(file.Name) { -// continue -// } - -// return true -// } - -// return false -// } diff --git a/pkg/image/scan.go b/pkg/image/scan.go index 49bf99215..4f313ccc5 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -18,8 +18,6 @@ var ( ErrNotImageFile = errors.New("not an image file") ) -// const mutexType = "image" - type FinderCreatorUpdater interface { FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error) @@ -227,177 +225,3 @@ func (h *ScanHandler) associateFolderBasedGallery(ctx context.Context, newImage return nil } - -// type Scanner struct { -// file.Scanner - -// StripFileExtension bool - -// CaseSensitiveFs bool -// TxnManager txn.Manager -// CreatorUpdater FinderCreatorUpdater -// Paths *paths.Paths -// PluginCache *plugin.Cache -// MutexManager *utils.MutexManager -// } - -// func FileScanner(hasher file.Hasher) file.Scanner { -// return file.Scanner{ -// Hasher: hasher, -// CalculateMD5: true, -// } -// } - -// func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (retImage *models.Image, err error) { -// scanned, err := scanner.Scanner.ScanExisting(existing, file) -// if err != nil { -// return nil, err -// } - -// i := existing.(*models.Image) - -// path := scanned.New.Path -// oldChecksum := i.Checksum -// changed := false - -// if scanned.ContentsChanged() { -// logger.Infof("%s has been updated: rescanning", path) - -// // regenerate the file details as well -// if err := SetFileDetails(i); err != nil { -// return nil, err -// } - -// changed = true -// } else if scanned.FileUpdated() { -// logger.Infof("Updated image file %s", path) - -// changed = true -// } - -// if changed { -// i.SetFile(*scanned.New) -// i.UpdatedAt = time.Now() - -// // we are operating on a checksum now, so grab a mutex on the checksum -// done := make(chan struct{}) -// scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// // free the mutex once transaction is complete -// defer close(done) -// var err error - -// // ensure no clashes of hashes -// if scanned.New.Checksum != "" && scanned.Old.Checksum != scanned.New.Checksum { -// dupe, _ := scanner.CreatorUpdater.FindByChecksum(ctx, i.Checksum) -// if dupe != nil { -// return fmt.Errorf("MD5 for file %s is the same as that of %s", path, dupe.Path) -// } -// } - -// err = scanner.CreatorUpdater.Update(ctx, i) -// return err -// }); err != nil { -// return nil, err -// } - -// retImage = i - -// // remove the old thumbnail if the checksum changed - we'll regenerate it -// if oldChecksum != scanned.New.Checksum { -// // remove cache dir of gallery -// err = os.Remove(scanner.Paths.Generated.GetThumbnailPath(oldChecksum, models.DefaultGthumbWidth)) -// if err != nil { -// logger.Errorf("Error deleting thumbnail image: %s", err) -// } -// } - -// scanner.PluginCache.ExecutePostHooks(ctx, retImage.ID, plugin.ImageUpdatePost, nil, nil) -// } - -// return -// } - -// func (scanner *Scanner) ScanNew(ctx context.Context, f file.SourceFile) (retImage *models.Image, err error) { -// scanned, err := scanner.Scanner.ScanNew(f) -// if err != nil { -// return nil, err -// } - -// path := f.Path() -// checksum := scanned.Checksum - -// // grab a mutex on the checksum -// done := make(chan struct{}) -// scanner.MutexManager.Claim(mutexType, checksum, done) -// defer close(done) - -// // check for image by checksum -// var existingImage *models.Image -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// var err error -// existingImage, err = scanner.CreatorUpdater.FindByChecksum(ctx, checksum) -// return err -// }); err != nil { -// return nil, err -// } - -// pathDisplayName := file.ZipPathDisplayName(path) - -// if existingImage != nil { -// exists := FileExists(existingImage.Path) -// if !scanner.CaseSensitiveFs { -// // #1426 - if file exists but is a case-insensitive match for the -// // original filename, then treat it as a move -// if exists && strings.EqualFold(path, existingImage.Path) { -// exists = false -// } -// } - -// if exists { -// logger.Infof("%s already exists. Duplicate of %s ", pathDisplayName, file.ZipPathDisplayName(existingImage.Path)) -// return nil, nil -// } else { -// logger.Infof("%s already exists. Updating path...", pathDisplayName) - -// existingImage.Path = path -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// return scanner.CreatorUpdater.Update(ctx, existingImage) -// }); err != nil { -// return nil, err -// } - -// retImage = existingImage - -// scanner.PluginCache.ExecutePostHooks(ctx, existingImage.ID, plugin.ImageUpdatePost, nil, nil) -// } -// } else { -// logger.Infof("%s doesn't exist. Creating new item...", pathDisplayName) -// currentTime := time.Now() -// newImage := &models.Image{ -// CreatedAt: currentTime, -// UpdatedAt: currentTime, -// } -// newImage.SetFile(*scanned) -// fn := GetFilename(newImage, scanner.StripFileExtension) -// newImage.Title = fn - -// if err := SetFileDetails(newImage); err != nil { -// logger.Error(err.Error()) -// return nil, err -// } - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// return scanner.CreatorUpdater.Create(ctx, newImage) -// }); err != nil { -// return nil, err -// } - -// retImage = newImage - -// scanner.PluginCache.ExecutePostHooks(ctx, retImage.ID, plugin.ImageCreatePost, nil, nil) -// } - -// return -// } diff --git a/pkg/scene/scan.go b/pkg/scene/scan.go index bc9e7bd2e..cf9b0d6fc 100644 --- a/pkg/scene/scan.go +++ b/pkg/scene/scan.go @@ -16,8 +16,6 @@ var ( ErrNotVideoFile = errors.New("not a video file") ) -// const mutexType = "scene" - type CreatorUpdater interface { FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Scene, error) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Scene, error) @@ -140,356 +138,3 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. return nil } - -// type videoFileCreator interface { -// NewVideoFile(path string) (*ffmpeg.VideoFile, error) -// } - -// type Scanner struct { -// file.Scanner - -// StripFileExtension bool -// UseFileMetadata bool -// FileNamingAlgorithm models.HashAlgorithm - -// CaseSensitiveFs bool -// TxnManager txn.Manager -// CreatorUpdater CreatorUpdater -// Paths *paths.Paths -// Screenshotter screenshotter -// VideoFileCreator videoFileCreator -// PluginCache *plugin.Cache -// MutexManager *utils.MutexManager -// } - -// func FileScanner(hasher file.Hasher, fileNamingAlgorithm models.HashAlgorithm, calculateMD5 bool) file.Scanner { -// return file.Scanner{ -// Hasher: hasher, -// CalculateOSHash: true, -// CalculateMD5: fileNamingAlgorithm == models.HashAlgorithmMd5 || calculateMD5, -// } -// } - -// func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (err error) { -// scanned, err := scanner.Scanner.ScanExisting(existing, file) -// if err != nil { -// return err -// } - -// s := existing.(*models.Scene) - -// path := scanned.New.Path -// interactive := getInteractive(path) - -// oldHash := s.GetHash(scanner.FileNamingAlgorithm) -// changed := false - -// var videoFile *ffmpeg.VideoFile - -// if scanned.ContentsChanged() { -// logger.Infof("%s has been updated: rescanning", path) - -// s.SetFile(*scanned.New) - -// videoFile, err = scanner.VideoFileCreator.NewVideoFile(path) -// if err != nil { -// return err -// } - -// if err := videoFileToScene(s, videoFile); err != nil { -// return err -// } -// changed = true -// } else if scanned.FileUpdated() || s.Interactive != interactive { -// logger.Infof("Updated scene file %s", path) - -// // update fields as needed -// s.SetFile(*scanned.New) -// changed = true -// } - -// // check for container -// if s.Format == nil { -// if videoFile == nil { -// videoFile, err = scanner.VideoFileCreator.NewVideoFile(path) -// if err != nil { -// return err -// } -// } -// container, err := ffmpeg.MatchContainer(videoFile.Container, path) -// if err != nil { -// return fmt.Errorf("getting container for %s: %w", path, err) -// } -// logger.Infof("Adding container %s to file %s", container, path) -// containerStr := string(container) -// s.Format = &containerStr -// changed = true -// } - -// qb := scanner.CreatorUpdater - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// var err error - -// captions, er := qb.GetCaptions(ctx, s.ID) -// if er == nil { -// if len(captions) > 0 { -// clean, altered := CleanCaptions(s.Path, captions) -// if altered { -// er = qb.UpdateCaptions(ctx, s.ID, clean) -// if er == nil { -// logger.Debugf("Captions for %s cleaned: %s -> %s", path, captions, clean) -// } -// } -// } -// } -// return err -// }); err != nil { -// logger.Error(err.Error()) -// } - -// if changed { -// // we are operating on a checksum now, so grab a mutex on the checksum -// done := make(chan struct{}) -// if scanned.New.OSHash != "" { -// scanner.MutexManager.Claim(mutexType, scanned.New.OSHash, done) -// } -// if scanned.New.Checksum != "" { -// scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) -// } - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// defer close(done) -// qb := scanner.CreatorUpdater - -// // ensure no clashes of hashes -// if scanned.New.Checksum != "" && scanned.Old.Checksum != scanned.New.Checksum { -// dupe, _ := qb.FindByChecksum(ctx, *s.Checksum) -// if dupe != nil { -// return fmt.Errorf("MD5 for file %s is the same as that of %s", path, dupe.Path) -// } -// } - -// if scanned.New.OSHash != "" && scanned.Old.OSHash != scanned.New.OSHash { -// dupe, _ := qb.FindByOSHash(ctx, scanned.New.OSHash) -// if dupe != nil { -// return fmt.Errorf("OSHash for file %s is the same as that of %s", path, dupe.Path) -// } -// } - -// s.Interactive = interactive -// s.UpdatedAt = time.Now() - -// return qb.Update(ctx, s) -// }); err != nil { -// return err -// } - -// // Migrate any generated files if the hash has changed -// newHash := s.GetHash(scanner.FileNamingAlgorithm) -// if newHash != oldHash { -// MigrateHash(scanner.Paths, oldHash, newHash) -// } - -// scanner.PluginCache.ExecutePostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil) -// } - -// // We already have this item in the database -// // check for thumbnails, screenshots -// scanner.makeScreenshots(path, videoFile, s.GetHash(scanner.FileNamingAlgorithm)) - -// return nil -// } - -// func (scanner *Scanner) ScanNew(ctx context.Context, file file.SourceFile) (retScene *models.Scene, err error) { -// scanned, err := scanner.Scanner.ScanNew(file) -// if err != nil { -// return nil, err -// } - -// path := file.Path() -// checksum := scanned.Checksum -// oshash := scanned.OSHash - -// // grab a mutex on the checksum and oshash -// done := make(chan struct{}) -// if oshash != "" { -// scanner.MutexManager.Claim(mutexType, oshash, done) -// } -// if checksum != "" { -// scanner.MutexManager.Claim(mutexType, checksum, done) -// } - -// defer close(done) - -// // check for scene by checksum and oshash - MD5 should be -// // redundant, but check both -// var s *models.Scene -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// qb := scanner.CreatorUpdater -// if checksum != "" { -// s, _ = qb.FindByChecksum(ctx, checksum) -// } - -// if s == nil { -// s, _ = qb.FindByOSHash(ctx, oshash) -// } - -// return nil -// }); err != nil { -// return nil, err -// } - -// sceneHash := oshash - -// if scanner.FileNamingAlgorithm == models.HashAlgorithmMd5 { -// sceneHash = checksum -// } - -// interactive := getInteractive(file.Path()) - -// if s != nil { -// exists, _ := fsutil.FileExists(s.Path) -// if !scanner.CaseSensitiveFs { -// // #1426 - if file exists but is a case-insensitive match for the -// // original filename, then treat it as a move -// if exists && strings.EqualFold(path, s.Path) { -// exists = false -// } -// } - -// if exists { -// logger.Infof("%s already exists. Duplicate of %s", path, s.Path) -// } else { -// logger.Infof("%s already exists. Updating path...", path) -// scenePartial := models.ScenePartial{ -// Path: models.NewOptionalString(path), -// Interactive: models.NewOptionalBool(interactive), -// } -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// _, err := scanner.CreatorUpdater.UpdatePartial(ctx, s.ID, scenePartial) -// return err -// }); err != nil { -// return nil, err -// } - -// scanner.makeScreenshots(path, nil, sceneHash) -// scanner.PluginCache.ExecutePostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil) -// } -// } else { -// logger.Infof("%s doesn't exist. Creating new item...", path) -// currentTime := time.Now() - -// videoFile, err := scanner.VideoFileCreator.NewVideoFile(path) -// if err != nil { -// return nil, err -// } - -// title := filepath.Base(path) -// if scanner.StripFileExtension { -// title = stripExtension(title) -// } - -// if scanner.UseFileMetadata && videoFile.Title != "" { -// title = videoFile.Title -// } - -// newScene := models.Scene{ -// Path: path, -// FileModTime: &scanned.FileModTime, -// Title: title, -// CreatedAt: currentTime, -// UpdatedAt: currentTime, -// Interactive: interactive, -// } - -// if checksum != "" { -// newScene.Checksum = &checksum -// } -// if oshash != "" { -// newScene.OSHash = &oshash -// } - -// if err := videoFileToScene(&newScene, videoFile); err != nil { -// return nil, err -// } - -// if scanner.UseFileMetadata { -// newScene.Details = videoFile.Comment -// d := models.SQLiteDate{} -// _ = d.Scan(videoFile.CreationTime) -// newScene.Date = d.DatePtr() -// } - -// if err := txn.WithTxn(ctx, scanner.TxnManager, func(ctx context.Context) error { -// return scanner.CreatorUpdater.Create(ctx, &newScene) -// }); err != nil { -// return nil, err -// } - -// retScene = &newScene - -// scanner.makeScreenshots(path, videoFile, sceneHash) -// scanner.PluginCache.ExecutePostHooks(ctx, retScene.ID, plugin.SceneCreatePost, nil, nil) -// } - -// return retScene, nil -// } - -// func stripExtension(path string) string { -// ext := filepath.Ext(path) -// return strings.TrimSuffix(path, ext) -// } - -// func videoFileToScene(s *models.Scene, videoFile *ffmpeg.VideoFile) error { -// container, err := ffmpeg.MatchContainer(videoFile.Container, s.Path) -// if err != nil { -// return fmt.Errorf("matching container: %w", err) -// } - -// s.Duration = &videoFile.Duration -// s.VideoCodec = &videoFile.VideoCodec -// s.AudioCodec = &videoFile.AudioCodec -// containerStr := string(container) -// s.Format = &containerStr -// s.Width = &videoFile.Width -// s.Height = &videoFile.Height -// s.Framerate = &videoFile.FrameRate -// s.Bitrate = &videoFile.Bitrate -// size := strconv.FormatInt(videoFile.Size, 10) -// s.Size = &size - -// return nil -// } - -// func (h *ScanHandler) makeScreenshots(ctx context.Context, scene *models.Scene, f *file.VideoFile) { -// checksum := scene.GetHash() -// thumbPath := h.Paths.Scene.GetThumbnailScreenshotPath(checksum) -// normalPath := h.Paths.Scene.GetScreenshotPath(checksum) - -// thumbExists, _ := fsutil.FileExists(thumbPath) -// normalExists, _ := fsutil.FileExists(normalPath) - -// if thumbExists && normalExists { -// return -// } - -// if !thumbExists { -// logger.Debugf("Creating thumbnail for %s", f.Path) -// if err := h.Screenshotter.GenerateThumbnail(ctx, probeResult, checksum); err != nil { -// logger.Errorf("Error creating thumbnail for %s: %v", err) -// } -// } - -// if !normalExists { -// logger.Debugf("Creating screenshot for %s", f.Path) -// if err := h.Screenshotter.GenerateScreenshot(ctx, probeResult, checksum); err != nil { -// logger.Errorf("Error creating screenshot for %s: %v", err) -// } -// } -// } - -// func getInteractive(path string) bool { -// _, err := os.Stat(GetFunscriptPath(path)) -// return err == nil -// } diff --git a/pkg/utils/strings_test.go b/pkg/utils/strings_test.go new file mode 100644 index 000000000..92af02fdc --- /dev/null +++ b/pkg/utils/strings_test.go @@ -0,0 +1,12 @@ +package utils + +import "fmt" + +func ExampleStrFormat() { + fmt.Println(StrFormat("{foo} bar {baz}", StrFormatMap{ + "foo": "bar", + "baz": "abc", + })) + // Output: + // bar bar abc +} diff --git a/ui/v2.5/src/docs/en/Changelog/v0170.md b/ui/v2.5/src/docs/en/Changelog/v0170.md index 34024893d..97befa26c 100644 --- a/ui/v2.5/src/docs/en/Changelog/v0170.md +++ b/ui/v2.5/src/docs/en/Changelog/v0170.md @@ -1,13 +1,7 @@ -### **Warning:** this is an experimental release. Use at your own risk! - -Please ensure you back up your **entire** stash system before using this build. This build should be considered experimental. - After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored. -Please report all issues to the following Github issue: https://github.com/stashapp/stash/issues/2737 - ### 💥 Known issues and other changes -* Missing covers are not currently regenerated. Need to consider further, especially around scene cover redesign. +* Missing covers are not currently regenerated. * Import/export schema has changed and is incompatible with the previous version. ### ✨ New Features diff --git a/ui/v2.5/src/docs/en/MigrationNotes/32.md b/ui/v2.5/src/docs/en/MigrationNotes/32.md index a3c37a13b..f45c8f9b2 100644 --- a/ui/v2.5/src/docs/en/MigrationNotes/32.md +++ b/ui/v2.5/src/docs/en/MigrationNotes/32.md @@ -1,7 +1,3 @@ -### **Warning:** this is an experimental release. Use at your own risk! - -Please ensure you back up your **entire** stash system before using this build. This build should be considered experimental. - This migration significantly changes the way that stash stores information about your files. This migration is not reversible. After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored. diff --git a/ui/v2.5/src/docs/en/ReleaseNotes/20220826.md b/ui/v2.5/src/docs/en/ReleaseNotes/20220826.md deleted file mode 100644 index bd829d2ed..000000000 --- a/ui/v2.5/src/docs/en/ReleaseNotes/20220826.md +++ /dev/null @@ -1 +0,0 @@ -### **Warning:** Windows users will need to re-migrate from schema version 31 if they are upgrading from an older `files-refactor` build, or manually change the path separators in the `folders` table from `/` to `\` in the database. diff --git a/ui/v2.5/src/docs/en/ReleaseNotes/index.ts b/ui/v2.5/src/docs/en/ReleaseNotes/index.ts index f4b0f5111..8a1e5000e 100644 --- a/ui/v2.5/src/docs/en/ReleaseNotes/index.ts +++ b/ui/v2.5/src/docs/en/ReleaseNotes/index.ts @@ -1,5 +1,4 @@ import v0170 from "./v0170.md"; -import r20220826 from "./20220826.md"; export type Module = typeof v0170; @@ -11,11 +10,7 @@ interface IReleaseNotes { export const releaseNotes: IReleaseNotes[] = [ { - date: 20220801, + date: 20220906, content: v0170, }, - { - date: 20220826, - content: r20220826, - }, ]; diff --git a/ui/v2.5/src/docs/en/ReleaseNotes/v0170.md b/ui/v2.5/src/docs/en/ReleaseNotes/v0170.md index 9db6bbfb2..773f94da4 100644 --- a/ui/v2.5/src/docs/en/ReleaseNotes/v0170.md +++ b/ui/v2.5/src/docs/en/ReleaseNotes/v0170.md @@ -1,19 +1,10 @@ -### **Warning:** this is an experimental release. Use at your own risk! - -Please ensure you back up your **entire** stash system before using this build. This build should be considered experimental. - After migrating, please run a scan on your entire library to populate missing data, and to ingest identical files which were previously ignored. -Please report all issues to the following Github issue: https://github.com/stashapp/stash/issues/2737 - -### **Warning:** if you are upgrading from an older `files-refactor` build, you will need to re-migrate your system from a schema version 31 database. - ### 💥 Known issues * Missing covers are not currently regenerated. Need to consider further, especially around scene cover redesign. ### Other changes: * Import/export schema has changed and is incompatible with the previous version. -* Added support for filtering and sorting by file count. ([#2744](https://github.com/stashapp/stash/pull/2744)) * Changelog has been moved from the stats page to a section in the Settings page. * Object titles are now displayed as the file basename if the title is not explicitly set. The `Don't include file extension as part of the title` scan flag is no longer supported. * `Set name, date, details from embedded file metadata` scan flag is no longer supported. This functionality may be implemented as a built-in scraper in the future. \ No newline at end of file