diff --git a/internal/dlna/cds.go b/internal/dlna/cds.go index eb8b725c5..eedeef082 100644 --- a/internal/dlna/cds.go +++ b/internal/dlna/cds.go @@ -493,7 +493,7 @@ func (me *contentDirectoryService) getPageVideos(sceneFilter *models.SceneFilter } var err error - objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, page, host) + objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, me.repository.FileFinder, page, host) if err != nil { return err } diff --git a/internal/dlna/paging.go b/internal/dlna/paging.go index e5f65f96a..d5643da88 100644 --- a/internal/dlna/paging.go +++ b/internal/dlna/paging.go @@ -6,6 +6,7 @@ import ( "math" "strconv" + "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" ) @@ -59,7 +60,7 @@ func (p *scenePager) getPages(ctx context.Context, r scene.Queryer, total int) ( return objs, nil } -func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int, host string) ([]interface{}, error) { +func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, f file.Finder, page int, host string) ([]interface{}, error) { var objs []interface{} sort := "title" @@ -75,6 +76,10 @@ func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int, } for _, s := range scenes { + if err := s.LoadPrimaryFile(ctx, f); err != nil { + return nil, err + } + objs = append(objs, sceneToContainer(s, p.parentID, host)) } diff --git a/internal/manager/fingerprint.go b/internal/manager/fingerprint.go index 291536ed1..5c2c66352 100644 --- a/internal/manager/fingerprint.go +++ b/internal/manager/fingerprint.go @@ -84,8 +84,7 @@ func (c *fingerprintCalculator) CalculateFingerprints(f *file.BaseFile, o file.O ret = append(ret, *fp) // only calculate MD5 if enabled in config - // always re-calculate MD5 if the file already has it - calculateMD5 = c.Config.IsCalculateMD5() || f.Fingerprints.For(file.FingerprintTypeMD5) != nil + calculateMD5 = c.Config.IsCalculateMD5() } if calculateMD5 { diff --git a/pkg/file/file.go b/pkg/file/file.go index 611ceee50..3323bbd8e 100644 --- a/pkg/file/file.go +++ b/pkg/file/file.go @@ -50,7 +50,7 @@ func (e *DirEntry) info(fs FS, path string) (fs.FileInfo, error) { // File represents a file in the file system. type File interface { Base() *BaseFile - SetFingerprints(fp []Fingerprint) + SetFingerprints(fp Fingerprints) Open(fs FS) (io.ReadCloser, error) } @@ -76,7 +76,7 @@ type BaseFile struct { // SetFingerprints sets the fingerprints of the file. // If a fingerprint of the same type already exists, it is overwritten. -func (f *BaseFile) SetFingerprints(fp []Fingerprint) { +func (f *BaseFile) SetFingerprints(fp Fingerprints) { for _, v := range fp { f.SetFingerprint(v) } diff --git a/pkg/file/fingerprint.go b/pkg/file/fingerprint.go index 12a39fc9d..b5a8f9b5b 100644 --- a/pkg/file/fingerprint.go +++ b/pkg/file/fingerprint.go @@ -14,6 +14,18 @@ type Fingerprint struct { type Fingerprints []Fingerprint +func (f *Fingerprints) Remove(type_ string) { + var ret Fingerprints + + for _, ff := range *f { + if ff.Type != type_ { + ret = append(ret, ff) + } + } + + *f = ret +} + func (f Fingerprints) Equals(other Fingerprints) bool { if len(f) != len(other) { return false diff --git a/pkg/file/scan.go b/pkg/file/scan.go index 04cc389f2..ceb1f54dc 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -885,59 +885,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File) updated := !fileModTime.Equal(base.ModTime) if !updated { - var err error - - isMissingMetdata := s.isMissingMetadata(existing) - // set missing information - if isMissingMetdata { - existing, err = s.setMissingMetadata(ctx, f, existing) - if err != nil { - return nil, err - } - } - - // calculate missing fingerprints - existing, err = s.setMissingFingerprints(ctx, f, existing) - if err != nil { - return nil, err - } - - handlerRequired := false - if err := s.withDB(ctx, func(ctx context.Context) error { - // check if the handler needs to be run - handlerRequired = s.isHandlerRequired(ctx, existing) - return nil - }); err != nil { - return nil, err - } - - if !handlerRequired { - // if this file is a zip file, then we need to rescan the contents - // as well. We do this by returning the file, instead of nil. - if isMissingMetdata { - return existing, nil - } - - return nil, nil - } - - if err := s.withTxn(ctx, func(ctx context.Context) error { - if err := s.fireHandlers(ctx, existing); err != nil { - return err - } - - return nil - }); err != nil { - return nil, err - } - - // if this file is a zip file, then we need to rescan the contents - // as well. We do this by returning the file, instead of nil. - if isMissingMetdata { - return existing, nil - } - - return nil, nil + return s.onUnchangedFile(ctx, f, existing) } logger.Infof("%s has been updated: rescanning", path) @@ -952,6 +900,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File) return nil, err } + s.removeOutdatedFingerprints(existing, fp) existing.SetFingerprints(fp) existing, err = s.fireDecorators(ctx, f.fs, existing) @@ -976,3 +925,86 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File) return existing, nil } + +func (s *scanJob) removeOutdatedFingerprints(existing File, fp Fingerprints) { + // HACK - if no MD5 fingerprint was returned, and the oshash is changed + // then remove the MD5 fingerprint + oshash := fp.For(FingerprintTypeOshash) + if oshash == nil { + return + } + + existingOshash := existing.Base().Fingerprints.For(FingerprintTypeOshash) + if existingOshash == nil || *existingOshash == *oshash { + // missing oshash or same oshash - nothing to do + return + } + + md5 := fp.For(FingerprintTypeMD5) + + if md5 != nil { + // nothing to do + return + } + + // oshash has changed, MD5 is missing - remove MD5 from the existing fingerprints + logger.Infof("Removing outdated checksum from %s", existing.Base().Path) + existing.Base().Fingerprints.Remove(FingerprintTypeMD5) +} + +// returns a file only if it was updated +func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File) (File, error) { + var err error + + isMissingMetdata := s.isMissingMetadata(existing) + // set missing information + if isMissingMetdata { + existing, err = s.setMissingMetadata(ctx, f, existing) + if err != nil { + return nil, err + } + } + + // calculate missing fingerprints + existing, err = s.setMissingFingerprints(ctx, f, existing) + if err != nil { + return nil, err + } + + handlerRequired := false + if err := s.withDB(ctx, func(ctx context.Context) error { + // check if the handler needs to be run + handlerRequired = s.isHandlerRequired(ctx, existing) + return nil + }); err != nil { + return nil, err + } + + if !handlerRequired { + // if this file is a zip file, then we need to rescan the contents + // as well. We do this by returning the file, instead of nil. + if isMissingMetdata { + return existing, nil + } + + return nil, nil + } + + if err := s.withTxn(ctx, func(ctx context.Context) error { + if err := s.fireHandlers(ctx, existing); err != nil { + return err + } + + return nil + }); err != nil { + return nil, err + } + + // if this file is a zip file, then we need to rescan the contents + // as well. We do this by returning the file, instead of nil. + if isMissingMetdata { + return existing, nil + } + + return nil, nil +} diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 5133bb172..6b20d9342 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -758,6 +758,16 @@ func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) { return c.client.Me(ctx) } +func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput { + for _, vv := range v { + if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash { + return v + } + } + + return append(v, toAdd) +} + func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpoint string, imagePath string) (*string, error) { draft := graphql.SceneDraftInput{} var image io.Reader @@ -820,7 +830,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo Algorithm: graphql.FingerprintAlgorithmOshash, Duration: int(duration), } - fingerprints = append(fingerprints, &fingerprint) + fingerprints = appendFingerprintUnique(fingerprints, &fingerprint) } if checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5); checksum != "" { @@ -829,7 +839,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo Algorithm: graphql.FingerprintAlgorithmMd5, Duration: int(duration), } - fingerprints = append(fingerprints, &fingerprint) + fingerprints = appendFingerprintUnique(fingerprints, &fingerprint) } if phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash); phash != 0 { @@ -838,7 +848,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo Algorithm: graphql.FingerprintAlgorithmPhash, Duration: int(duration), } - fingerprints = append(fingerprints, &fingerprint) + fingerprints = appendFingerprintUnique(fingerprints, &fingerprint) } } }