Various bug fixes (#2938)

* Don't recalculate MD5 if not enabled

Remove MD5 if oshash has changed and MD5 was not calculated.

* Fix panic in paged DLNA
* Prevent identical hashes in stash-box drafts

internal/dlna/cds.go

@@ -493,7 +493,7 @@ func (me *contentDirectoryService) getPageVideos(sceneFilter *models.SceneFilter
 		}
 
 		var err error
-		objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, page, host)
+		objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, me.repository.FileFinder, page, host)
 		if err != nil {
 			return err
 		}

internal/dlna/paging.go

@@ -6,6 +6,7 @@ import (
 	"math"
 	"strconv"
 
+	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 )
@@ -59,7 +60,7 @@ func (p *scenePager) getPages(ctx context.Context, r scene.Queryer, total int) (
 	return objs, nil
 }
 
-func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int, host string) ([]interface{}, error) {
+func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, f file.Finder, page int, host string) ([]interface{}, error) {
 	var objs []interface{}
 
 	sort := "title"
@@ -75,6 +76,10 @@ func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int,
 	}
 
 	for _, s := range scenes {
+		if err := s.LoadPrimaryFile(ctx, f); err != nil {
+			return nil, err
+		}
+
 		objs = append(objs, sceneToContainer(s, p.parentID, host))
 	}
 

internal/manager/fingerprint.go

@@ -84,8 +84,7 @@ func (c *fingerprintCalculator) CalculateFingerprints(f *file.BaseFile, o file.O
 		ret = append(ret, *fp)
 
 		// only calculate MD5 if enabled in config
-		// always re-calculate MD5 if the file already has it
+		calculateMD5 = c.Config.IsCalculateMD5()
-		calculateMD5 = c.Config.IsCalculateMD5() || f.Fingerprints.For(file.FingerprintTypeMD5) != nil
 	}
 
 	if calculateMD5 {

pkg/file/file.go

@@ -50,7 +50,7 @@ func (e *DirEntry) info(fs FS, path string) (fs.FileInfo, error) {
 // File represents a file in the file system.
 type File interface {
 	Base() *BaseFile
-	SetFingerprints(fp []Fingerprint)
+	SetFingerprints(fp Fingerprints)
 	Open(fs FS) (io.ReadCloser, error)
 }
 
@@ -76,7 +76,7 @@ type BaseFile struct {
 
 // SetFingerprints sets the fingerprints of the file.
 // If a fingerprint of the same type already exists, it is overwritten.
-func (f *BaseFile) SetFingerprints(fp []Fingerprint) {
+func (f *BaseFile) SetFingerprints(fp Fingerprints) {
 	for _, v := range fp {
 		f.SetFingerprint(v)
 	}

pkg/file/fingerprint.go

@@ -14,6 +14,18 @@ type Fingerprint struct {
 
 type Fingerprints []Fingerprint
 
+func (f *Fingerprints) Remove(type_ string) {
+	var ret Fingerprints
+
+	for _, ff := range *f {
+		if ff.Type != type_ {
+			ret = append(ret, ff)
+		}
+	}
+
+	*f = ret
+}
+
 func (f Fingerprints) Equals(other Fingerprints) bool {
 	if len(f) != len(other) {
 		return false

pkg/file/scan.go

@@ -885,59 +885,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
 	updated := !fileModTime.Equal(base.ModTime)
 
 	if !updated {
-		var err error
+		return s.onUnchangedFile(ctx, f, existing)
-
-		isMissingMetdata := s.isMissingMetadata(existing)
-		// set missing information
-		if isMissingMetdata {
-			existing, err = s.setMissingMetadata(ctx, f, existing)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		// calculate missing fingerprints
-		existing, err = s.setMissingFingerprints(ctx, f, existing)
-		if err != nil {
-			return nil, err
-		}
-
-		handlerRequired := false
-		if err := s.withDB(ctx, func(ctx context.Context) error {
-			// check if the handler needs to be run
-			handlerRequired = s.isHandlerRequired(ctx, existing)
-			return nil
-		}); err != nil {
-			return nil, err
-		}
-
-		if !handlerRequired {
-			// if this file is a zip file, then we need to rescan the contents
-			// as well. We do this by returning the file, instead of nil.
-			if isMissingMetdata {
-				return existing, nil
-			}
-
-			return nil, nil
-		}
-
-		if err := s.withTxn(ctx, func(ctx context.Context) error {
-			if err := s.fireHandlers(ctx, existing); err != nil {
-				return err
-			}
-
-			return nil
-		}); err != nil {
-			return nil, err
-		}
-
-		// if this file is a zip file, then we need to rescan the contents
-		// as well. We do this by returning the file, instead of nil.
-		if isMissingMetdata {
-			return existing, nil
-		}
-
-		return nil, nil
 	}
 
 	logger.Infof("%s has been updated: rescanning", path)
@@ -952,6 +900,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
 		return nil, err
 	}
 
+	s.removeOutdatedFingerprints(existing, fp)
 	existing.SetFingerprints(fp)
 
 	existing, err = s.fireDecorators(ctx, f.fs, existing)
@@ -976,3 +925,86 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
 
 	return existing, nil
 }
+
+func (s *scanJob) removeOutdatedFingerprints(existing File, fp Fingerprints) {
+	// HACK - if no MD5 fingerprint was returned, and the oshash is changed
+	// then remove the MD5 fingerprint
+	oshash := fp.For(FingerprintTypeOshash)
+	if oshash == nil {
+		return
+	}
+
+	existingOshash := existing.Base().Fingerprints.For(FingerprintTypeOshash)
+	if existingOshash == nil || *existingOshash == *oshash {
+		// missing oshash or same oshash - nothing to do
+		return
+	}
+
+	md5 := fp.For(FingerprintTypeMD5)
+
+	if md5 != nil {
+		// nothing to do
+		return
+	}
+
+	// oshash has changed, MD5 is missing - remove MD5 from the existing fingerprints
+	logger.Infof("Removing outdated checksum from %s", existing.Base().Path)
+	existing.Base().Fingerprints.Remove(FingerprintTypeMD5)
+}
+
+// returns a file only if it was updated
+func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File) (File, error) {
+	var err error
+
+	isMissingMetdata := s.isMissingMetadata(existing)
+	// set missing information
+	if isMissingMetdata {
+		existing, err = s.setMissingMetadata(ctx, f, existing)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// calculate missing fingerprints
+	existing, err = s.setMissingFingerprints(ctx, f, existing)
+	if err != nil {
+		return nil, err
+	}
+
+	handlerRequired := false
+	if err := s.withDB(ctx, func(ctx context.Context) error {
+		// check if the handler needs to be run
+		handlerRequired = s.isHandlerRequired(ctx, existing)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	if !handlerRequired {
+		// if this file is a zip file, then we need to rescan the contents
+		// as well. We do this by returning the file, instead of nil.
+		if isMissingMetdata {
+			return existing, nil
+		}
+
+		return nil, nil
+	}
+
+	if err := s.withTxn(ctx, func(ctx context.Context) error {
+		if err := s.fireHandlers(ctx, existing); err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	// if this file is a zip file, then we need to rescan the contents
+	// as well. We do this by returning the file, instead of nil.
+	if isMissingMetdata {
+		return existing, nil
+	}
+
+	return nil, nil
+}

pkg/scraper/stashbox/stash_box.go

@@ -758,6 +758,16 @@ func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
 	return c.client.Me(ctx)
 }
 
+func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput {
+	for _, vv := range v {
+		if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash {
+			return v
+		}
+	}
+
+	return append(v, toAdd)
+}
+
 func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpoint string, imagePath string) (*string, error) {
 	draft := graphql.SceneDraftInput{}
 	var image io.Reader
@@ -820,7 +830,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 					Algorithm: graphql.FingerprintAlgorithmOshash,
 					Duration:  int(duration),
 				}
-				fingerprints = append(fingerprints, &fingerprint)
+				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
 			}
 
 			if checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5); checksum != "" {
@@ -829,7 +839,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 					Algorithm: graphql.FingerprintAlgorithmMd5,
 					Duration:  int(duration),
 				}
-				fingerprints = append(fingerprints, &fingerprint)
+				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
 			}
 
 			if phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash); phash != 0 {
@@ -838,7 +848,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 					Algorithm: graphql.FingerprintAlgorithmPhash,
 					Duration:  int(duration),
 				}
-				fingerprints = append(fingerprints, &fingerprint)
+				fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
 			}
 		}
 	}