Rooba
/
stash
mirror of https://github.com/stashapp/stash.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Various bug fixes (#2938) 

* Don't recalculate MD5 if not enabled

Remove MD5 if oshash has changed and MD5 was not calculated.

* Fix panic in paged DLNA
* Prevent identical hashes in stash-box drafts
This commit is contained in:
WithoutPants 2022-09-21 15:39:41 +10:00 committed by GitHub
parent cffcd9f4b8
commit b74428cb42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 120 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/dlna/cds.go
View File

@ -493,7 +493,7 @@ func (me *contentDirectoryService) getPageVideos(sceneFilter *models.SceneFilter
}
var err error
objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, page, host)
objs, err = pager.getPageVideos(ctx, me.repository.SceneFinder, me.repository.FileFinder, page, host)
if err != nil {
return err
}

7
internal/dlna/paging.go
View File

@ -6,6 +6,7 @@ import (
"math"
"strconv"
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scene"
)
@ -59,7 +60,7 @@ func (p *scenePager) getPages(ctx context.Context, r scene.Queryer, total int) (
return objs, nil
}
func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int, host string) ([]interface{}, error) {
func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, f file.Finder, page int, host string) ([]interface{}, error) {
var objs []interface{}
sort := "title"
@ -75,6 +76,10 @@ func (p *scenePager) getPageVideos(ctx context.Context, r SceneFinder, page int,
}
for _, s := range scenes {
if err := s.LoadPrimaryFile(ctx, f); err != nil {
return nil, err
}
objs = append(objs, sceneToContainer(s, p.parentID, host))
}

3
internal/manager/fingerprint.go
View File

@ -84,8 +84,7 @@ func (c *fingerprintCalculator) CalculateFingerprints(f *file.BaseFile, o file.O
ret = append(ret, *fp)
// only calculate MD5 if enabled in config
// always re-calculate MD5 if the file already has it
calculateMD5 = c.Config.IsCalculateMD5() || f.Fingerprints.For(file.FingerprintTypeMD5) != nil
calculateMD5 = c.Config.IsCalculateMD5()
}
if calculateMD5 {

4
pkg/file/file.go
View File

@ -50,7 +50,7 @@ func (e *DirEntry) info(fs FS, path string) (fs.FileInfo, error) {
// File represents a file in the file system.
type File interface {
Base() *BaseFile
SetFingerprints(fp []Fingerprint)
SetFingerprints(fp Fingerprints)
Open(fs FS) (io.ReadCloser, error)
}
@ -76,7 +76,7 @@ type BaseFile struct {
// SetFingerprints sets the fingerprints of the file.
// If a fingerprint of the same type already exists, it is overwritten.
func (f *BaseFile) SetFingerprints(fp []Fingerprint) {
func (f *BaseFile) SetFingerprints(fp Fingerprints) {
for _, v := range fp {
f.SetFingerprint(v)
}

12
pkg/file/fingerprint.go
View File

@ -14,6 +14,18 @@ type Fingerprint struct {
type Fingerprints []Fingerprint
func (f *Fingerprints) Remove(type_ string) {
var ret Fingerprints
for _, ff := range *f {
if ff.Type != type_ {
ret = append(ret, ff)
}
}
*f = ret
}
func (f Fingerprints) Equals(other Fingerprints) bool {
if len(f) != len(other) {
return false

138
pkg/file/scan.go
View File

@ -885,59 +885,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
updated := !fileModTime.Equal(base.ModTime)
if !updated {
var err error
isMissingMetdata := s.isMissingMetadata(existing)
// set missing information
if isMissingMetdata {
existing, err = s.setMissingMetadata(ctx, f, existing)
if err != nil {
return nil, err
}
}
// calculate missing fingerprints
existing, err = s.setMissingFingerprints(ctx, f, existing)
if err != nil {
return nil, err
}
handlerRequired := false
if err := s.withDB(ctx, func(ctx context.Context) error {
// check if the handler needs to be run
handlerRequired = s.isHandlerRequired(ctx, existing)
return nil
}); err != nil {
return nil, err
}
if !handlerRequired {
// if this file is a zip file, then we need to rescan the contents
// as well. We do this by returning the file, instead of nil.
if isMissingMetdata {
return existing, nil
}
return nil, nil
}
if err := s.withTxn(ctx, func(ctx context.Context) error {
if err := s.fireHandlers(ctx, existing); err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// if this file is a zip file, then we need to rescan the contents
// as well. We do this by returning the file, instead of nil.
if isMissingMetdata {
return existing, nil
}
return nil, nil
return s.onUnchangedFile(ctx, f, existing)
}
logger.Infof("%s has been updated: rescanning", path)
@ -952,6 +900,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
return nil, err
}
s.removeOutdatedFingerprints(existing, fp)
existing.SetFingerprints(fp)
existing, err = s.fireDecorators(ctx, f.fs, existing)
@ -976,3 +925,86 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File)
return existing, nil
}
func (s *scanJob) removeOutdatedFingerprints(existing File, fp Fingerprints) {
// HACK - if no MD5 fingerprint was returned, and the oshash is changed
// then remove the MD5 fingerprint
oshash := fp.For(FingerprintTypeOshash)
if oshash == nil {
return
}
existingOshash := existing.Base().Fingerprints.For(FingerprintTypeOshash)
if existingOshash == nil || *existingOshash == *oshash {
// missing oshash or same oshash - nothing to do
return
}
md5 := fp.For(FingerprintTypeMD5)
if md5 != nil {
// nothing to do
return
}
// oshash has changed, MD5 is missing - remove MD5 from the existing fingerprints
logger.Infof("Removing outdated checksum from %s", existing.Base().Path)
existing.Base().Fingerprints.Remove(FingerprintTypeMD5)
}
// returns a file only if it was updated
func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File) (File, error) {
var err error
isMissingMetdata := s.isMissingMetadata(existing)
// set missing information
if isMissingMetdata {
existing, err = s.setMissingMetadata(ctx, f, existing)
if err != nil {
return nil, err
}
}
// calculate missing fingerprints
existing, err = s.setMissingFingerprints(ctx, f, existing)
if err != nil {
return nil, err
}
handlerRequired := false
if err := s.withDB(ctx, func(ctx context.Context) error {
// check if the handler needs to be run
handlerRequired = s.isHandlerRequired(ctx, existing)
return nil
}); err != nil {
return nil, err
}
if !handlerRequired {
// if this file is a zip file, then we need to rescan the contents
// as well. We do this by returning the file, instead of nil.
if isMissingMetdata {
return existing, nil
}
return nil, nil
}
if err := s.withTxn(ctx, func(ctx context.Context) error {
if err := s.fireHandlers(ctx, existing); err != nil {
return err
}
return nil
}); err != nil {
return nil, err
}
// if this file is a zip file, then we need to rescan the contents
// as well. We do this by returning the file, instead of nil.
if isMissingMetdata {
return existing, nil
}
return nil, nil
}

16
pkg/scraper/stashbox/stash_box.go
View File

@ -758,6 +758,16 @@ func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
return c.client.Me(ctx)
}
func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput {
for _, vv := range v {
if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash {
return v
}
}
return append(v, toAdd)
}
func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpoint string, imagePath string) (*string, error) {
draft := graphql.SceneDraftInput{}
var image io.Reader
@ -820,7 +830,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
Algorithm: graphql.FingerprintAlgorithmOshash,
Duration: int(duration),
}
fingerprints = append(fingerprints, &fingerprint)
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
}
if checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5); checksum != "" {
@ -829,7 +839,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
Algorithm: graphql.FingerprintAlgorithmMd5,
Duration: int(duration),
}
fingerprints = append(fingerprints, &fingerprint)
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
}
if phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash); phash != 0 {
@ -838,7 +848,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
Algorithm: graphql.FingerprintAlgorithmPhash,
Duration: int(duration),
}
fingerprints = append(fingerprints, &fingerprint)
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
}
}
}