diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e408e38af..eced257b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,13 +73,8 @@ jobs: # Static validation happens in the linter workflow in parallel to this workflow # Run Dynamic validation here, to make sure we pass all the projects integration tests - # - # create UI file so that the embed doesn't fail - name: Test Backend - run: | - mkdir -p ui/v2.5/build - touch ui/v2.5/build/index.html - docker exec -t build /bin/bash -c "make it" + run: docker exec -t build /bin/bash -c "make it" - name: Build UI # skip UI build for pull requests if UI is unchanged (UI was cached) diff --git a/.gitignore b/.gitignore index ec5e351a0..4002f4168 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,6 @@ node_modules *.db -stash +/stash dist .DS_Store diff --git a/Makefile b/Makefile index f369b490f..6f28ac4ed 100644 --- a/Makefile +++ b/Makefile @@ -50,14 +50,17 @@ ifndef OFFICIAL_BUILD $(eval OFFICIAL_BUILD := false) endif -build: pre-build ifdef IS_WIN_OS +ifndef SUPPRESS_WINDOWSGUI PLATFORM_SPECIFIC_LDFLAGS := -H windowsgui endif +endif + +build: pre-build build: - $(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/pkg/api.githash=$(GITHASH)') - $(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/manager/config.officialBuild=$(OFFICIAL_BUILD)') - go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" + $(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/internal/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/internal/api.githash=$(GITHASH)') + $(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/internal/manager/config.officialBuild=$(OFFICIAL_BUILD)') + go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS) $(PLATFORM_SPECIFIC_LDFLAGS)" ./cmd/stash # strips debug symbols from the release build build-release: EXTRA_LDFLAGS := -s -w @@ -140,6 +143,12 @@ cross-compile-all: make cross-compile-linux-arm32v7 make cross-compile-linux-arm32v6 +# ensures a file is present in ui/v2.5/build since this is required +# for the embedded ui library +touch-ui: + @mkdir -p ui/v2.5/build + @touch ui/v2.5/build/index.html + # Regenerates GraphQL files generate: generate-backend generate-frontend @@ -148,8 +157,8 @@ generate-frontend: cd ui/v2.5 && yarn run gqlgen .PHONY: generate-backend -generate-backend: - go generate -mod=vendor +generate-backend: touch-ui + go generate -mod=vendor ./cmd/stash # Regenerates stash-box client files .PHONY: generate-stash-box-client diff --git a/main.go b/cmd/stash/main.go similarity index 78% rename from main.go rename to cmd/stash/main.go index fa9e4d790..b85bbfe98 100644 --- a/main.go +++ b/cmd/stash/main.go @@ -2,7 +2,6 @@ package main import ( - "embed" "fmt" "os" "os/signal" @@ -10,19 +9,13 @@ import ( "syscall" "github.com/apenwarr/fixconsole" - "github.com/stashapp/stash/pkg/api" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/api" + "github.com/stashapp/stash/internal/manager" _ "github.com/golang-migrate/migrate/v4/database/sqlite3" _ "github.com/golang-migrate/migrate/v4/source/file" ) -//go:embed ui/v2.5/build -var uiBox embed.FS - -//go:embed ui/login -var loginUIBox embed.FS - func init() { // On Windows, attach to parent shell err := fixconsole.FixConsoleIfNeeded() @@ -33,7 +26,7 @@ func init() { func main() { manager.Initialize() - api.Start(uiBox, loginUIBox) + api.Start() blockForever() diff --git a/main_test.go b/cmd/stash/main_test.go similarity index 100% rename from main_test.go rename to cmd/stash/main_test.go diff --git a/gqlgen.yml b/gqlgen.yml index 5cd2c0915..7ddc3be69 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -8,7 +8,7 @@ exec: model: filename: pkg/models/generated_models.go resolver: - filename: pkg/api/resolver.go + filename: internal/api/resolver.go type: Resolver struct_tag: gqlgen diff --git a/pkg/api/authentication.go b/internal/api/authentication.go similarity index 97% rename from pkg/api/authentication.go rename to internal/api/authentication.go index e5358affe..a9770c203 100644 --- a/pkg/api/authentication.go +++ b/internal/api/authentication.go @@ -7,9 +7,9 @@ import ( "net/url" "strings" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/session" ) diff --git a/pkg/utils/byterange.go b/internal/api/byterange.go similarity index 77% rename from pkg/utils/byterange.go rename to internal/api/byterange.go index 5fb031ea7..564444579 100644 --- a/pkg/utils/byterange.go +++ b/internal/api/byterange.go @@ -1,22 +1,22 @@ -package utils +package api import ( "strconv" "strings" ) -type ByteRange struct { +type byteRange struct { Start int64 End *int64 RawString string } -func CreateByteRange(s string) ByteRange { +func createByteRange(s string) byteRange { // strip bytes= r := strings.TrimPrefix(s, "bytes=") e := strings.Split(r, "-") - ret := ByteRange{ + ret := byteRange{ RawString: s, } if len(e) > 0 { @@ -30,7 +30,7 @@ func CreateByteRange(s string) ByteRange { return ret } -func (r ByteRange) ToHeaderValue(fileLength int64) string { +func (r byteRange) toHeaderValue(fileLength int64) string { if r.End == nil { return "" } @@ -38,7 +38,7 @@ func (r ByteRange) ToHeaderValue(fileLength int64) string { return "bytes " + strconv.FormatInt(r.Start, 10) + "-" + strconv.FormatInt(end, 10) + "/" + strconv.FormatInt(fileLength, 10) } -func (r ByteRange) Apply(bytes []byte) []byte { +func (r byteRange) apply(bytes []byte) []byte { if r.End == nil { return bytes[r.Start:] } diff --git a/pkg/api/changeset_translator.go b/internal/api/changeset_translator.go similarity index 100% rename from pkg/api/changeset_translator.go rename to internal/api/changeset_translator.go diff --git a/pkg/api/check_version.go b/internal/api/check_version.go similarity index 100% rename from pkg/api/check_version.go rename to internal/api/check_version.go diff --git a/pkg/api/context_keys.go b/internal/api/context_keys.go similarity index 100% rename from pkg/api/context_keys.go rename to internal/api/context_keys.go diff --git a/internal/api/dir_list.go b/internal/api/dir_list.go new file mode 100644 index 000000000..e33c215a1 --- /dev/null +++ b/internal/api/dir_list.go @@ -0,0 +1,48 @@ +package api + +import ( + "io/fs" + "os" + "path/filepath" + + "golang.org/x/text/collate" +) + +type dirLister []fs.DirEntry + +func (s dirLister) Len() int { + return len(s) +} + +func (s dirLister) Swap(i, j int) { + s[j], s[i] = s[i], s[j] +} + +func (s dirLister) Bytes(i int) []byte { + return []byte(s[i].Name()) +} + +// listDir will return the contents of a given directory path as a string slice +func listDir(col *collate.Collator, path string) ([]string, error) { + var dirPaths []string + files, err := os.ReadDir(path) + if err != nil { + path = filepath.Dir(path) + files, err = os.ReadDir(path) + if err != nil { + return dirPaths, err + } + } + + if col != nil { + col.Sort(dirLister(files)) + } + + for _, file := range files { + if !file.IsDir() { + continue + } + dirPaths = append(dirPaths, filepath.Join(path, file.Name())) + } + return dirPaths, nil +} diff --git a/pkg/api/images.go b/internal/api/images.go similarity index 91% rename from pkg/api/images.go rename to internal/api/images.go index 41f18c025..7e6f8b402 100644 --- a/pkg/api/images.go +++ b/internal/api/images.go @@ -6,10 +6,10 @@ import ( "os" "strings" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/internal/static" + "github.com/stashapp/stash/pkg/hash" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/static" - "github.com/stashapp/stash/pkg/utils" ) type imageBox struct { @@ -102,7 +102,7 @@ func getRandomPerformerImageUsingName(name, gender, customPath string) ([]byte, } imageFiles := box.files - index := utils.IntFromString(name) % uint64(len(imageFiles)) + index := hash.IntFromString(name) % uint64(len(imageFiles)) img, err := box.box.Open(imageFiles[index]) if err != nil { return nil, err diff --git a/pkg/api/locale.go b/internal/api/locale.go similarity index 100% rename from pkg/api/locale.go rename to internal/api/locale.go diff --git a/pkg/api/resolver.go b/internal/api/resolver.go similarity index 99% rename from pkg/api/resolver.go rename to internal/api/resolver.go index bf588e93d..3dbdd9fa0 100644 --- a/pkg/api/resolver.go +++ b/internal/api/resolver.go @@ -6,8 +6,8 @@ import ( "sort" "strconv" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scraper" diff --git a/pkg/api/resolver_model_gallery.go b/internal/api/resolver_model_gallery.go similarity index 100% rename from pkg/api/resolver_model_gallery.go rename to internal/api/resolver_model_gallery.go diff --git a/pkg/api/resolver_model_image.go b/internal/api/resolver_model_image.go similarity index 98% rename from pkg/api/resolver_model_image.go rename to internal/api/resolver_model_image.go index 854f0e338..a77437dfb 100644 --- a/pkg/api/resolver_model_image.go +++ b/internal/api/resolver_model_image.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_model_movie.go b/internal/api/resolver_model_movie.go similarity index 98% rename from pkg/api/resolver_model_movie.go rename to internal/api/resolver_model_movie.go index 0b3597d49..57e979828 100644 --- a/pkg/api/resolver_model_movie.go +++ b/internal/api/resolver_model_movie.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/api/resolver_model_performer.go b/internal/api/resolver_model_performer.go similarity index 99% rename from pkg/api/resolver_model_performer.go rename to internal/api/resolver_model_performer.go index ea52873df..364c346ca 100644 --- a/pkg/api/resolver_model_performer.go +++ b/internal/api/resolver_model_performer.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" diff --git a/pkg/api/resolver_model_scene.go b/internal/api/resolver_model_scene.go similarity index 98% rename from pkg/api/resolver_model_scene.go rename to internal/api/resolver_model_scene.go index af836b561..90a925966 100644 --- a/pkg/api/resolver_model_scene.go +++ b/internal/api/resolver_model_scene.go @@ -4,8 +4,8 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/api/urlbuilders" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/api/resolver_model_scene_marker.go b/internal/api/resolver_model_scene_marker.go similarity index 97% rename from pkg/api/resolver_model_scene_marker.go rename to internal/api/resolver_model_scene_marker.go index 3a6ada5bc..64d418bd1 100644 --- a/pkg/api/resolver_model_scene_marker.go +++ b/internal/api/resolver_model_scene_marker.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_model_studio.go b/internal/api/resolver_model_studio.go similarity index 98% rename from pkg/api/resolver_model_studio.go rename to internal/api/resolver_model_studio.go index 32c7c5399..d2b6b44c1 100644 --- a/pkg/api/resolver_model_studio.go +++ b/internal/api/resolver_model_studio.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" diff --git a/pkg/api/resolver_model_tag.go b/internal/api/resolver_model_tag.go similarity index 98% rename from pkg/api/resolver_model_tag.go rename to internal/api/resolver_model_tag.go index 3b90463ca..cb406e5fc 100644 --- a/pkg/api/resolver_model_tag.go +++ b/internal/api/resolver_model_tag.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/api/urlbuilders" + "github.com/stashapp/stash/internal/api/urlbuilders" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" diff --git a/pkg/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go similarity index 97% rename from pkg/api/resolver_mutation_configure.go rename to internal/api/resolver_mutation_configure.go index 6e78220c8..db4482e12 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -6,11 +6,11 @@ import ( "fmt" "path/filepath" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) var ErrOverriddenConfig = errors.New("cannot set overridden value") @@ -40,7 +40,7 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co } } if isNew { - exists, err := utils.DirExists(s.Path) + exists, err := fsutil.DirExists(s.Path) if !exists { return makeConfigGeneralResult(), err } @@ -63,7 +63,7 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co } if !optional || value != "" { - if err := utils.EnsureDir(value); err != nil { + if err := fsutil.EnsureDir(value); err != nil { return err } } @@ -211,6 +211,7 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co if input.LogLevel != nil && *input.LogLevel != c.GetLogLevel() { c.Set(config.LogLevel, input.LogLevel) + logger := manager.GetInstance().Logger logger.SetLogLevel(*input.LogLevel) } diff --git a/pkg/api/resolver_mutation_dlna.go b/internal/api/resolver_mutation_dlna.go similarity index 96% rename from pkg/api/resolver_mutation_dlna.go rename to internal/api/resolver_mutation_dlna.go index 50b1cb108..6f43a9e6f 100644 --- a/pkg/api/resolver_mutation_dlna.go +++ b/internal/api/resolver_mutation_dlna.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_mutation_gallery.go b/internal/api/resolver_mutation_gallery.go similarity index 95% rename from pkg/api/resolver_mutation_gallery.go rename to internal/api/resolver_mutation_gallery.go index 42206b6cd..cd04a0313 100644 --- a/pkg/api/resolver_mutation_gallery.go +++ b/internal/api/resolver_mutation_gallery.go @@ -9,11 +9,14 @@ import ( "strconv" "time" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/image" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -35,7 +38,7 @@ func (r *mutationResolver) GalleryCreate(ctx context.Context, input models.Galle } // for manually created galleries, generate checksum from title - checksum := utils.MD5FromString(input.Title) + checksum := md5.FromString(input.Title) // Populate a new performer from the input currentTime := time.Now() @@ -110,7 +113,7 @@ func (r *mutationResolver) GalleryCreate(ctx context.Context, input models.Galle } func (r *mutationResolver) updateGalleryPerformers(qb models.GalleryReaderWriter, galleryID int, performerIDs []string) error { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return err } @@ -118,7 +121,7 @@ func (r *mutationResolver) updateGalleryPerformers(qb models.GalleryReaderWriter } func (r *mutationResolver) updateGalleryTags(qb models.GalleryReaderWriter, galleryID int, tagIDs []string) error { - ids, err := utils.StringSliceToIntSlice(tagIDs) + ids, err := stringslice.StringSliceToIntSlice(tagIDs) if err != nil { return err } @@ -126,7 +129,7 @@ func (r *mutationResolver) updateGalleryTags(qb models.GalleryReaderWriter, gall } func (r *mutationResolver) updateGalleryScenes(qb models.GalleryReaderWriter, galleryID int, sceneIDs []string) error { - ids, err := utils.StringSliceToIntSlice(sceneIDs) + ids, err := stringslice.StringSliceToIntSlice(sceneIDs) if err != nil { return err } @@ -225,7 +228,7 @@ func (r *mutationResolver) galleryUpdate(input models.GalleryUpdateInput, transl // if gallery is not zip-based, then generate the checksum from the title if !originalGallery.Path.Valid { - checksum := utils.MD5FromString(*input.Title) + checksum := md5.FromString(*input.Title) updatedGallery.Checksum = &checksum } @@ -392,7 +395,7 @@ func adjustGallerySceneIDs(qb models.GalleryReader, galleryID int, ids models.Bu } func (r *mutationResolver) GalleryDestroy(ctx context.Context, input models.GalleryDestroyInput) (bool, error) { - galleryIDs, err := utils.StringSliceToIntSlice(input.Ids) + galleryIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return false, err } @@ -529,7 +532,7 @@ func (r *mutationResolver) AddGalleryImages(ctx context.Context, input models.Ga return false, err } - imageIDs, err := utils.StringSliceToIntSlice(input.ImageIds) + imageIDs, err := stringslice.StringSliceToIntSlice(input.ImageIds) if err != nil { return false, err } @@ -554,7 +557,7 @@ func (r *mutationResolver) AddGalleryImages(ctx context.Context, input models.Ga return err } - newIDs = utils.IntAppendUniques(newIDs, imageIDs) + newIDs = intslice.IntAppendUniques(newIDs, imageIDs) return qb.UpdateImages(galleryID, newIDs) }); err != nil { return false, err @@ -569,7 +572,7 @@ func (r *mutationResolver) RemoveGalleryImages(ctx context.Context, input models return false, err } - imageIDs, err := utils.StringSliceToIntSlice(input.ImageIds) + imageIDs, err := stringslice.StringSliceToIntSlice(input.ImageIds) if err != nil { return false, err } @@ -594,7 +597,7 @@ func (r *mutationResolver) RemoveGalleryImages(ctx context.Context, input models return err } - newIDs = utils.IntExclude(newIDs, imageIDs) + newIDs = intslice.IntExclude(newIDs, imageIDs) return qb.UpdateImages(galleryID, newIDs) }); err != nil { return false, err diff --git a/pkg/api/resolver_mutation_image.go b/internal/api/resolver_mutation_image.go similarity index 96% rename from pkg/api/resolver_mutation_image.go rename to internal/api/resolver_mutation_image.go index cd0b75aa6..4c05c0ee6 100644 --- a/pkg/api/resolver_mutation_image.go +++ b/internal/api/resolver_mutation_image.go @@ -6,11 +6,12 @@ import ( "strconv" "time" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/image" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -133,7 +134,7 @@ func (r *mutationResolver) imageUpdate(input models.ImageUpdateInput, translator } func (r *mutationResolver) updateImageGalleries(qb models.ImageReaderWriter, imageID int, galleryIDs []string) error { - ids, err := utils.StringSliceToIntSlice(galleryIDs) + ids, err := stringslice.StringSliceToIntSlice(galleryIDs) if err != nil { return err } @@ -141,7 +142,7 @@ func (r *mutationResolver) updateImageGalleries(qb models.ImageReaderWriter, ima } func (r *mutationResolver) updateImagePerformers(qb models.ImageReaderWriter, imageID int, performerIDs []string) error { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return err } @@ -149,7 +150,7 @@ func (r *mutationResolver) updateImagePerformers(qb models.ImageReaderWriter, im } func (r *mutationResolver) updateImageTags(qb models.ImageReaderWriter, imageID int, tagsIDs []string) error { - ids, err := utils.StringSliceToIntSlice(tagsIDs) + ids, err := stringslice.StringSliceToIntSlice(tagsIDs) if err != nil { return err } @@ -157,7 +158,7 @@ func (r *mutationResolver) updateImageTags(qb models.ImageReaderWriter, imageID } func (r *mutationResolver) BulkImageUpdate(ctx context.Context, input models.BulkImageUpdateInput) (ret []*models.Image, err error) { - imageIDs, err := utils.StringSliceToIntSlice(input.Ids) + imageIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return nil, err } @@ -320,7 +321,7 @@ func (r *mutationResolver) ImageDestroy(ctx context.Context, input models.ImageD } func (r *mutationResolver) ImagesDestroy(ctx context.Context, input models.ImagesDestroyInput) (ret bool, err error) { - imageIDs, err := utils.StringSliceToIntSlice(input.Ids) + imageIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return false, err } diff --git a/pkg/api/resolver_mutation_job.go b/internal/api/resolver_mutation_job.go similarity index 90% rename from pkg/api/resolver_mutation_job.go rename to internal/api/resolver_mutation_job.go index 61e17b14f..541746833 100644 --- a/pkg/api/resolver_mutation_job.go +++ b/internal/api/resolver_mutation_job.go @@ -4,7 +4,7 @@ import ( "context" "strconv" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" ) func (r *mutationResolver) StopJob(ctx context.Context, jobID string) (bool, error) { diff --git a/pkg/api/resolver_mutation_metadata.go b/internal/api/resolver_mutation_metadata.go similarity index 91% rename from pkg/api/resolver_mutation_metadata.go rename to internal/api/resolver_mutation_metadata.go index ff49347a3..04626d9b4 100644 --- a/pkg/api/resolver_mutation_metadata.go +++ b/internal/api/resolver_mutation_metadata.go @@ -9,12 +9,12 @@ import ( "sync" "time" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func (r *mutationResolver) MetadataScan(ctx context.Context, input models.ScanMetadataInput) (string, error) { @@ -113,7 +113,7 @@ func (r *mutationResolver) BackupDatabase(ctx context.Context, input models.Back mgr := manager.GetInstance() var backupPath string if download { - if err := utils.EnsureDir(mgr.Paths.Generated.Downloads); err != nil { + if err := fsutil.EnsureDir(mgr.Paths.Generated.Downloads); err != nil { return nil, fmt.Errorf("could not create backup directory %v: %w", mgr.Paths.Generated.Downloads, err) } f, err := os.CreateTemp(mgr.Paths.Generated.Downloads, "backup*.sqlite") @@ -133,7 +133,10 @@ func (r *mutationResolver) BackupDatabase(ctx context.Context, input models.Back } if download { - downloadHash := mgr.DownloadStore.RegisterFile(backupPath, "", false) + downloadHash, err := mgr.DownloadStore.RegisterFile(backupPath, "", false) + if err != nil { + return nil, fmt.Errorf("error registering file for download: %w", err) + } logger.Debugf("Generated backup file %s with hash %s", backupPath, downloadHash) baseURL, _ := ctx.Value(BaseURLCtxKey).(string) diff --git a/pkg/api/resolver_mutation_movie.go b/internal/api/resolver_mutation_movie.go similarity index 96% rename from pkg/api/resolver_mutation_movie.go rename to internal/api/resolver_mutation_movie.go index 59413f148..325d022d3 100644 --- a/pkg/api/resolver_mutation_movie.go +++ b/internal/api/resolver_mutation_movie.go @@ -7,8 +7,10 @@ import ( "strconv" "time" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -25,7 +27,7 @@ func (r *mutationResolver) getMovie(ctx context.Context, id int) (ret *models.Mo func (r *mutationResolver) MovieCreate(ctx context.Context, input models.MovieCreateInput) (*models.Movie, error) { // generate checksum from movie name rather than image - checksum := utils.MD5FromString(input.Name) + checksum := md5.FromString(input.Name) var frontimageData []byte var backimageData []byte @@ -156,7 +158,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp if input.Name != nil { // generate checksum from movie name rather than image - checksum := utils.MD5FromString(*input.Name) + checksum := md5.FromString(*input.Name) updatedMovie.Name = &sql.NullString{String: *input.Name, Valid: true} updatedMovie.Checksum = &checksum } @@ -222,7 +224,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp } func (r *mutationResolver) BulkMovieUpdate(ctx context.Context, input models.BulkMovieUpdateInput) ([]*models.Movie, error) { - movieIDs, err := utils.StringSliceToIntSlice(input.Ids) + movieIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return nil, err } @@ -304,7 +306,7 @@ func (r *mutationResolver) MovieDestroy(ctx context.Context, input models.MovieD } func (r *mutationResolver) MoviesDestroy(ctx context.Context, movieIDs []string) (bool, error) { - ids, err := utils.StringSliceToIntSlice(movieIDs) + ids, err := stringslice.StringSliceToIntSlice(movieIDs) if err != nil { return false, err } diff --git a/pkg/api/resolver_mutation_performer.go b/internal/api/resolver_mutation_performer.go similarity index 97% rename from pkg/api/resolver_mutation_performer.go rename to internal/api/resolver_mutation_performer.go index 90e33b78b..55f83ac31 100644 --- a/pkg/api/resolver_mutation_performer.go +++ b/internal/api/resolver_mutation_performer.go @@ -7,9 +7,11 @@ import ( "strconv" "time" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/performer" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -26,7 +28,7 @@ func (r *mutationResolver) getPerformer(ctx context.Context, id int) (ret *model func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.PerformerCreateInput) (*models.Performer, error) { // generate checksum from performer name rather than image - checksum := utils.MD5FromString(input.Name) + checksum := md5.FromString(input.Name) var imageData []byte var err error @@ -186,7 +188,7 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per if input.Name != nil { // generate checksum from performer name rather than image - checksum := utils.MD5FromString(*input.Name) + checksum := md5.FromString(*input.Name) updatedPerformer.Name = &sql.NullString{String: *input.Name, Valid: true} updatedPerformer.Checksum = &checksum @@ -285,7 +287,7 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per } func (r *mutationResolver) updatePerformerTags(qb models.PerformerReaderWriter, performerID int, tagsIDs []string) error { - ids, err := utils.StringSliceToIntSlice(tagsIDs) + ids, err := stringslice.StringSliceToIntSlice(tagsIDs) if err != nil { return err } @@ -293,7 +295,7 @@ func (r *mutationResolver) updatePerformerTags(qb models.PerformerReaderWriter, } func (r *mutationResolver) BulkPerformerUpdate(ctx context.Context, input models.BulkPerformerUpdateInput) ([]*models.Performer, error) { - performerIDs, err := utils.StringSliceToIntSlice(input.Ids) + performerIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return nil, err } @@ -420,7 +422,7 @@ func (r *mutationResolver) PerformerDestroy(ctx context.Context, input models.Pe } func (r *mutationResolver) PerformersDestroy(ctx context.Context, performerIDs []string) (bool, error) { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return false, err } diff --git a/pkg/api/resolver_mutation_plugin.go b/internal/api/resolver_mutation_plugin.go similarity index 92% rename from pkg/api/resolver_mutation_plugin.go rename to internal/api/resolver_mutation_plugin.go index 656356ff7..48a2a29e2 100644 --- a/pkg/api/resolver_mutation_plugin.go +++ b/internal/api/resolver_mutation_plugin.go @@ -3,8 +3,8 @@ package api import ( "context" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_mutation_saved_filter.go b/internal/api/resolver_mutation_saved_filter.go similarity index 100% rename from pkg/api/resolver_mutation_saved_filter.go rename to internal/api/resolver_mutation_saved_filter.go diff --git a/pkg/api/resolver_mutation_scene.go b/internal/api/resolver_mutation_scene.go similarity index 96% rename from pkg/api/resolver_mutation_scene.go rename to internal/api/resolver_mutation_scene.go index ff747a2b0..3d8a6b238 100644 --- a/pkg/api/resolver_mutation_scene.go +++ b/internal/api/resolver_mutation_scene.go @@ -7,12 +7,14 @@ import ( "strconv" "time" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -181,7 +183,7 @@ func (r *mutationResolver) sceneUpdate(ctx context.Context, input models.SceneUp } func (r *mutationResolver) updateScenePerformers(qb models.SceneReaderWriter, sceneID int, performerIDs []string) error { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return err } @@ -215,7 +217,7 @@ func (r *mutationResolver) updateSceneMovies(qb models.SceneReaderWriter, sceneI } func (r *mutationResolver) updateSceneTags(qb models.SceneReaderWriter, sceneID int, tagsIDs []string) error { - ids, err := utils.StringSliceToIntSlice(tagsIDs) + ids, err := stringslice.StringSliceToIntSlice(tagsIDs) if err != nil { return err } @@ -223,7 +225,7 @@ func (r *mutationResolver) updateSceneTags(qb models.SceneReaderWriter, sceneID } func (r *mutationResolver) updateSceneGalleries(qb models.SceneReaderWriter, sceneID int, galleryIDs []string) error { - ids, err := utils.StringSliceToIntSlice(galleryIDs) + ids, err := stringslice.StringSliceToIntSlice(galleryIDs) if err != nil { return err } @@ -231,7 +233,7 @@ func (r *mutationResolver) updateSceneGalleries(qb models.SceneReaderWriter, sce } func (r *mutationResolver) BulkSceneUpdate(ctx context.Context, input models.BulkSceneUpdateInput) ([]*models.Scene, error) { - sceneIDs, err := utils.StringSliceToIntSlice(input.Ids) + sceneIDs, err := stringslice.StringSliceToIntSlice(input.Ids) if err != nil { return nil, err } @@ -593,7 +595,7 @@ func (r *mutationResolver) SceneMarkerCreate(ctx context.Context, input models.S UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, } - tagIDs, err := utils.StringSliceToIntSlice(input.TagIds) + tagIDs, err := stringslice.StringSliceToIntSlice(input.TagIds) if err != nil { return nil, err } @@ -633,7 +635,7 @@ func (r *mutationResolver) SceneMarkerUpdate(ctx context.Context, input models.S UpdatedAt: models.SQLiteTimestamp{Timestamp: time.Now()}, } - tagIDs, err := utils.StringSliceToIntSlice(input.TagIds) + tagIDs, err := stringslice.StringSliceToIntSlice(input.TagIds) if err != nil { return nil, err } @@ -746,7 +748,7 @@ func (r *mutationResolver) changeMarker(ctx context.Context, changeType int, cha // Save the marker tags // If this tag is the primary tag, then let's not add it. - tagIDs = utils.IntExclude(tagIDs, []int{changedMarker.PrimaryTagID}) + tagIDs = intslice.IntExclude(tagIDs, []int{changedMarker.PrimaryTagID}) return qb.UpdateTags(sceneMarker.ID, tagIDs) }); err != nil { fileDeleter.Rollback() diff --git a/pkg/api/resolver_mutation_scraper.go b/internal/api/resolver_mutation_scraper.go similarity index 83% rename from pkg/api/resolver_mutation_scraper.go rename to internal/api/resolver_mutation_scraper.go index 0f82fdcb2..a3b6ee8ab 100644 --- a/pkg/api/resolver_mutation_scraper.go +++ b/internal/api/resolver_mutation_scraper.go @@ -3,7 +3,7 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" ) func (r *mutationResolver) ReloadScrapers(ctx context.Context) (bool, error) { diff --git a/pkg/api/resolver_mutation_stash_box.go b/internal/api/resolver_mutation_stash_box.go similarity index 96% rename from pkg/api/resolver_mutation_stash_box.go rename to internal/api/resolver_mutation_stash_box.go index 1c9d6d34b..2b6d259ad 100644 --- a/pkg/api/resolver_mutation_stash_box.go +++ b/internal/api/resolver_mutation_stash_box.go @@ -5,8 +5,8 @@ import ( "fmt" "strconv" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper/stashbox" ) diff --git a/pkg/api/resolver_mutation_studio.go b/internal/api/resolver_mutation_studio.go similarity index 95% rename from pkg/api/resolver_mutation_studio.go rename to internal/api/resolver_mutation_studio.go index 5353c7594..a60e29516 100644 --- a/pkg/api/resolver_mutation_studio.go +++ b/internal/api/resolver_mutation_studio.go @@ -6,9 +6,11 @@ import ( "strconv" "time" + "github.com/stashapp/stash/pkg/hash/md5" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/studio" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/utils" @@ -27,7 +29,7 @@ func (r *mutationResolver) getStudio(ctx context.Context, id int) (ret *models.S func (r *mutationResolver) StudioCreate(ctx context.Context, input models.StudioCreateInput) (*models.Studio, error) { // generate checksum from studio name rather than image - checksum := utils.MD5FromString(input.Name) + checksum := md5.FromString(input.Name) var imageData []byte var err error @@ -137,7 +139,7 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio } if input.Name != nil { // generate checksum from studio name rather than image - checksum := utils.MD5FromString(*input.Name) + checksum := md5.FromString(*input.Name) updatedStudio.Name = &sql.NullString{String: *input.Name, Valid: true} updatedStudio.Checksum = &checksum } @@ -219,7 +221,7 @@ func (r *mutationResolver) StudioDestroy(ctx context.Context, input models.Studi } func (r *mutationResolver) StudiosDestroy(ctx context.Context, studioIDs []string) (bool, error) { - ids, err := utils.StringSliceToIntSlice(studioIDs) + ids, err := stringslice.StringSliceToIntSlice(studioIDs) if err != nil { return false, err } diff --git a/pkg/api/resolver_mutation_tag.go b/internal/api/resolver_mutation_tag.go similarity index 94% rename from pkg/api/resolver_mutation_tag.go rename to internal/api/resolver_mutation_tag.go index 6b9cc14d2..dc806492b 100644 --- a/pkg/api/resolver_mutation_tag.go +++ b/internal/api/resolver_mutation_tag.go @@ -9,6 +9,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/tag" "github.com/stashapp/stash/pkg/utils" ) @@ -48,14 +49,14 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input models.TagCreate var childIDs []int if len(input.ParentIds) > 0 { - parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds) + parentIDs, err = stringslice.StringSliceToIntSlice(input.ParentIds) if err != nil { return nil, err } } if len(input.ChildIds) > 0 { - childIDs, err = utils.StringSliceToIntSlice(input.ChildIds) + childIDs, err = stringslice.StringSliceToIntSlice(input.ChildIds) if err != nil { return nil, err } @@ -148,14 +149,14 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate var childIDs []int if translator.hasField("parent_ids") { - parentIDs, err = utils.StringSliceToIntSlice(input.ParentIds) + parentIDs, err = stringslice.StringSliceToIntSlice(input.ParentIds) if err != nil { return nil, err } } if translator.hasField("child_ids") { - childIDs, err = utils.StringSliceToIntSlice(input.ChildIds) + childIDs, err = stringslice.StringSliceToIntSlice(input.ChildIds) if err != nil { return nil, err } @@ -264,7 +265,7 @@ func (r *mutationResolver) TagDestroy(ctx context.Context, input models.TagDestr } func (r *mutationResolver) TagsDestroy(ctx context.Context, tagIDs []string) (bool, error) { - ids, err := utils.StringSliceToIntSlice(tagIDs) + ids, err := stringslice.StringSliceToIntSlice(tagIDs) if err != nil { return false, err } @@ -290,7 +291,7 @@ func (r *mutationResolver) TagsDestroy(ctx context.Context, tagIDs []string) (bo } func (r *mutationResolver) TagsMerge(ctx context.Context, input models.TagsMergeInput) (*models.Tag, error) { - source, err := utils.StringSliceToIntSlice(input.Source) + source, err := stringslice.StringSliceToIntSlice(input.Source) if err != nil { return nil, err } diff --git a/pkg/api/resolver_mutation_tag_test.go b/internal/api/resolver_mutation_tag_test.go similarity index 100% rename from pkg/api/resolver_mutation_tag_test.go rename to internal/api/resolver_mutation_tag_test.go diff --git a/pkg/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go similarity index 94% rename from pkg/api/resolver_query_configuration.go rename to internal/api/resolver_query_configuration.go index 1383e9b62..4c8dadeae 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -3,12 +3,13 @@ package api import ( "context" "fmt" + "path/filepath" "strings" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper/stashbox" - "github.com/stashapp/stash/pkg/utils" "golang.org/x/text/collate" ) @@ -27,19 +28,37 @@ func (r *queryResolver) Directory(ctx context.Context, path, locale *string) (*m if path != nil { dirPath = *path } - currentDir := utils.GetDir(dirPath) - directories, err := utils.ListDir(col, currentDir) + currentDir := getDir(dirPath) + directories, err := listDir(col, currentDir) if err != nil { return directory, err } directory.Path = currentDir - directory.Parent = utils.GetParent(currentDir) + directory.Parent = getParent(currentDir) directory.Directories = directories return directory, err } +func getDir(path string) string { + if path == "" { + path = fsutil.GetHomeDirectory() + } + + return path +} + +func getParent(path string) *string { + isRoot := path[len(path)-1:] == "/" + if isRoot { + return nil + } else { + parentPath := filepath.Clean(path + "/..") + return &parentPath + } +} + func makeConfigResult() *models.ConfigResult { return &models.ConfigResult{ General: makeConfigGeneralResult(), diff --git a/pkg/api/resolver_query_dlna.go b/internal/api/resolver_query_dlna.go similarity index 82% rename from pkg/api/resolver_query_dlna.go rename to internal/api/resolver_query_dlna.go index c0d4bc124..8d616f463 100644 --- a/pkg/api/resolver_query_dlna.go +++ b/internal/api/resolver_query_dlna.go @@ -3,7 +3,7 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_query_find_gallery.go b/internal/api/resolver_query_find_gallery.go similarity index 100% rename from pkg/api/resolver_query_find_gallery.go rename to internal/api/resolver_query_find_gallery.go diff --git a/pkg/api/resolver_query_find_image.go b/internal/api/resolver_query_find_image.go similarity index 86% rename from pkg/api/resolver_query_find_image.go rename to internal/api/resolver_query_find_image.go index 5de841454..d26a8b081 100644 --- a/pkg/api/resolver_query_find_image.go +++ b/internal/api/resolver_query_find_image.go @@ -6,7 +6,7 @@ import ( "github.com/99designs/gqlgen/graphql" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) func (r *queryResolver) FindImage(ctx context.Context, id *string, checksum *string) (*models.Image, error) { @@ -47,11 +47,11 @@ func (r *queryResolver) FindImages(ctx context.Context, imageFilter *models.Imag result, err := qb.Query(models.ImageQueryOptions{ QueryOptions: models.QueryOptions{ FindFilter: filter, - Count: utils.StrInclude(fields, "count"), + Count: stringslice.StrInclude(fields, "count"), }, ImageFilter: imageFilter, - Megapixels: utils.StrInclude(fields, "megapixels"), - TotalSize: utils.StrInclude(fields, "filesize"), + Megapixels: stringslice.StrInclude(fields, "megapixels"), + TotalSize: stringslice.StrInclude(fields, "filesize"), }) if err != nil { return err diff --git a/pkg/api/resolver_query_find_movie.go b/internal/api/resolver_query_find_movie.go similarity index 100% rename from pkg/api/resolver_query_find_movie.go rename to internal/api/resolver_query_find_movie.go diff --git a/pkg/api/resolver_query_find_performer.go b/internal/api/resolver_query_find_performer.go similarity index 100% rename from pkg/api/resolver_query_find_performer.go rename to internal/api/resolver_query_find_performer.go diff --git a/pkg/api/resolver_query_find_saved_filter.go b/internal/api/resolver_query_find_saved_filter.go similarity index 100% rename from pkg/api/resolver_query_find_saved_filter.go rename to internal/api/resolver_query_find_saved_filter.go diff --git a/pkg/api/resolver_query_find_scene.go b/internal/api/resolver_query_find_scene.go similarity index 90% rename from pkg/api/resolver_query_find_scene.go rename to internal/api/resolver_query_find_scene.go index 3b789ea32..180e25a32 100644 --- a/pkg/api/resolver_query_find_scene.go +++ b/internal/api/resolver_query_find_scene.go @@ -5,9 +5,9 @@ import ( "strconv" "github.com/99designs/gqlgen/graphql" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) func (r *queryResolver) FindScene(ctx context.Context, id *string, checksum *string) (*models.Scene, error) { @@ -86,11 +86,11 @@ func (r *queryResolver) FindScenes(ctx context.Context, sceneFilter *models.Scen result, err = repo.Scene().Query(models.SceneQueryOptions{ QueryOptions: models.QueryOptions{ FindFilter: filter, - Count: utils.StrInclude(fields, "count"), + Count: stringslice.StrInclude(fields, "count"), }, SceneFilter: sceneFilter, - TotalDuration: utils.StrInclude(fields, "duration"), - TotalSize: utils.StrInclude(fields, "filesize"), + TotalDuration: stringslice.StrInclude(fields, "duration"), + TotalSize: stringslice.StrInclude(fields, "filesize"), }) if err == nil { scenes, err = result.Resolve() @@ -141,11 +141,11 @@ func (r *queryResolver) FindScenesByPathRegex(ctx context.Context, filter *model result, err := repo.Scene().Query(models.SceneQueryOptions{ QueryOptions: models.QueryOptions{ FindFilter: queryFilter, - Count: utils.StrInclude(fields, "count"), + Count: stringslice.StrInclude(fields, "count"), }, SceneFilter: sceneFilter, - TotalDuration: utils.StrInclude(fields, "duration"), - TotalSize: utils.StrInclude(fields, "filesize"), + TotalDuration: stringslice.StrInclude(fields, "duration"), + TotalSize: stringslice.StrInclude(fields, "filesize"), }) if err != nil { return err diff --git a/pkg/api/resolver_query_find_scene_marker.go b/internal/api/resolver_query_find_scene_marker.go similarity index 100% rename from pkg/api/resolver_query_find_scene_marker.go rename to internal/api/resolver_query_find_scene_marker.go diff --git a/pkg/api/resolver_query_find_studio.go b/internal/api/resolver_query_find_studio.go similarity index 100% rename from pkg/api/resolver_query_find_studio.go rename to internal/api/resolver_query_find_studio.go diff --git a/pkg/api/resolver_query_find_tag.go b/internal/api/resolver_query_find_tag.go similarity index 100% rename from pkg/api/resolver_query_find_tag.go rename to internal/api/resolver_query_find_tag.go diff --git a/pkg/api/resolver_query_job.go b/internal/api/resolver_query_job.go similarity index 95% rename from pkg/api/resolver_query_job.go rename to internal/api/resolver_query_job.go index 3a8f4de97..06f090190 100644 --- a/pkg/api/resolver_query_job.go +++ b/internal/api/resolver_query_job.go @@ -4,8 +4,8 @@ import ( "context" "strconv" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/job" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_query_logs.go b/internal/api/resolver_query_logs.go similarity index 82% rename from pkg/api/resolver_query_logs.go rename to internal/api/resolver_query_logs.go index 2373205c4..c6ca6d9fd 100644 --- a/pkg/api/resolver_query_logs.go +++ b/internal/api/resolver_query_logs.go @@ -3,11 +3,12 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) func (r *queryResolver) Logs(ctx context.Context) ([]*models.LogEntry, error) { + logger := manager.GetInstance().Logger logCache := logger.GetLogCache() ret := make([]*models.LogEntry, len(logCache)) diff --git a/pkg/api/resolver_query_metadata.go b/internal/api/resolver_query_metadata.go similarity index 82% rename from pkg/api/resolver_query_metadata.go rename to internal/api/resolver_query_metadata.go index a1bacebb0..d96beb407 100644 --- a/pkg/api/resolver_query_metadata.go +++ b/internal/api/resolver_query_metadata.go @@ -3,7 +3,7 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_query_plugin.go b/internal/api/resolver_query_plugin.go similarity index 89% rename from pkg/api/resolver_query_plugin.go rename to internal/api/resolver_query_plugin.go index 059328e12..87c93dfcc 100644 --- a/pkg/api/resolver_query_plugin.go +++ b/internal/api/resolver_query_plugin.go @@ -3,7 +3,7 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_query_scene.go b/internal/api/resolver_query_scene.go similarity index 83% rename from pkg/api/resolver_query_scene.go rename to internal/api/resolver_query_scene.go index 236913689..3d593d911 100644 --- a/pkg/api/resolver_query_scene.go +++ b/internal/api/resolver_query_scene.go @@ -5,9 +5,9 @@ import ( "errors" "strconv" - "github.com/stashapp/stash/pkg/api/urlbuilders" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/api/urlbuilders" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_query_scraper.go b/internal/api/resolver_query_scraper.go similarity index 99% rename from pkg/api/resolver_query_scraper.go rename to internal/api/resolver_query_scraper.go index 5512f36c5..cafe05f85 100644 --- a/pkg/api/resolver_query_scraper.go +++ b/internal/api/resolver_query_scraper.go @@ -6,7 +6,7 @@ import ( "fmt" "strconv" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/scraper/stashbox" diff --git a/pkg/api/resolver_subscription_job.go b/internal/api/resolver_subscription_job.go similarity index 95% rename from pkg/api/resolver_subscription_job.go rename to internal/api/resolver_subscription_job.go index 2ee28df96..8e2da6654 100644 --- a/pkg/api/resolver_subscription_job.go +++ b/internal/api/resolver_subscription_job.go @@ -3,8 +3,8 @@ package api import ( "context" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/job" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/api/resolver_subscription_logging.go b/internal/api/resolver_subscription_logging.go similarity index 84% rename from pkg/api/resolver_subscription_logging.go rename to internal/api/resolver_subscription_logging.go index db6b9c8b2..17241e12b 100644 --- a/pkg/api/resolver_subscription_logging.go +++ b/internal/api/resolver_subscription_logging.go @@ -3,7 +3,8 @@ package api import ( "context" - "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/internal/log" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/models" ) @@ -26,7 +27,7 @@ func getLogLevel(logType string) models.LogLevel { } } -func logEntriesFromLogItems(logItems []logger.LogItem) []*models.LogEntry { +func logEntriesFromLogItems(logItems []log.LogItem) []*models.LogEntry { ret := make([]*models.LogEntry, len(logItems)) for i, entry := range logItems { @@ -43,6 +44,7 @@ func logEntriesFromLogItems(logItems []logger.LogItem) []*models.LogEntry { func (r *subscriptionResolver) LoggingSubscribe(ctx context.Context) (<-chan []*models.LogEntry, error) { ret := make(chan []*models.LogEntry, 100) stop := make(chan int, 1) + logger := manager.GetInstance().Logger logSub := logger.SubscribeToLog(stop) go func() { diff --git a/pkg/api/routes_downloads.go b/internal/api/routes_downloads.go similarity index 94% rename from pkg/api/routes_downloads.go rename to internal/api/routes_downloads.go index dbe138fe7..16ceedaab 100644 --- a/pkg/api/routes_downloads.go +++ b/internal/api/routes_downloads.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/go-chi/chi" - "github.com/stashapp/stash/pkg/manager" + "github.com/stashapp/stash/internal/manager" ) type downloadsRoutes struct{} diff --git a/pkg/api/routes_image.go b/internal/api/routes_image.go similarity index 93% rename from pkg/api/routes_image.go rename to internal/api/routes_image.go index 092d77668..f27590d4d 100644 --- a/pkg/api/routes_image.go +++ b/internal/api/routes_image.go @@ -6,11 +6,11 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type imageRoutes struct { @@ -39,7 +39,7 @@ func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) { w.Header().Add("Cache-Control", "max-age=604800000") // if the thumbnail doesn't exist, encode on the fly - exists, _ := utils.FileExists(filepath) + exists, _ := fsutil.FileExists(filepath) if exists { http.ServeFile(w, r, filepath) } else { @@ -55,7 +55,7 @@ func (rs imageRoutes) Thumbnail(w http.ResponseWriter, r *http.Request) { // write the generated thumbnail to disk if enabled if manager.GetInstance().Config.IsWriteImageThumbnails() { - if err := utils.WriteFile(filepath, data); err != nil { + if err := fsutil.WriteFile(filepath, data); err != nil { logger.Errorf("error writing thumbnail for image %s: %s", img.Path, err) } } diff --git a/pkg/api/routes_movie.go b/internal/api/routes_movie.go similarity index 93% rename from pkg/api/routes_movie.go rename to internal/api/routes_movie.go index 4018d39ed..439b1e4d3 100644 --- a/pkg/api/routes_movie.go +++ b/internal/api/routes_movie.go @@ -6,8 +6,8 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -43,7 +43,7 @@ func (rs movieRoutes) FrontImage(w http.ResponseWriter, r *http.Request) { } if len(image) == 0 { - _, image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) + image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) } if err := utils.ServeImage(image, w, r); err != nil { @@ -66,7 +66,7 @@ func (rs movieRoutes) BackImage(w http.ResponseWriter, r *http.Request) { } if len(image) == 0 { - _, image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) + image, _ = utils.ProcessBase64Image(models.DefaultMovieImage) } if err := utils.ServeImage(image, w, r); err != nil { diff --git a/pkg/api/routes_performer.go b/internal/api/routes_performer.go similarity index 95% rename from pkg/api/routes_performer.go rename to internal/api/routes_performer.go index 101fb2932..e5c0bb862 100644 --- a/pkg/api/routes_performer.go +++ b/internal/api/routes_performer.go @@ -6,9 +6,9 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/api/routes_scene.go b/internal/api/routes_scene.go similarity index 93% rename from pkg/api/routes_scene.go rename to internal/api/routes_scene.go index 9bd9eefe9..c638b7407 100644 --- a/pkg/api/routes_scene.go +++ b/internal/api/routes_scene.go @@ -7,11 +7,13 @@ import ( "strings" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/utils" ) @@ -122,13 +124,13 @@ func (rs sceneRoutes) StreamHLS(w http.ResponseWriter, r *http.Request) { ffmpeg.WriteHLSPlaylist(*videoFile, r.URL.String(), &str) - requestByteRange := utils.CreateByteRange(r.Header.Get("Range")) + requestByteRange := createByteRange(r.Header.Get("Range")) if requestByteRange.RawString != "" { logger.Debugf("Requested range: %s", requestByteRange.RawString) } - ret := requestByteRange.Apply([]byte(str.String())) - rangeStr := requestByteRange.ToHeaderValue(int64(str.Len())) + ret := requestByteRange.apply([]byte(str.String())) + rangeStr := requestByteRange.toHeaderValue(int64(str.Len())) w.Header().Set("Content-Range", rangeStr) if n, err := w.Write(ret); err != nil { @@ -201,7 +203,15 @@ func (rs sceneRoutes) Screenshot(w http.ResponseWriter, r *http.Request) { func (rs sceneRoutes) Preview(w http.ResponseWriter, r *http.Request) { scene := r.Context().Value(sceneKey).(*models.Scene) filepath := manager.GetInstance().Paths.Scene.GetStreamPreviewPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm())) - utils.ServeFileNoCache(w, r, filepath) + serveFileNoCache(w, r, filepath) +} + +// serveFileNoCache serves the provided file, ensuring that the response +// contains headers to prevent caching. +func serveFileNoCache(w http.ResponseWriter, r *http.Request, filepath string) { + w.Header().Add("Cache-Control", "no-cache") + + http.ServeFile(w, r, filepath) } func (rs sceneRoutes) Webp(w http.ResponseWriter, r *http.Request) { @@ -269,9 +279,9 @@ func (rs sceneRoutes) ChapterVtt(w http.ResponseWriter, r *http.Request) { } func (rs sceneRoutes) Funscript(w http.ResponseWriter, r *http.Request) { - scene := r.Context().Value(sceneKey).(*models.Scene) - funscript := utils.GetFunscriptPath(scene.Path) - utils.ServeFileNoCache(w, r, funscript) + s := r.Context().Value(sceneKey).(*models.Scene) + funscript := scene.GetFunscriptPath(s.Path) + serveFileNoCache(w, r, funscript) } func (rs sceneRoutes) InteractiveHeatmap(w http.ResponseWriter, r *http.Request) { @@ -340,7 +350,7 @@ func (rs sceneRoutes) SceneMarkerPreview(w http.ResponseWriter, r *http.Request) filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamPreviewImagePath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) // If the image doesn't exist, send the placeholder - exists, _ := utils.FileExists(filepath) + exists, _ := fsutil.FileExists(filepath) if !exists { w.Header().Set("Content-Type", "image/png") w.Header().Set("Cache-Control", "no-store") @@ -373,7 +383,7 @@ func (rs sceneRoutes) SceneMarkerScreenshot(w http.ResponseWriter, r *http.Reque filepath := manager.GetInstance().Paths.SceneMarkers.GetStreamScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), int(sceneMarker.Seconds)) // If the image doesn't exist, send the placeholder - exists, _ := utils.FileExists(filepath) + exists, _ := fsutil.FileExists(filepath) if !exists { w.Header().Set("Content-Type", "image/png") w.Header().Set("Cache-Control", "no-store") diff --git a/pkg/api/routes_studio.go b/internal/api/routes_studio.go similarity index 94% rename from pkg/api/routes_studio.go rename to internal/api/routes_studio.go index 2cea188ec..18f78b30c 100644 --- a/pkg/api/routes_studio.go +++ b/internal/api/routes_studio.go @@ -8,8 +8,8 @@ import ( "syscall" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -45,7 +45,7 @@ func (rs studioRoutes) Image(w http.ResponseWriter, r *http.Request) { } if len(image) == 0 { - _, image, _ = utils.ProcessBase64Image(models.DefaultStudioImage) + image, _ = utils.ProcessBase64Image(models.DefaultStudioImage) } if err := utils.ServeImage(image, w, r); err != nil { diff --git a/pkg/api/routes_tag.go b/internal/api/routes_tag.go similarity index 97% rename from pkg/api/routes_tag.go rename to internal/api/routes_tag.go index cebb628cc..8ffdc62c9 100644 --- a/pkg/api/routes_tag.go +++ b/internal/api/routes_tag.go @@ -6,8 +6,8 @@ import ( "strconv" "github.com/go-chi/chi" + "github.com/stashapp/stash/internal/manager" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/api/scraped_content.go b/internal/api/scraped_content.go similarity index 100% rename from pkg/api/scraped_content.go rename to internal/api/scraped_content.go diff --git a/pkg/api/server.go b/internal/api/server.go similarity index 96% rename from pkg/api/server.go rename to internal/api/server.go index b162312d3..756aefe8f 100644 --- a/pkg/api/server.go +++ b/internal/api/server.go @@ -3,7 +3,6 @@ package api import ( "context" "crypto/tls" - "embed" "errors" "fmt" "io/fs" @@ -23,23 +22,26 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/gorilla/websocket" + "github.com/vearutop/statigz" "github.com/go-chi/httplog" "github.com/rs/cors" - "github.com/stashapp/stash/pkg/desktop" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" - "github.com/vearutop/statigz" + "github.com/stashapp/stash/ui" ) var version string var buildstamp string var githash string -func Start(uiBox embed.FS, loginUIBox embed.FS) { +var uiBox = ui.UIBox +var loginUIBox = ui.LoginUIBox + +func Start() { initialiseImages() r := chi.NewRouter() @@ -144,7 +146,7 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) { // search for custom.css in current directory, then $HOME/.stash fn := c.GetCSSPath() - exists, _ := utils.FileExists(fn) + exists, _ := fsutil.FileExists(fn) if !exists { return } @@ -192,7 +194,7 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) { // Serve the web app r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) { - const uiRootDir = "ui/v2.5/build" + const uiRootDir = "v2.5/build" ext := path.Ext(r.URL.Path) @@ -271,7 +273,6 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) { } manager.GetInstance().Shutdown(0) }() - desktop.Start(manager.GetInstance(), &FaviconProvider{uiBox: uiBox}) } func printVersion() { diff --git a/pkg/api/session.go b/internal/api/session.go similarity index 94% rename from pkg/api/session.go rename to internal/api/session.go index c31845755..785d93381 100644 --- a/pkg/api/session.go +++ b/internal/api/session.go @@ -7,12 +7,12 @@ import ( "html/template" "net/http" - "github.com/stashapp/stash/pkg/manager" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/session" ) -const loginRootDir = "ui/login" +const loginRootDir = "login" const returnURLParam = "returnURL" func getLoginPage(loginUIBox embed.FS) []byte { diff --git a/pkg/api/types.go b/internal/api/types.go similarity index 100% rename from pkg/api/types.go rename to internal/api/types.go diff --git a/pkg/api/urlbuilders/gallery.go b/internal/api/urlbuilders/gallery.go similarity index 100% rename from pkg/api/urlbuilders/gallery.go rename to internal/api/urlbuilders/gallery.go diff --git a/pkg/api/urlbuilders/image.go b/internal/api/urlbuilders/image.go similarity index 100% rename from pkg/api/urlbuilders/image.go rename to internal/api/urlbuilders/image.go diff --git a/pkg/api/urlbuilders/movie.go b/internal/api/urlbuilders/movie.go similarity index 100% rename from pkg/api/urlbuilders/movie.go rename to internal/api/urlbuilders/movie.go diff --git a/pkg/api/urlbuilders/performer.go b/internal/api/urlbuilders/performer.go similarity index 100% rename from pkg/api/urlbuilders/performer.go rename to internal/api/urlbuilders/performer.go diff --git a/pkg/api/urlbuilders/scene.go b/internal/api/urlbuilders/scene.go similarity index 100% rename from pkg/api/urlbuilders/scene.go rename to internal/api/urlbuilders/scene.go diff --git a/pkg/api/urlbuilders/studio.go b/internal/api/urlbuilders/studio.go similarity index 100% rename from pkg/api/urlbuilders/studio.go rename to internal/api/urlbuilders/studio.go diff --git a/pkg/api/urlbuilders/tag.go b/internal/api/urlbuilders/tag.go similarity index 100% rename from pkg/api/urlbuilders/tag.go rename to internal/api/urlbuilders/tag.go diff --git a/pkg/autotag/gallery.go b/internal/autotag/gallery.go similarity index 100% rename from pkg/autotag/gallery.go rename to internal/autotag/gallery.go diff --git a/pkg/autotag/gallery_test.go b/internal/autotag/gallery_test.go similarity index 100% rename from pkg/autotag/gallery_test.go rename to internal/autotag/gallery_test.go diff --git a/pkg/autotag/image.go b/internal/autotag/image.go similarity index 100% rename from pkg/autotag/image.go rename to internal/autotag/image.go diff --git a/pkg/autotag/image_test.go b/internal/autotag/image_test.go similarity index 100% rename from pkg/autotag/image_test.go rename to internal/autotag/image_test.go diff --git a/pkg/autotag/integration_test.go b/internal/autotag/integration_test.go similarity index 99% rename from pkg/autotag/integration_test.go rename to internal/autotag/integration_test.go index d288c9e28..65f118303 100644 --- a/pkg/autotag/integration_test.go +++ b/internal/autotag/integration_test.go @@ -11,9 +11,9 @@ import ( "testing" "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/sqlite" - "github.com/stashapp/stash/pkg/utils" _ "github.com/golang-migrate/migrate/v4/database/sqlite3" _ "github.com/golang-migrate/migrate/v4/source/file" @@ -149,7 +149,7 @@ func createScenes(sqb models.SceneReaderWriter) error { func makeScene(name string, expectedResult bool) *models.Scene { scene := &models.Scene{ - Checksum: sql.NullString{String: utils.MD5FromString(name), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(name), Valid: true}, Path: name, } @@ -211,7 +211,7 @@ func createImages(sqb models.ImageReaderWriter) error { func makeImage(name string, expectedResult bool) *models.Image { image := &models.Image{ - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Path: name, } @@ -273,7 +273,7 @@ func createGalleries(sqb models.GalleryReaderWriter) error { func makeGallery(name string, expectedResult bool) *models.Gallery { gallery := &models.Gallery{ - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Path: models.NullString(name), } diff --git a/pkg/autotag/performer.go b/internal/autotag/performer.go similarity index 100% rename from pkg/autotag/performer.go rename to internal/autotag/performer.go diff --git a/pkg/autotag/performer_test.go b/internal/autotag/performer_test.go similarity index 100% rename from pkg/autotag/performer_test.go rename to internal/autotag/performer_test.go diff --git a/pkg/autotag/scene.go b/internal/autotag/scene.go similarity index 100% rename from pkg/autotag/scene.go rename to internal/autotag/scene.go diff --git a/pkg/autotag/scene_test.go b/internal/autotag/scene_test.go similarity index 100% rename from pkg/autotag/scene_test.go rename to internal/autotag/scene_test.go diff --git a/pkg/autotag/studio.go b/internal/autotag/studio.go similarity index 100% rename from pkg/autotag/studio.go rename to internal/autotag/studio.go diff --git a/pkg/autotag/studio_test.go b/internal/autotag/studio_test.go similarity index 100% rename from pkg/autotag/studio_test.go rename to internal/autotag/studio_test.go diff --git a/pkg/autotag/tag.go b/internal/autotag/tag.go similarity index 100% rename from pkg/autotag/tag.go rename to internal/autotag/tag.go diff --git a/pkg/autotag/tag_test.go b/internal/autotag/tag_test.go similarity index 100% rename from pkg/autotag/tag_test.go rename to internal/autotag/tag_test.go diff --git a/pkg/autotag/tagger.go b/internal/autotag/tagger.go similarity index 100% rename from pkg/autotag/tagger.go rename to internal/autotag/tagger.go diff --git a/pkg/desktop/desktop.go b/internal/desktop/desktop.go similarity index 90% rename from pkg/desktop/desktop.go rename to internal/desktop/desktop.go index cad8ade3c..7ff70a684 100644 --- a/pkg/desktop/desktop.go +++ b/internal/desktop/desktop.go @@ -3,7 +3,6 @@ package desktop import ( "io/ioutil" "os" - "os/exec" "path" "path/filepath" "runtime" @@ -11,9 +10,9 @@ import ( "strings" "github.com/pkg/browser" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/utils" "golang.org/x/term" ) @@ -26,6 +25,7 @@ type FaviconProvider interface { GetFaviconPng() []byte } +// Start starts the desktop icon process. It blocks until the process exits. func Start(shutdownHandler ShutdownHandler, faviconProvider FaviconProvider) { if IsDesktop() { c := config.GetInstance() @@ -81,11 +81,6 @@ func IsServerDockerized() bool { return isServerDockerized() } -// Set a command to execute in the background, instead of spawning a shell window -func HideExecShell(cmd *exec.Cmd) { - hideExecShell(cmd) -} - // writeStashIcon writes the current stash logo to config/icon.png func writeStashIcon(faviconProvider FaviconProvider) { c := config.GetInstance() @@ -119,7 +114,7 @@ func IsAllowedAutoUpdate() bool { logger.Errorf("Cannot get executable path: %s", err) return false } - if utils.IsPathInDir("/usr", executablePath) || utils.IsPathInDir("/opt", executablePath) { + if fsutil.IsPathInDir("/usr", executablePath) || fsutil.IsPathInDir("/opt", executablePath) { return false } @@ -136,7 +131,7 @@ func getIconPath() string { } func RevealInFileManager(path string) { - exists, err := utils.FileExists(path) + exists, err := fsutil.FileExists(path) if err != nil { logger.Errorf("Error checking file: %s", err) return diff --git a/pkg/desktop/desktop_platform_darwin.go b/internal/desktop/desktop_platform_darwin.go similarity index 95% rename from pkg/desktop/desktop_platform_darwin.go rename to internal/desktop/desktop_platform_darwin.go index 53c9776f2..268a321cb 100644 --- a/pkg/desktop/desktop_platform_darwin.go +++ b/internal/desktop/desktop_platform_darwin.go @@ -19,10 +19,6 @@ func isServerDockerized() bool { return false } -func hideExecShell(cmd *exec.Cmd) { - -} - func sendNotification(notificationTitle string, notificationText string) { notification := gosxnotifier.NewNotification(notificationText) notification.Title = notificationTitle diff --git a/pkg/desktop/desktop_platform_linux.go b/internal/desktop/desktop_platform_linux.go similarity index 95% rename from pkg/desktop/desktop_platform_linux.go rename to internal/desktop/desktop_platform_linux.go index b1893c0e7..1d6e1a9a3 100644 --- a/pkg/desktop/desktop_platform_linux.go +++ b/internal/desktop/desktop_platform_linux.go @@ -27,10 +27,6 @@ func isServerDockerized() bool { return false } -func hideExecShell(cmd *exec.Cmd) { - -} - func sendNotification(notificationTitle string, notificationText string) { err := exec.Command("notify-send", "-i", getIconPath(), notificationTitle, notificationText, "-a", "Stash").Run() if err != nil { diff --git a/pkg/desktop/desktop_platform_windows.go b/internal/desktop/desktop_platform_windows.go similarity index 75% rename from pkg/desktop/desktop_platform_windows.go rename to internal/desktop/desktop_platform_windows.go index 7a887d508..6158ba14f 100644 --- a/pkg/desktop/desktop_platform_windows.go +++ b/internal/desktop/desktop_platform_windows.go @@ -5,9 +5,6 @@ package desktop import ( "os/exec" - "syscall" - - "golang.org/x/sys/windows" "github.com/go-toast/toast" "github.com/stashapp/stash/pkg/logger" @@ -27,12 +24,6 @@ func isServerDockerized() bool { return false } -// On Windows, calling exec.Cmd.Start() will create a cmd window, even if we live in the taskbar. -// We don't want every ffmpeg / plugin to pop up a window. -func hideExecShell(cmd *exec.Cmd) { - cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.DETACHED_PROCESS} -} - func sendNotification(notificationTitle string, notificationText string) { notification := toast.Notification{ AppID: "Stash", diff --git a/pkg/desktop/icon_windows.syso b/internal/desktop/icon_windows.syso similarity index 100% rename from pkg/desktop/icon_windows.syso rename to internal/desktop/icon_windows.syso diff --git a/pkg/desktop/systray_linux.go b/internal/desktop/systray_linux.go similarity index 100% rename from pkg/desktop/systray_linux.go rename to internal/desktop/systray_linux.go diff --git a/pkg/desktop/systray_nonlinux.go b/internal/desktop/systray_nonlinux.go similarity index 98% rename from pkg/desktop/systray_nonlinux.go rename to internal/desktop/systray_nonlinux.go index b9f72cbac..7fd3513ce 100644 --- a/pkg/desktop/systray_nonlinux.go +++ b/internal/desktop/systray_nonlinux.go @@ -7,8 +7,8 @@ import ( "strings" "github.com/kermieisinthehouse/systray" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" ) // MUST be run on the main goroutine or will have no effect on macOS diff --git a/pkg/dlna/cd-service-desc.go b/internal/dlna/cd-service-desc.go similarity index 100% rename from pkg/dlna/cd-service-desc.go rename to internal/dlna/cd-service-desc.go diff --git a/pkg/dlna/cds.go b/internal/dlna/cds.go similarity index 99% rename from pkg/dlna/cds.go rename to internal/dlna/cds.go index d9dc6f546..4544b8759 100644 --- a/pkg/dlna/cds.go +++ b/internal/dlna/cds.go @@ -40,7 +40,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) var pageSize = 100 @@ -491,7 +491,7 @@ func (me *contentDirectoryService) getPageVideos(sceneFilter *models.SceneFilter } func getPageFromID(paths []string) *int { - i := utils.StrIndex(paths, "page") + i := stringslice.StrIndex(paths, "page") if i == -1 || i+1 >= len(paths) { return nil } diff --git a/pkg/dlna/cds_test.go b/internal/dlna/cds_test.go similarity index 100% rename from pkg/dlna/cds_test.go rename to internal/dlna/cds_test.go diff --git a/pkg/dlna/cm-service-desc.go b/internal/dlna/cm-service-desc.go similarity index 100% rename from pkg/dlna/cm-service-desc.go rename to internal/dlna/cm-service-desc.go diff --git a/pkg/dlna/cms.go b/internal/dlna/cms.go similarity index 100% rename from pkg/dlna/cms.go rename to internal/dlna/cms.go diff --git a/pkg/dlna/dms.go b/internal/dlna/dms.go similarity index 100% rename from pkg/dlna/dms.go rename to internal/dlna/dms.go diff --git a/pkg/dlna/html.go b/internal/dlna/html.go similarity index 100% rename from pkg/dlna/html.go rename to internal/dlna/html.go diff --git a/pkg/dlna/mrrs.go b/internal/dlna/mrrs.go similarity index 100% rename from pkg/dlna/mrrs.go rename to internal/dlna/mrrs.go diff --git a/pkg/dlna/paging.go b/internal/dlna/paging.go similarity index 100% rename from pkg/dlna/paging.go rename to internal/dlna/paging.go diff --git a/pkg/dlna/service.go b/internal/dlna/service.go similarity index 96% rename from pkg/dlna/service.go rename to internal/dlna/service.go index d0b3ba196..6c14e8ee5 100644 --- a/pkg/dlna/service.go +++ b/internal/dlna/service.go @@ -9,7 +9,6 @@ import ( "time" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" ) @@ -28,9 +27,15 @@ type sceneServer interface { ServeScreenshot(scene *models.Scene, w http.ResponseWriter, r *http.Request) } +type Config interface { + GetDLNAInterfaces() []string + GetDLNAServerName() string + GetDLNADefaultIPWhitelist() []string +} + type Service struct { txnManager models.TransactionManager - config *config.Instance + config Config sceneServer sceneServer ipWhitelistMgr *ipWhitelistManager @@ -162,7 +167,7 @@ func (s *Service) init() error { // } // NewService initialises and returns a new DLNA service. -func NewService(txnManager models.TransactionManager, cfg *config.Instance, sceneServer sceneServer) *Service { +func NewService(txnManager models.TransactionManager, cfg Config, sceneServer sceneServer) *Service { ret := &Service{ txnManager: txnManager, sceneServer: sceneServer, diff --git a/pkg/dlna/whitelist.go b/internal/dlna/whitelist.go similarity index 95% rename from pkg/dlna/whitelist.go rename to internal/dlna/whitelist.go index 88c3914c5..447e68702 100644 --- a/pkg/dlna/whitelist.go +++ b/internal/dlna/whitelist.go @@ -4,9 +4,8 @@ import ( "sync" "time" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) // only keep the 10 most recent IP addresses @@ -21,7 +20,7 @@ type tempIPWhitelist struct { type ipWhitelistManager struct { recentIPAddresses []string - config *config.Instance + config Config tempWhitelist []tempIPWhitelist mutex sync.Mutex } @@ -32,7 +31,7 @@ func (m *ipWhitelistManager) addRecent(addr string) bool { m.mutex.Lock() defer m.mutex.Unlock() - i := utils.StrIndex(m.recentIPAddresses, addr) + i := stringslice.StrIndex(m.recentIPAddresses, addr) if i != -1 { if i == 0 { // don't do anything if it's already at the start diff --git a/pkg/dlna/xmsr-service-desc.go b/internal/dlna/xmsr-service-desc.go similarity index 100% rename from pkg/dlna/xmsr-service-desc.go rename to internal/dlna/xmsr-service-desc.go diff --git a/pkg/identify/identify.go b/internal/identify/identify.go similarity index 100% rename from pkg/identify/identify.go rename to internal/identify/identify.go diff --git a/pkg/identify/identify_test.go b/internal/identify/identify_test.go similarity index 99% rename from pkg/identify/identify_test.go rename to internal/identify/identify_test.go index 679c78bc5..3a36015f4 100644 --- a/pkg/identify/identify_test.go +++ b/internal/identify/identify_test.go @@ -8,7 +8,7 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/mocks" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stretchr/testify/mock" ) @@ -18,7 +18,7 @@ type mockSceneScraper struct { } func (s mockSceneScraper) ScrapeScene(ctx context.Context, sceneID int) (*models.ScrapedScene, error) { - if utils.IntInclude(s.errIDs, sceneID) { + if intslice.IntInclude(s.errIDs, sceneID) { return nil, errors.New("scrape scene error") } return s.results[sceneID], nil diff --git a/pkg/identify/performer.go b/internal/identify/performer.go similarity index 97% rename from pkg/identify/performer.go rename to internal/identify/performer.go index 4d0855388..495c3eb8e 100644 --- a/pkg/identify/performer.go +++ b/internal/identify/performer.go @@ -6,8 +6,8 @@ import ( "strconv" "time" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func getPerformerID(endpoint string, r models.Repository, p *models.ScrapedPerformer, createMissing bool) (*int, error) { @@ -50,7 +50,7 @@ func scrapedToPerformerInput(performer *models.ScrapedPerformer) models.Performe currentTime := time.Now() ret := models.Performer{ Name: sql.NullString{String: *performer.Name, Valid: true}, - Checksum: utils.MD5FromString(*performer.Name), + Checksum: md5.FromString(*performer.Name), CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, Favorite: sql.NullBool{Bool: false, Valid: true}, diff --git a/pkg/identify/performer_test.go b/internal/identify/performer_test.go similarity index 100% rename from pkg/identify/performer_test.go rename to internal/identify/performer_test.go diff --git a/pkg/identify/scene.go b/internal/identify/scene.go similarity index 93% rename from pkg/identify/scene.go rename to internal/identify/scene.go index a8b5d4cff..166755451 100644 --- a/pkg/identify/scene.go +++ b/internal/identify/scene.go @@ -9,6 +9,8 @@ import ( "time" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/utils" ) @@ -89,12 +91,12 @@ func (g sceneRelationships) performers(ignoreMale bool) ([]int, error) { } if performerID != nil { - performerIDs = utils.IntAppendUnique(performerIDs, *performerID) + performerIDs = intslice.IntAppendUnique(performerIDs, *performerID) } } // don't return if nothing was added - if utils.SliceSame(originalPerformerIDs, performerIDs) { + if sliceutil.SliceSame(originalPerformerIDs, performerIDs) { return nil, nil } @@ -137,7 +139,7 @@ func (g sceneRelationships) tags() ([]int, error) { return nil, fmt.Errorf("error converting tag ID %s: %w", *t.StoredID, err) } - tagIDs = utils.IntAppendUnique(tagIDs, int(tagID)) + tagIDs = intslice.IntAppendUnique(tagIDs, int(tagID)) } else if createMissing { now := time.Now() created, err := r.Tag().Create(models.Tag{ @@ -154,7 +156,7 @@ func (g sceneRelationships) tags() ([]int, error) { } // don't return if nothing was added - if utils.SliceSame(originalTagIDs, tagIDs) { + if sliceutil.SliceSame(originalTagIDs, tagIDs) { return nil, nil } @@ -216,7 +218,7 @@ func (g sceneRelationships) stashIDs() ([]models.StashID, error) { Endpoint: endpoint, }) - if utils.SliceSame(originalStashIDs, stashIDs) { + if sliceutil.SliceSame(originalStashIDs, stashIDs) { return nil, nil } diff --git a/pkg/identify/scene_test.go b/internal/identify/scene_test.go similarity index 100% rename from pkg/identify/scene_test.go rename to internal/identify/scene_test.go diff --git a/pkg/identify/studio.go b/internal/identify/studio.go similarity index 92% rename from pkg/identify/studio.go rename to internal/identify/studio.go index 4a4c2924b..86cb6b737 100644 --- a/pkg/identify/studio.go +++ b/internal/identify/studio.go @@ -5,8 +5,8 @@ import ( "fmt" "time" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func createMissingStudio(endpoint string, repo models.Repository, studio *models.ScrapedStudio) (*int64, error) { @@ -34,7 +34,7 @@ func scrapedToStudioInput(studio *models.ScrapedStudio) models.Studio { currentTime := time.Now() ret := models.Studio{ Name: sql.NullString{String: studio.Name, Valid: true}, - Checksum: utils.MD5FromString(studio.Name), + Checksum: md5.FromString(studio.Name), CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, } diff --git a/pkg/identify/studio_test.go b/internal/identify/studio_test.go similarity index 100% rename from pkg/identify/studio_test.go rename to internal/identify/studio_test.go diff --git a/pkg/logger/hook.go b/internal/log/hook.go similarity index 96% rename from pkg/logger/hook.go rename to internal/log/hook.go index b0e0417dc..33ee2847d 100644 --- a/pkg/logger/hook.go +++ b/internal/log/hook.go @@ -1,4 +1,4 @@ -package logger +package log import ( "io" diff --git a/internal/log/logger.go b/internal/log/logger.go new file mode 100644 index 000000000..f55ad725b --- /dev/null +++ b/internal/log/logger.go @@ -0,0 +1,306 @@ +package log + +import ( + "fmt" + "os" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +type LogItem struct { + Time time.Time `json:"time"` + Type string `json:"type"` + Message string `json:"message"` +} + +type Logger struct { + logger *logrus.Logger + progressLogger *logrus.Logger + mutex sync.Mutex + logCache []LogItem + logSubs []chan []LogItem + waiting bool + lastBroadcast time.Time + logBuffer []LogItem +} + +func NewLogger() *Logger { + ret := &Logger{ + logger: logrus.New(), + progressLogger: logrus.New(), + lastBroadcast: time.Now(), + } + + ret.progressLogger.SetFormatter(new(ProgressFormatter)) + + return ret +} + +// Init initialises the logger based on a logging configuration +func (log *Logger) Init(logFile string, logOut bool, logLevel string) { + var file *os.File + customFormatter := new(logrus.TextFormatter) + customFormatter.TimestampFormat = "2006-01-02 15:04:05" + customFormatter.ForceColors = true + customFormatter.FullTimestamp = true + log.logger.SetOutput(os.Stderr) + log.logger.SetFormatter(customFormatter) + + // #1837 - trigger the console to use color-mode since it won't be + // otherwise triggered until the first log entry + // this is covers the situation where the logger is only logging to file + // and therefore does not trigger the console color-mode - resulting in + // the access log colouring not being applied + _, _ = customFormatter.Format(logrus.NewEntry(log.logger)) + + if logFile != "" { + var err error + file, err = os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + + if err != nil { + fmt.Printf("Could not open '%s' for log output due to error: %s\n", logFile, err.Error()) + } + } + + if file != nil { + if logOut { + // log to file separately disabling colours + fileFormatter := new(logrus.TextFormatter) + fileFormatter.TimestampFormat = customFormatter.TimestampFormat + fileFormatter.FullTimestamp = customFormatter.FullTimestamp + log.logger.AddHook(&fileLogHook{ + Writer: file, + Formatter: fileFormatter, + }) + } else { + // logging to file only + // turn off the colouring for the file + customFormatter.ForceColors = false + log.logger.Out = file + } + } + + // otherwise, output to StdErr + + log.SetLogLevel(logLevel) +} + +func (log *Logger) SetLogLevel(level string) { + log.logger.Level = logLevelFromString(level) +} + +func logLevelFromString(level string) logrus.Level { + ret := logrus.InfoLevel + + switch level { + case "Debug": + ret = logrus.DebugLevel + case "Warning": + ret = logrus.WarnLevel + case "Error": + ret = logrus.ErrorLevel + case "Trace": + ret = logrus.TraceLevel + } + + return ret +} + +func (log *Logger) addLogItem(l *LogItem) { + log.mutex.Lock() + l.Time = time.Now() + log.logCache = append([]LogItem{*l}, log.logCache...) + if len(log.logCache) > 30 { + log.logCache = log.logCache[:len(log.logCache)-1] + } + log.mutex.Unlock() + go log.broadcastLogItem(l) +} + +func (log *Logger) GetLogCache() []LogItem { + log.mutex.Lock() + + ret := make([]LogItem, len(log.logCache)) + copy(ret, log.logCache) + + log.mutex.Unlock() + + return ret +} + +func (log *Logger) SubscribeToLog(stop chan int) <-chan []LogItem { + ret := make(chan []LogItem, 100) + + go func() { + <-stop + log.unsubscribeFromLog(ret) + }() + + log.mutex.Lock() + log.logSubs = append(log.logSubs, ret) + log.mutex.Unlock() + + return ret +} + +func (log *Logger) unsubscribeFromLog(toRemove chan []LogItem) { + log.mutex.Lock() + for i, c := range log.logSubs { + if c == toRemove { + log.logSubs = append(log.logSubs[:i], log.logSubs[i+1:]...) + } + } + close(toRemove) + log.mutex.Unlock() +} + +func (log *Logger) doBroadcastLogItems() { + // assumes mutex held + + for _, c := range log.logSubs { + // don't block waiting to broadcast + select { + case c <- log.logBuffer: + default: + } + } + + log.logBuffer = nil + log.waiting = false + log.lastBroadcast = time.Now() +} + +func (log *Logger) broadcastLogItem(l *LogItem) { + log.mutex.Lock() + + log.logBuffer = append(log.logBuffer, *l) + + // don't send more than once per second + if !log.waiting { + // if last broadcast was under a second ago, wait until a second has + // passed + timeSinceBroadcast := time.Since(log.lastBroadcast) + if timeSinceBroadcast.Seconds() < 1 { + log.waiting = true + time.AfterFunc(time.Second-timeSinceBroadcast, func() { + log.mutex.Lock() + log.doBroadcastLogItems() + log.mutex.Unlock() + }) + } else { + log.doBroadcastLogItems() + } + } + + // if waiting then adding it to the buffer is sufficient + log.mutex.Unlock() +} + +func (log *Logger) Progressf(format string, args ...interface{}) { + log.progressLogger.Infof(format, args...) + l := &LogItem{ + Type: "progress", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Trace(args ...interface{}) { + log.logger.Trace(args...) + l := &LogItem{ + Type: "trace", + Message: fmt.Sprint(args...), + } + log.addLogItem(l) +} + +func (log *Logger) Tracef(format string, args ...interface{}) { + log.logger.Tracef(format, args...) + l := &LogItem{ + Type: "trace", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Debug(args ...interface{}) { + log.logger.Debug(args...) + l := &LogItem{ + Type: "debug", + Message: fmt.Sprint(args...), + } + log.addLogItem(l) +} + +func (log *Logger) Debugf(format string, args ...interface{}) { + log.logger.Debugf(format, args...) + l := &LogItem{ + Type: "debug", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Info(args ...interface{}) { + log.logger.Info(args...) + l := &LogItem{ + Type: "info", + Message: fmt.Sprint(args...), + } + log.addLogItem(l) +} + +func (log *Logger) Infof(format string, args ...interface{}) { + log.logger.Infof(format, args...) + l := &LogItem{ + Type: "info", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Warn(args ...interface{}) { + log.logger.Warn(args...) + l := &LogItem{ + Type: "warn", + Message: fmt.Sprint(args...), + } + log.addLogItem(l) +} + +func (log *Logger) Warnf(format string, args ...interface{}) { + log.logger.Warnf(format, args...) + l := &LogItem{ + Type: "warn", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Error(args ...interface{}) { + log.logger.Error(args...) + l := &LogItem{ + Type: "error", + Message: fmt.Sprint(args...), + } + log.addLogItem(l) +} + +func (log *Logger) Errorf(format string, args ...interface{}) { + log.logger.Errorf(format, args...) + l := &LogItem{ + Type: "error", + Message: fmt.Sprintf(format, args...), + } + log.addLogItem(l) +} + +func (log *Logger) Fatal(args ...interface{}) { + log.logger.Fatal(args...) +} + +func (log *Logger) Fatalf(format string, args ...interface{}) { + log.logger.Fatalf(format, args...) +} diff --git a/internal/log/progress_formatter.go b/internal/log/progress_formatter.go new file mode 100644 index 000000000..eb3052218 --- /dev/null +++ b/internal/log/progress_formatter.go @@ -0,0 +1,12 @@ +package log + +import ( + "github.com/sirupsen/logrus" +) + +type ProgressFormatter struct{} + +func (f *ProgressFormatter) Format(entry *logrus.Entry) ([]byte, error) { + msg := []byte("Processing --> " + entry.Message + "\r") + return msg, nil +} diff --git a/pkg/manager/apikey.go b/internal/manager/apikey.go similarity index 95% rename from pkg/manager/apikey.go rename to internal/manager/apikey.go index 09147c9db..f0ffaccff 100644 --- a/pkg/manager/apikey.go +++ b/internal/manager/apikey.go @@ -5,7 +5,7 @@ import ( "time" "github.com/golang-jwt/jwt/v4" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/internal/manager/config" ) var ErrInvalidToken = errors.New("invalid apikey") diff --git a/pkg/manager/checksum.go b/internal/manager/checksum.go similarity index 97% rename from pkg/manager/checksum.go rename to internal/manager/checksum.go index 5a36bca43..469f2c47f 100644 --- a/pkg/manager/checksum.go +++ b/internal/manager/checksum.go @@ -4,8 +4,8 @@ import ( "context" "errors" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/manager/config/config.go b/internal/manager/config/config.go similarity index 98% rename from pkg/manager/config/config.go rename to internal/manager/config/config.go index 9af0070a0..7870041b4 100644 --- a/pkg/manager/config/config.go +++ b/internal/manager/config/config.go @@ -15,10 +15,11 @@ import ( "github.com/spf13/viper" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/hash" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/paths" ) var officialBuild string @@ -263,8 +264,8 @@ func (i *Instance) InitTLS() { paths.GetStashHomeDirectory(), } - i.certFile = utils.FindInPaths(tlsPaths, "stash.crt") - i.keyFile = utils.FindInPaths(tlsPaths, "stash.key") + i.certFile = fsutil.FindInPaths(tlsPaths, "stash.crt") + i.keyFile = fsutil.FindInPaths(tlsPaths, "stash.key") } func (i *Instance) GetTLSFiles() (certFile, keyFile string) { @@ -919,7 +920,7 @@ func (i *Instance) GetCSSPath() string { func (i *Instance) GetCSS() string { fn := i.GetCSSPath() - exists, _ := utils.FileExists(fn) + exists, _ := fsutil.FileExists(fn) if !exists { return "" } @@ -1252,12 +1253,18 @@ func (i *Instance) setInitialConfig(write bool) error { const apiKeyLength = 32 if string(i.GetJWTSignKey()) == "" { - signKey := utils.GenerateRandomKey(apiKeyLength) + signKey, err := hash.GenerateRandomKey(apiKeyLength) + if err != nil { + return fmt.Errorf("error generating JWTSignKey: %w", err) + } i.Set(JWTSignKey, signKey) } if string(i.GetSessionStoreKey()) == "" { - sessionStoreKey := utils.GenerateRandomKey(apiKeyLength) + sessionStoreKey, err := hash.GenerateRandomKey(apiKeyLength) + if err != nil { + return fmt.Errorf("error generating session store key: %w", err) + } i.Set(SessionStoreKey, sessionStoreKey) } diff --git a/pkg/manager/config/config_concurrency_test.go b/internal/manager/config/config_concurrency_test.go similarity index 100% rename from pkg/manager/config/config_concurrency_test.go rename to internal/manager/config/config_concurrency_test.go diff --git a/pkg/manager/config/init.go b/internal/manager/config/init.go similarity index 96% rename from pkg/manager/config/init.go rename to internal/manager/config/init.go index 02322ec62..f512999a6 100644 --- a/pkg/manager/config/init.go +++ b/internal/manager/config/init.go @@ -11,8 +11,8 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) var ( @@ -90,11 +90,11 @@ func initConfig(instance *Instance, flags flagStruct) error { v.SetConfigFile(configFile) // if file does not exist, assume it is a new system - if exists, _ := utils.FileExists(configFile); !exists { + if exists, _ := fsutil.FileExists(configFile); !exists { instance.isNewSystem = true // ensure we can write to the file - if err := utils.Touch(configFile); err != nil { + if err := fsutil.Touch(configFile); err != nil { return fmt.Errorf(`could not write to provided config path "%s": %s`, configFile, err.Error()) } else { // remove the file diff --git a/pkg/manager/config/urlmap.go b/internal/manager/config/urlmap.go similarity index 100% rename from pkg/manager/config/urlmap.go rename to internal/manager/config/urlmap.go diff --git a/pkg/manager/config/urlmap_test.go b/internal/manager/config/urlmap_test.go similarity index 100% rename from pkg/manager/config/urlmap_test.go rename to internal/manager/config/urlmap_test.go diff --git a/pkg/manager/downloads.go b/internal/manager/downloads.go similarity index 88% rename from pkg/manager/downloads.go rename to internal/manager/downloads.go index 675a08525..274b717b7 100644 --- a/pkg/manager/downloads.go +++ b/internal/manager/downloads.go @@ -6,8 +6,8 @@ import ( "sync" "time" + "github.com/stashapp/stash/pkg/hash" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) // DownloadStore manages single-use generated files for the UI to download. @@ -30,31 +30,35 @@ func NewDownloadStore() *DownloadStore { } } -func (s *DownloadStore) RegisterFile(fp string, contentType string, keep bool) string { +func (s *DownloadStore) RegisterFile(fp string, contentType string, keep bool) (string, error) { const keyLength = 4 const attempts = 100 // keep generating random keys until we get a free one // prevent infinite loop by only attempting a finite amount of times - var hash string + var h string generate := true a := 0 s.mutex.Lock() for generate && a < attempts { - hash = utils.GenerateRandomKey(keyLength) - _, generate = s.m[hash] + var err error + h, err = hash.GenerateRandomKey(keyLength) + if err != nil { + return "", err + } + _, generate = s.m[h] a++ } - s.m[hash] = &storeFile{ + s.m[h] = &storeFile{ path: fp, contentType: contentType, keep: keep, } s.mutex.Unlock() - return hash + return h, nil } func (s *DownloadStore) Serve(hash string, w http.ResponseWriter, r *http.Request) { diff --git a/pkg/manager/exclude_files.go b/internal/manager/exclude_files.go similarity index 100% rename from pkg/manager/exclude_files.go rename to internal/manager/exclude_files.go diff --git a/pkg/manager/exclude_files_test.go b/internal/manager/exclude_files_test.go similarity index 100% rename from pkg/manager/exclude_files_test.go rename to internal/manager/exclude_files_test.go diff --git a/pkg/api/favicon.go b/internal/manager/favicon.go similarity index 90% rename from pkg/api/favicon.go rename to internal/manager/favicon.go index 1a760bd3b..9be34e3c0 100644 --- a/pkg/api/favicon.go +++ b/internal/manager/favicon.go @@ -1,11 +1,11 @@ -package api +package manager import ( "embed" "runtime" ) -const faviconDir = "ui/v2.5/build/" +const faviconDir = "v2.5/build/" type FaviconProvider struct { uiBox embed.FS diff --git a/pkg/manager/filename_parser.go b/internal/manager/filename_parser.go similarity index 100% rename from pkg/manager/filename_parser.go rename to internal/manager/filename_parser.go diff --git a/pkg/manager/gallery.go b/internal/manager/gallery.go similarity index 100% rename from pkg/manager/gallery.go rename to internal/manager/gallery.go diff --git a/pkg/manager/generator.go b/internal/manager/generator.go similarity index 87% rename from pkg/manager/generator.go rename to internal/manager/generator.go index 9a6126cc2..7e0514b43 100644 --- a/pkg/manager/generator.go +++ b/internal/manager/generator.go @@ -4,15 +4,14 @@ import ( "bytes" "fmt" "math" - "os/exec" "runtime" "strconv" "strings" - "github.com/stashapp/stash/pkg/desktop" + "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) type GeneratorInfo struct { @@ -33,7 +32,7 @@ type GeneratorInfo struct { } func newGeneratorInfo(videoFile ffmpeg.VideoFile) (*GeneratorInfo, error) { - exists, err := utils.FileExists(videoFile.Path) + exists, err := fsutil.FileExists(videoFile.Path) if !exists { logger.Errorf("video file not found") return nil, err @@ -53,12 +52,12 @@ func (g *GeneratorInfo) calculateFrameRate(videoStream *ffmpeg.FFProbeStream) er numberOfFrames, _ := strconv.Atoi(videoStream.NbFrames) - if numberOfFrames == 0 && utils.IsValidFloat64(framerate) && g.VideoFile.Duration > 0 { // TODO: test + if numberOfFrames == 0 && isValidFloat64(framerate) && g.VideoFile.Duration > 0 { // TODO: test numberOfFrames = int(framerate * g.VideoFile.Duration) } // If we are missing the frame count or frame rate then seek through the file and extract the info with regex - if numberOfFrames == 0 || !utils.IsValidFloat64(framerate) { + if numberOfFrames == 0 || !isValidFloat64(framerate) { args := []string{ "-nostats", "-i", g.VideoFile.Path, @@ -73,7 +72,6 @@ func (g *GeneratorInfo) calculateFrameRate(videoStream *ffmpeg.FFProbeStream) er } command := exec.Command(string(instance.FFMPEG), args...) - desktop.HideExecShell(command) var stdErrBuffer bytes.Buffer command.Stderr = &stdErrBuffer // Frames go to stderr rather than stdout if err := command.Run(); err == nil { @@ -81,7 +79,7 @@ func (g *GeneratorInfo) calculateFrameRate(videoStream *ffmpeg.FFProbeStream) er if numberOfFrames == 0 { numberOfFrames = ffmpeg.GetFrameFromRegex(stdErrString) } - if !utils.IsValidFloat64(framerate) { + if !isValidFloat64(framerate) { time := ffmpeg.GetTimeFromRegex(stdErrString) framerate = math.Round((float64(numberOfFrames)/time)*100) / 100 } @@ -89,7 +87,7 @@ func (g *GeneratorInfo) calculateFrameRate(videoStream *ffmpeg.FFProbeStream) er } // Something seriously wrong with this file - if numberOfFrames == 0 || !utils.IsValidFloat64(framerate) { + if numberOfFrames == 0 || !isValidFloat64(framerate) { logger.Errorf( "number of frames or framerate is 0. nb_frames <%s> framerate <%f> duration <%f>", videoStream.NbFrames, @@ -104,6 +102,11 @@ func (g *GeneratorInfo) calculateFrameRate(videoStream *ffmpeg.FFProbeStream) er return nil } +// isValidFloat64 ensures the given value is a valid number (not NaN) which is not equal to 0 +func isValidFloat64(value float64) bool { + return !math.IsNaN(value) && value != 0 +} + func (g *GeneratorInfo) configure() error { videoStream := g.VideoFile.VideoStream if videoStream == nil { diff --git a/pkg/manager/generator_interactive_heatmap_speed.go b/internal/manager/generator_interactive_heatmap_speed.go similarity index 100% rename from pkg/manager/generator_interactive_heatmap_speed.go rename to internal/manager/generator_interactive_heatmap_speed.go diff --git a/pkg/manager/generator_phash.go b/internal/manager/generator_phash.go similarity index 96% rename from pkg/manager/generator_phash.go rename to internal/manager/generator_phash.go index 367dc8a0c..5ee5695a5 100644 --- a/pkg/manager/generator_phash.go +++ b/internal/manager/generator_phash.go @@ -10,8 +10,8 @@ import ( "github.com/disintegration/imaging" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) type PhashGenerator struct { @@ -23,7 +23,7 @@ type PhashGenerator struct { } func NewPhashGenerator(videoFile ffmpeg.VideoFile, checksum string) (*PhashGenerator, error) { - exists, err := utils.FileExists(videoFile.Path) + exists, err := fsutil.FileExists(videoFile.Path) if !exists { return nil, err } diff --git a/pkg/manager/generator_preview.go b/internal/manager/generator_preview.go similarity index 94% rename from pkg/manager/generator_preview.go rename to internal/manager/generator_preview.go index 56ad6725c..1a3672aba 100644 --- a/pkg/manager/generator_preview.go +++ b/internal/manager/generator_preview.go @@ -7,8 +7,8 @@ import ( "path/filepath" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) type PreviewGenerator struct { @@ -28,7 +28,7 @@ type PreviewGenerator struct { } func NewPreviewGenerator(videoFile ffmpeg.VideoFile, videoChecksum string, videoFilename string, imageFilename string, outputDirectory string, generateVideo bool, generateImage bool, previewPreset string) (*PreviewGenerator, error) { - exists, err := utils.FileExists(videoFile.Path) + exists, err := fsutil.FileExists(videoFile.Path) if !exists { return nil, err } @@ -92,7 +92,7 @@ func (g *PreviewGenerator) generateConcatFile() error { func (g *PreviewGenerator) generateVideo(encoder *ffmpeg.Encoder, fallback bool) error { outputPath := filepath.Join(g.OutputDirectory, g.VideoFilename) - outputExists, _ := utils.FileExists(outputPath) + outputExists, _ := fsutil.FileExists(outputPath) if !g.Overwrite && outputExists { return nil } @@ -143,7 +143,7 @@ func (g *PreviewGenerator) generateVideo(encoder *ffmpeg.Encoder, fallback bool) func (g *PreviewGenerator) generateImage(encoder *ffmpeg.Encoder) error { outputPath := filepath.Join(g.OutputDirectory, g.ImageFilename) - outputExists, _ := utils.FileExists(outputPath) + outputExists, _ := fsutil.FileExists(outputPath) if !g.Overwrite && outputExists { return nil } @@ -153,7 +153,7 @@ func (g *PreviewGenerator) generateImage(encoder *ffmpeg.Encoder) error { if err := encoder.ScenePreviewVideoToImage(g.Info.VideoFile, 640, videoPreviewPath, tmpOutputPath); err != nil { return err } - if err := utils.SafeMove(tmpOutputPath, outputPath); err != nil { + if err := fsutil.SafeMove(tmpOutputPath, outputPath); err != nil { return err } logger.Debug("created video preview image: ", outputPath) diff --git a/pkg/manager/generator_sprite.go b/internal/manager/generator_sprite.go similarity index 97% rename from pkg/manager/generator_sprite.go rename to internal/manager/generator_sprite.go index 72c45e124..70622d832 100644 --- a/pkg/manager/generator_sprite.go +++ b/internal/manager/generator_sprite.go @@ -13,6 +13,7 @@ import ( "github.com/disintegration/imaging" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/utils" ) @@ -31,7 +32,7 @@ type SpriteGenerator struct { } func NewSpriteGenerator(videoFile ffmpeg.VideoFile, videoChecksum string, imageOutputPath string, vttOutputPath string, rows int, cols int) (*SpriteGenerator, error) { - exists, err := utils.FileExists(videoFile.Path) + exists, err := fsutil.FileExists(videoFile.Path) if !exists { return nil, err } @@ -205,11 +206,11 @@ func (g *SpriteGenerator) generateSpriteVTT(encoder *ffmpeg.Encoder) error { } func (g *SpriteGenerator) imageExists() bool { - exists, _ := utils.FileExists(g.ImageOutputPath) + exists, _ := fsutil.FileExists(g.ImageOutputPath) return exists } func (g *SpriteGenerator) vttExists() bool { - exists, _ := utils.FileExists(g.VTTOutputPath) + exists, _ := fsutil.FileExists(g.VTTOutputPath) return exists } diff --git a/pkg/manager/image.go b/internal/manager/image.go similarity index 94% rename from pkg/manager/image.go rename to internal/manager/image.go index 5c45c12c8..c7eb781f6 100644 --- a/pkg/manager/image.go +++ b/internal/manager/image.go @@ -2,10 +2,11 @@ package manager import ( "archive/zip" - "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/manager/config" "strings" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/logger" ) diff --git a/pkg/manager/import.go b/internal/manager/import.go similarity index 100% rename from pkg/manager/import.go rename to internal/manager/import.go diff --git a/pkg/manager/json_utils.go b/internal/manager/json_utils.go similarity index 96% rename from pkg/manager/json_utils.go rename to internal/manager/json_utils.go index 9a04e45cf..9a3330a61 100644 --- a/pkg/manager/json_utils.go +++ b/internal/manager/json_utils.go @@ -1,8 +1,8 @@ package manager import ( - "github.com/stashapp/stash/pkg/manager/jsonschema" - "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/models/paths" ) type jsonUtils struct { diff --git a/pkg/manager/manager.go b/internal/manager/manager.go similarity index 81% rename from pkg/manager/manager.go rename to internal/manager/manager.go index b313909b0..83be48067 100644 --- a/pkg/manager/manager.go +++ b/internal/manager/manager.go @@ -4,29 +4,36 @@ import ( "context" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "runtime/pprof" + "strings" "sync" "time" + "github.com/stashapp/stash/internal/desktop" + "github.com/stashapp/stash/internal/dlna" + "github.com/stashapp/stash/internal/log" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/database" - "github.com/stashapp/stash/pkg/dlna" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/session" "github.com/stashapp/stash/pkg/sqlite" "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/ui" ) type singleton struct { Config *config.Instance + Logger *log.Logger Paths *paths.Paths @@ -66,12 +73,12 @@ func Initialize() *singleton { panic(fmt.Sprintf("error initializing configuration: %s", err.Error())) } - initLog() + l := initLog() initProfiling(cfg.GetCPUProfilePath()) instance = &singleton{ Config: cfg, - JobManager: job.NewManager(), + Logger: l, DownloadStore: NewDownloadStore(), PluginCache: plugin.NewCache(cfg), @@ -80,6 +87,8 @@ func Initialize() *singleton { scanSubs: &subscriptionManager{}, } + instance.JobManager = initJobManager() + sceneServer := SceneServer{ TXNManager: instance.TxnManager, } @@ -127,6 +136,35 @@ func Initialize() *singleton { return instance } +func initJobManager() *job.Manager { + ret := job.NewManager() + + // desktop notifications + ctx := context.Background() + c := ret.Subscribe(context.Background()) + go func() { + for { + select { + case j := <-c.RemovedJob: + if instance.Config.GetNotificationsEnabled() { + cleanDesc := strings.TrimRight(j.Description, ".") + timeElapsed := j.EndTime.Sub(*j.StartTime) + + desktop.SendNotification("Task Finished", "Task \""+cleanDesc+"\" is finished in "+formatDuration(timeElapsed)+".") + } + case <-ctx.Done(): + return + } + } + }() + + return ret +} + +func formatDuration(t time.Duration) string { + return fmt.Sprintf("%02.f:%02.f:%02.f", t.Hours(), t.Minutes(), t.Seconds()) +} + func initSecurity(cfg *config.Instance) { if err := session.CheckExternalAccessTripwire(cfg); err != nil { session.LogExternalAccessError(*err) @@ -189,9 +227,13 @@ func initFFMPEG() error { return nil } -func initLog() { +func initLog() *log.Logger { config := config.GetInstance() - logger.Init(config.GetLogFile(), config.GetLogOut(), config.GetLogLevel()) + l := log.NewLogger() + l.Init(config.GetLogFile(), config.GetLogOut(), config.GetLogLevel()) + logger.Logger = l + + return l } // PostInit initialises the paths, caches and txnManager after the initial @@ -212,6 +254,8 @@ func (s *singleton) PostInit(ctx context.Context) error { } s.ScraperCache = instance.initScraperCache() + writeStashIcon() + go desktop.Start(instance, &FaviconProvider{uiBox: ui.UIBox}) // clear the downloads and tmp directories // #1021 - only clear these directories if the generated folder is non-empty @@ -219,10 +263,10 @@ func (s *singleton) PostInit(ctx context.Context) error { const deleteTimeout = 1 * time.Second utils.Timeout(func() { - if err := utils.EmptyDir(instance.Paths.Generated.Downloads); err != nil { + if err := fsutil.EmptyDir(instance.Paths.Generated.Downloads); err != nil { logger.Warnf("could not empty Downloads directory: %v", err) } - if err := utils.EmptyDir(instance.Paths.Generated.Tmp); err != nil { + if err := fsutil.EmptyDir(instance.Paths.Generated.Tmp); err != nil { logger.Warnf("could not empty Tmp directory: %v", err) } }, deleteTimeout, func(done chan struct{}) { @@ -243,6 +287,18 @@ func (s *singleton) PostInit(ctx context.Context) error { return nil } +func writeStashIcon() { + p := FaviconProvider{ + uiBox: ui.UIBox, + } + + iconPath := filepath.Join(instance.Config.GetConfigPath(), "icon.png") + err := ioutil.WriteFile(iconPath, p.GetFaviconPng(), 0644) + if err != nil { + logger.Errorf("Couldn't write icon file: %s", err.Error()) + } +} + // initScraperCache initializes a new scraper cache and returns it. func (s *singleton) initScraperCache() *scraper.Cache { ret, err := scraper.NewCache(config.GetInstance(), s.TxnManager) @@ -258,22 +314,22 @@ func (s *singleton) RefreshConfig() { s.Paths = paths.NewPaths(s.Config.GetGeneratedPath()) config := s.Config if config.Validate() == nil { - if err := utils.EnsureDir(s.Paths.Generated.Screenshots); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.Screenshots); err != nil { logger.Warnf("could not create directory for Screenshots: %v", err) } - if err := utils.EnsureDir(s.Paths.Generated.Vtt); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.Vtt); err != nil { logger.Warnf("could not create directory for VTT: %v", err) } - if err := utils.EnsureDir(s.Paths.Generated.Markers); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.Markers); err != nil { logger.Warnf("could not create directory for Markers: %v", err) } - if err := utils.EnsureDir(s.Paths.Generated.Transcodes); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.Transcodes); err != nil { logger.Warnf("could not create directory for Transcodes: %v", err) } - if err := utils.EnsureDir(s.Paths.Generated.Downloads); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.Downloads); err != nil { logger.Warnf("could not create directory for Downloads: %v", err) } - if err := utils.EnsureDir(s.Paths.Generated.InteractiveHeatmap); err != nil { + if err := fsutil.EnsureDir(s.Paths.Generated.InteractiveHeatmap); err != nil { logger.Warnf("could not create directory for Interactive Heatmaps: %v", err) } } @@ -287,7 +343,7 @@ func (s *singleton) RefreshScraperCache() { func setSetupDefaults(input *models.SetupInput) { if input.ConfigLocation == "" { - input.ConfigLocation = filepath.Join(utils.GetHomeDirectory(), ".stash", "config.yml") + input.ConfigLocation = filepath.Join(fsutil.GetHomeDirectory(), ".stash", "config.yml") } configDir := filepath.Dir(input.ConfigLocation) @@ -308,13 +364,13 @@ func (s *singleton) Setup(ctx context.Context, input models.SetupInput) error { // don't do anything if config is already set in the environment if !config.FileEnvSet() { configDir := filepath.Dir(input.ConfigLocation) - if exists, _ := utils.DirExists(configDir); !exists { + if exists, _ := fsutil.DirExists(configDir); !exists { if err := os.Mkdir(configDir, 0755); err != nil { return fmt.Errorf("error creating config directory: %v", err) } } - if err := utils.Touch(input.ConfigLocation); err != nil { + if err := fsutil.Touch(input.ConfigLocation); err != nil { return fmt.Errorf("error creating config file: %v", err) } @@ -323,7 +379,7 @@ func (s *singleton) Setup(ctx context.Context, input models.SetupInput) error { // create the generated directory if it does not exist if !c.HasOverride(config.Generated) { - if exists, _ := utils.DirExists(input.GeneratedLocation); !exists { + if exists, _ := fsutil.DirExists(input.GeneratedLocation); !exists { if err := os.Mkdir(input.GeneratedLocation, 0755); err != nil { return fmt.Errorf("error creating generated directory: %v", err) } diff --git a/pkg/manager/manager_tasks.go b/internal/manager/manager_tasks.go similarity index 97% rename from pkg/manager/manager_tasks.go rename to internal/manager/manager_tasks.go index 9713bfd42..b2cac4c32 100644 --- a/pkg/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -7,26 +7,26 @@ import ( "strconv" "sync" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func isGallery(pathname string) bool { gExt := config.GetInstance().GetGalleryExtensions() - return utils.MatchExtension(pathname, gExt) + return fsutil.MatchExtension(pathname, gExt) } func isVideo(pathname string) bool { vidExt := config.GetInstance().GetVideoExtensions() - return utils.MatchExtension(pathname, vidExt) + return fsutil.MatchExtension(pathname, vidExt) } func isImage(pathname string) bool { imgExt := config.GetInstance().GetImageExtensions() - return utils.MatchExtension(pathname, imgExt) + return fsutil.MatchExtension(pathname, imgExt) } func getScanPaths(inputPaths []string) []*models.StashConfig { diff --git a/pkg/manager/post_migrate.go b/internal/manager/post_migrate.go similarity index 100% rename from pkg/manager/post_migrate.go rename to internal/manager/post_migrate.go diff --git a/pkg/manager/running_streams.go b/internal/manager/running_streams.go similarity index 95% rename from pkg/manager/running_streams.go rename to internal/manager/running_streams.go index 433e6e4ac..1babc1e3a 100644 --- a/pkg/manager/running_streams.go +++ b/internal/manager/running_streams.go @@ -4,9 +4,10 @@ import ( "net/http" "sync" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -99,7 +100,7 @@ func (s *SceneServer) ServeScreenshot(scene *models.Scene, w http.ResponseWriter filepath := GetInstance().Paths.Scene.GetScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm())) // fall back to the scene image blob if the file isn't present - screenshotExists, _ := utils.FileExists(filepath) + screenshotExists, _ := fsutil.FileExists(filepath) if screenshotExists { http.ServeFile(w, r, filepath) } else { diff --git a/pkg/manager/scene.go b/internal/manager/scene.go similarity index 98% rename from pkg/manager/scene.go rename to internal/manager/scene.go index 4e5fbb82d..abc2c9a1f 100644 --- a/pkg/manager/scene.go +++ b/internal/manager/scene.go @@ -3,10 +3,10 @@ package manager import ( "fmt" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/ffmpeg" - "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func GetSceneFileContainer(scene *models.Scene) (ffmpeg.Container, error) { @@ -183,6 +183,6 @@ func HasTranscode(scene *models.Scene, fileNamingAlgo models.HashAlgorithm) bool } transcodePath := instance.Paths.Scene.GetTranscodePath(sceneHash) - ret, _ := utils.FileExists(transcodePath) + ret, _ := fsutil.FileExists(transcodePath) return ret } diff --git a/pkg/manager/screenshot.go b/internal/manager/screenshot.go similarity index 100% rename from pkg/manager/screenshot.go rename to internal/manager/screenshot.go diff --git a/pkg/manager/studio.go b/internal/manager/studio.go similarity index 100% rename from pkg/manager/studio.go rename to internal/manager/studio.go diff --git a/pkg/manager/subscribe.go b/internal/manager/subscribe.go similarity index 100% rename from pkg/manager/subscribe.go rename to internal/manager/subscribe.go diff --git a/pkg/manager/task.go b/internal/manager/task.go similarity index 100% rename from pkg/manager/task.go rename to internal/manager/task.go diff --git a/pkg/manager/task_autotag.go b/internal/manager/task_autotag.go similarity index 99% rename from pkg/manager/task_autotag.go rename to internal/manager/task_autotag.go index 1fc15b39a..0a5610722 100644 --- a/pkg/manager/task_autotag.go +++ b/internal/manager/task_autotag.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/stashapp/stash/pkg/autotag" + "github.com/stashapp/stash/internal/autotag" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" diff --git a/pkg/manager/task_clean.go b/internal/manager/task_clean.go similarity index 96% rename from pkg/manager/task_clean.go rename to internal/manager/task_clean.go index 213256541..c3b9c480d 100644 --- a/pkg/manager/task_clean.go +++ b/internal/manager/task_clean.go @@ -5,16 +5,16 @@ import ( "fmt" "path/filepath" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" - "github.com/stashapp/stash/pkg/utils" ) type cleanJob struct { @@ -285,7 +285,7 @@ func (j *cleanJob) shouldClean(path string) bool { // #1102 - clean anything in generated path generatedPath := config.GetInstance().GetGeneratedPath() - if !fileExists || getStashFromPath(path) == nil || utils.IsPathInDir(generatedPath, path) { + if !fileExists || getStashFromPath(path) == nil || fsutil.IsPathInDir(generatedPath, path) { logger.Infof("File not found. Marking to clean: \"%s\"", path) return true } @@ -305,7 +305,7 @@ func (j *cleanJob) shouldCleanScene(s *models.Scene) bool { } config := config.GetInstance() - if !utils.MatchExtension(s.Path, config.GetVideoExtensions()) { + if !fsutil.MatchExtension(s.Path, config.GetVideoExtensions()) { logger.Infof("File extension does not match video extensions. Marking to clean: \"%s\"", s.Path) return true } @@ -337,7 +337,7 @@ func (j *cleanJob) shouldCleanGallery(g *models.Gallery, qb models.ImageReader) config := config.GetInstance() if g.Zip { - if !utils.MatchExtension(path, config.GetGalleryExtensions()) { + if !fsutil.MatchExtension(path, config.GetGalleryExtensions()) { logger.Infof("File extension does not match gallery extensions. Marking to clean: \"%s\"", path) return true } @@ -379,7 +379,7 @@ func (j *cleanJob) shouldCleanImage(s *models.Image) bool { } config := config.GetInstance() - if !utils.MatchExtension(s.Path, config.GetImageExtensions()) { + if !fsutil.MatchExtension(s.Path, config.GetImageExtensions()) { logger.Infof("File extension does not match image extensions. Marking to clean: \"%s\"", s.Path) return true } @@ -490,7 +490,7 @@ func (j *cleanJob) deleteImage(ctx context.Context, imageID int) { func getStashFromPath(pathToCheck string) *models.StashConfig { for _, s := range config.GetInstance().GetStashPaths() { - if utils.IsPathInDir(s.Path, filepath.Dir(pathToCheck)) { + if fsutil.IsPathInDir(s.Path, filepath.Dir(pathToCheck)) { return s } } @@ -499,7 +499,7 @@ func getStashFromPath(pathToCheck string) *models.StashConfig { func getStashFromDirPath(pathToCheck string) *models.StashConfig { for _, s := range config.GetInstance().GetStashPaths() { - if utils.IsPathInDir(s.Path, pathToCheck) { + if fsutil.IsPathInDir(s.Path, pathToCheck) { return s } } diff --git a/pkg/manager/task_export.go b/internal/manager/task_export.go similarity index 93% rename from pkg/manager/task_export.go rename to internal/manager/task_export.go index fbfda537c..ca86a049f 100644 --- a/pkg/manager/task_export.go +++ b/internal/manager/task_export.go @@ -11,16 +11,20 @@ import ( "sync" "time" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/gallery" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/manager/jsonschema" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/movie" "github.com/stashapp/stash/pkg/performer" "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/tag" "github.com/stashapp/stash/pkg/utils" @@ -59,7 +63,7 @@ func newExportSpec(input *models.ExportObjectTypeInput) *exportSpec { return &exportSpec{} } - ids, _ := utils.StringSliceToIntSlice(input.Ids) + ids, _ := stringslice.StringSliceToIntSlice(input.Ids) ret := &exportSpec{ IDs: ids, @@ -112,7 +116,7 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) { } defer func() { - err := utils.RemoveDir(t.baseDir) + err := fsutil.RemoveDir(t.baseDir) if err != nil { logger.Errorf("error removing directory %s: %s", t.baseDir, err.Error()) } @@ -173,7 +177,7 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) { func (t *ExportTask) generateDownload() error { // zip the files and register a download link - if err := utils.EnsureDir(instance.Paths.Generated.Downloads); err != nil { + if err := fsutil.EnsureDir(instance.Paths.Generated.Downloads); err != nil { return err } z, err := os.CreateTemp(instance.Paths.Generated.Downloads, "export*.zip") @@ -187,7 +191,10 @@ func (t *ExportTask) generateDownload() error { return err } - t.DownloadHash = instance.DownloadStore.RegisterFile(z.Name(), "", false) + t.DownloadHash, err = instance.DownloadStore.RegisterFile(z.Name(), "", false) + if err != nil { + return fmt.Errorf("error registering file for download: %w", err) + } logger.Debugf("Generated zip file %s with hash %s", z.Name(), t.DownloadHash) return nil } @@ -285,7 +292,7 @@ func (t *ExportTask) populateMovieScenes(repo models.ReaderRepository) { } for _, s := range scenes { - t.scenes.IDs = utils.IntAppendUnique(t.scenes.IDs, s.ID) + t.scenes.IDs = intslice.IntAppendUnique(t.scenes.IDs, s.ID) } } } @@ -315,7 +322,7 @@ func (t *ExportTask) populateGalleryImages(repo models.ReaderRepository) { } for _, i := range images { - t.images.IDs = utils.IntAppendUnique(t.images.IDs, i.ID) + t.images.IDs = intslice.IntAppendUnique(t.images.IDs, i.ID) } } } @@ -425,26 +432,26 @@ func exportScene(wg *sync.WaitGroup, jobChan <-chan *models.Scene, repo models.R if t.includeDependencies { if s.StudioID.Valid { - t.studios.IDs = utils.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64)) + t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64)) } - t.galleries.IDs = utils.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(galleries)) + t.galleries.IDs = intslice.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(galleries)) tagIDs, err := scene.GetDependentTagIDs(tagReader, sceneMarkerReader, s) if err != nil { logger.Errorf("[scenes] <%s> error getting scene tags: %s", sceneHash, err.Error()) continue } - t.tags.IDs = utils.IntAppendUniques(t.tags.IDs, tagIDs) + t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tagIDs) movieIDs, err := scene.GetDependentMovieIDs(sceneReader, s) if err != nil { logger.Errorf("[scenes] <%s> error getting scene movies: %s", sceneHash, err.Error()) continue } - t.movies.IDs = utils.IntAppendUniques(t.movies.IDs, movieIDs) + t.movies.IDs = intslice.IntAppendUniques(t.movies.IDs, movieIDs) - t.performers.IDs = utils.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) + t.performers.IDs = intslice.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) } sceneJSON, err := t.json.getScene(sceneHash) @@ -547,12 +554,12 @@ func exportImage(wg *sync.WaitGroup, jobChan <-chan *models.Image, repo models.R if t.includeDependencies { if s.StudioID.Valid { - t.studios.IDs = utils.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64)) + t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64)) } - t.galleries.IDs = utils.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(imageGalleries)) - t.tags.IDs = utils.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) - t.performers.IDs = utils.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) + t.galleries.IDs = intslice.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(imageGalleries)) + t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) + t.performers.IDs = intslice.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) } imageJSON, err := t.json.getImage(imageHash) @@ -661,11 +668,11 @@ func exportGallery(wg *sync.WaitGroup, jobChan <-chan *models.Gallery, repo mode if t.includeDependencies { if g.StudioID.Valid { - t.studios.IDs = utils.IntAppendUnique(t.studios.IDs, int(g.StudioID.Int64)) + t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(g.StudioID.Int64)) } - t.tags.IDs = utils.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) - t.performers.IDs = utils.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) + t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) + t.performers.IDs = intslice.IntAppendUniques(t.performers.IDs, performer.GetIDs(performers)) } galleryJSON, err := t.json.getGallery(galleryHash) @@ -741,7 +748,7 @@ func (t *ExportTask) exportPerformer(wg *sync.WaitGroup, jobChan <-chan *models. newPerformerJSON.Tags = tag.GetNames(tags) if t.includeDependencies { - t.tags.IDs = utils.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) + t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags)) } performerJSON, err := t.json.getPerformer(p.Checksum) @@ -854,7 +861,7 @@ func (t *ExportTask) ExportTags(workers int, repo models.ReaderRepository) { logger.Progressf("[tags] %d of %d", index, len(tags)) // generate checksum on the fly by name, since we don't store it - checksum := utils.MD5FromString(tag.Name) + checksum := md5.FromString(tag.Name) t.Mappings.Tags = append(t.Mappings.Tags, jsonschema.PathNameMapping{Name: tag.Name, Checksum: checksum}) jobCh <- tag // feed workers @@ -880,7 +887,7 @@ func (t *ExportTask) exportTag(wg *sync.WaitGroup, jobChan <-chan *models.Tag, r } // generate checksum on the fly by name, since we don't store it - checksum := utils.MD5FromString(thisTag.Name) + checksum := md5.FromString(thisTag.Name) tagJSON, err := t.json.getTag(checksum) if err == nil && jsonschema.CompareJSON(*tagJSON, *newTagJSON) { @@ -950,7 +957,7 @@ func (t *ExportTask) exportMovie(wg *sync.WaitGroup, jobChan <-chan *models.Movi if t.includeDependencies { if m.StudioID.Valid { - t.studios.IDs = utils.IntAppendUnique(t.studios.IDs, int(m.StudioID.Int64)) + t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(m.StudioID.Int64)) } } diff --git a/pkg/manager/task_generate.go b/internal/manager/task_generate.go similarity index 96% rename from pkg/manager/task_generate.go rename to internal/manager/task_generate.go index 7aaec4fba..44bc63eec 100644 --- a/pkg/manager/task_generate.go +++ b/internal/manager/task_generate.go @@ -7,11 +7,12 @@ import ( "time" "github.com/remeh/sizedwaitgroup" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -57,11 +58,11 @@ func (j *GenerateJob) Execute(ctx context.Context, progress *job.Progress) { defer close(queue) var totals totalsGenerate - sceneIDs, err := utils.StringSliceToIntSlice(j.input.SceneIDs) + sceneIDs, err := stringslice.StringSliceToIntSlice(j.input.SceneIDs) if err != nil { logger.Error(err.Error()) } - markerIDs, err := utils.StringSliceToIntSlice(j.input.MarkerIDs) + markerIDs, err := stringslice.StringSliceToIntSlice(j.input.MarkerIDs) if err != nil { logger.Error(err.Error()) } diff --git a/pkg/manager/task_generate_interactive_heatmap_speed.go b/internal/manager/task_generate_interactive_heatmap_speed.go similarity index 89% rename from pkg/manager/task_generate_interactive_heatmap_speed.go rename to internal/manager/task_generate_interactive_heatmap_speed.go index ef12e5aac..4b2952d50 100644 --- a/pkg/manager/task_generate_interactive_heatmap_speed.go +++ b/internal/manager/task_generate_interactive_heatmap_speed.go @@ -5,9 +5,10 @@ import ( "database/sql" "fmt" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/scene" ) type GenerateInteractiveHeatmapSpeedTask struct { @@ -27,7 +28,7 @@ func (t *GenerateInteractiveHeatmapSpeedTask) Start(ctx context.Context) { } videoChecksum := t.Scene.GetHash(t.fileNamingAlgorithm) - funscriptPath := utils.GetFunscriptPath(t.Scene.Path) + funscriptPath := scene.GetFunscriptPath(t.Scene.Path) heatmapPath := instance.Paths.Scene.GetInteractiveHeatmapPath(videoChecksum) generator := NewInteractiveHeatmapSpeedGenerator(funscriptPath, heatmapPath) @@ -82,6 +83,6 @@ func (t *GenerateInteractiveHeatmapSpeedTask) doesHeatmapExist(sceneChecksum str return false } - imageExists, _ := utils.FileExists(instance.Paths.Scene.GetInteractiveHeatmapPath(sceneChecksum)) + imageExists, _ := fsutil.FileExists(instance.Paths.Scene.GetInteractiveHeatmapPath(sceneChecksum)) return imageExists } diff --git a/pkg/manager/task_generate_markers.go b/internal/manager/task_generate_markers.go similarity index 94% rename from pkg/manager/task_generate_markers.go rename to internal/manager/task_generate_markers.go index 94dcce751..e1d96f0eb 100644 --- a/pkg/manager/task_generate_markers.go +++ b/internal/manager/task_generate_markers.go @@ -7,9 +7,9 @@ import ( "strconv" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type GenerateMarkersTask struct { @@ -91,7 +91,7 @@ func (t *GenerateMarkersTask) generateSceneMarkers() { // Make the folder for the scenes markers markersFolder := filepath.Join(instance.Paths.Generated.Markers, sceneHash) - if err := utils.EnsureDir(markersFolder); err != nil { + if err := fsutil.EnsureDir(markersFolder); err != nil { logger.Warnf("could not create the markers folder (%v): %v", markersFolder, err) } @@ -130,7 +130,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene if err := encoder.SceneMarkerVideo(*videoFile, options); err != nil { logger.Errorf("[generator] failed to generate marker video: %s", err) } else { - _ = utils.SafeMove(options.OutputPath, videoPath) + _ = fsutil.SafeMove(options.OutputPath, videoPath) logger.Debug("created marker video: ", videoPath) } } @@ -143,7 +143,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene if err := encoder.SceneMarkerImage(*videoFile, options); err != nil { logger.Errorf("[generator] failed to generate marker image: %s", err) } else { - _ = utils.SafeMove(options.OutputPath, imagePath) + _ = fsutil.SafeMove(options.OutputPath, imagePath) logger.Debug("created marker image: ", imagePath) } } @@ -161,7 +161,7 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene if err := encoder.Screenshot(*videoFile, screenshotOptions); err != nil { logger.Errorf("[generator] failed to generate marker screenshot: %s", err) } else { - _ = utils.SafeMove(screenshotOptions.OutputPath, screenshotPath) + _ = fsutil.SafeMove(screenshotOptions.OutputPath, screenshotPath) logger.Debug("created marker screenshot: ", screenshotPath) } } @@ -213,7 +213,7 @@ func (t *GenerateMarkersTask) videoExists(sceneChecksum string, seconds int) boo } videoPath := instance.Paths.SceneMarkers.GetStreamPath(sceneChecksum, seconds) - videoExists, _ := utils.FileExists(videoPath) + videoExists, _ := fsutil.FileExists(videoPath) return videoExists } @@ -224,7 +224,7 @@ func (t *GenerateMarkersTask) imageExists(sceneChecksum string, seconds int) boo } imagePath := instance.Paths.SceneMarkers.GetStreamPreviewImagePath(sceneChecksum, seconds) - imageExists, _ := utils.FileExists(imagePath) + imageExists, _ := fsutil.FileExists(imagePath) return imageExists } @@ -235,7 +235,7 @@ func (t *GenerateMarkersTask) screenshotExists(sceneChecksum string, seconds int } screenshotPath := instance.Paths.SceneMarkers.GetStreamScreenshotPath(sceneChecksum, seconds) - screenshotExists, _ := utils.FileExists(screenshotPath) + screenshotExists, _ := fsutil.FileExists(screenshotPath) return screenshotExists } diff --git a/pkg/manager/task_generate_phash.go b/internal/manager/task_generate_phash.go similarity index 100% rename from pkg/manager/task_generate_phash.go rename to internal/manager/task_generate_phash.go diff --git a/pkg/manager/task_generate_preview.go b/internal/manager/task_generate_preview.go similarity index 89% rename from pkg/manager/task_generate_preview.go rename to internal/manager/task_generate_preview.go index 556c3fd68..3806fd221 100644 --- a/pkg/manager/task_generate_preview.go +++ b/internal/manager/task_generate_preview.go @@ -4,10 +4,10 @@ import ( "context" "fmt" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type GeneratePreviewTask struct { @@ -74,7 +74,7 @@ func (t *GeneratePreviewTask) doesVideoPreviewExist(sceneChecksum string) bool { return false } - videoExists, _ := utils.FileExists(instance.Paths.Scene.GetStreamPreviewPath(sceneChecksum)) + videoExists, _ := fsutil.FileExists(instance.Paths.Scene.GetStreamPreviewPath(sceneChecksum)) return videoExists } @@ -83,7 +83,7 @@ func (t *GeneratePreviewTask) doesImagePreviewExist(sceneChecksum string) bool { return false } - imageExists, _ := utils.FileExists(instance.Paths.Scene.GetStreamPreviewImagePath(sceneChecksum)) + imageExists, _ := fsutil.FileExists(instance.Paths.Scene.GetStreamPreviewImagePath(sceneChecksum)) return imageExists } diff --git a/pkg/manager/task_generate_screenshot.go b/internal/manager/task_generate_screenshot.go similarity index 100% rename from pkg/manager/task_generate_screenshot.go rename to internal/manager/task_generate_screenshot.go diff --git a/pkg/manager/task_generate_sprite.go b/internal/manager/task_generate_sprite.go similarity index 87% rename from pkg/manager/task_generate_sprite.go rename to internal/manager/task_generate_sprite.go index d47b225f1..cf4ae4cd3 100644 --- a/pkg/manager/task_generate_sprite.go +++ b/internal/manager/task_generate_sprite.go @@ -4,9 +4,9 @@ import ( "context" "fmt" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type GenerateSpriteTask struct { @@ -59,7 +59,7 @@ func (t *GenerateSpriteTask) doesSpriteExist(sceneChecksum string) bool { return false } - imageExists, _ := utils.FileExists(instance.Paths.Scene.GetSpriteImageFilePath(sceneChecksum)) - vttExists, _ := utils.FileExists(instance.Paths.Scene.GetSpriteVttFilePath(sceneChecksum)) + imageExists, _ := fsutil.FileExists(instance.Paths.Scene.GetSpriteImageFilePath(sceneChecksum)) + vttExists, _ := fsutil.FileExists(instance.Paths.Scene.GetSpriteVttFilePath(sceneChecksum)) return imageExists && vttExists } diff --git a/pkg/manager/task_identify.go b/internal/manager/task_identify.go similarity index 97% rename from pkg/manager/task_identify.go rename to internal/manager/task_identify.go index aebbdbf9e..54bb063e4 100644 --- a/pkg/manager/task_identify.go +++ b/internal/manager/task_identify.go @@ -6,14 +6,14 @@ import ( "fmt" "strconv" - "github.com/stashapp/stash/pkg/identify" + "github.com/stashapp/stash/internal/identify" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/scraper/stashbox" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) var ErrInput = errors.New("invalid request input") @@ -57,7 +57,7 @@ func (j *IdentifyJob) Execute(ctx context.Context, progress *job.Progress) { return j.identifyAllScenes(ctx, r, sources) } - sceneIDs, err := utils.StringSliceToIntSlice(j.input.SceneIDs) + sceneIDs, err := stringslice.StringSliceToIntSlice(j.input.SceneIDs) if err != nil { return fmt.Errorf("invalid scene IDs: %w", err) } diff --git a/pkg/manager/task_import.go b/internal/manager/task_import.go similarity index 98% rename from pkg/manager/task_import.go rename to internal/manager/task_import.go index 70b494d7c..51424a14a 100644 --- a/pkg/manager/task_import.go +++ b/internal/manager/task_import.go @@ -11,20 +11,20 @@ import ( "path/filepath" "time" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/manager/jsonschema" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/movie" "github.com/stashapp/stash/pkg/performer" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/studio" "github.com/stashapp/stash/pkg/tag" - "github.com/stashapp/stash/pkg/utils" ) type ImportTask struct { @@ -82,7 +82,7 @@ func (t *ImportTask) GetDescription() string { func (t *ImportTask) Start(ctx context.Context) { if t.TmpZip != "" { defer func() { - err := utils.RemoveDir(t.BaseDir) + err := fsutil.RemoveDir(t.BaseDir) if err != nil { logger.Errorf("error removing directory %s: %s", t.BaseDir, err.Error()) } diff --git a/pkg/manager/task_migrate_hash.go b/internal/manager/task_migrate_hash.go similarity index 100% rename from pkg/manager/task_migrate_hash.go rename to internal/manager/task_migrate_hash.go diff --git a/pkg/manager/task_plugin.go b/internal/manager/task_plugin.go similarity index 100% rename from pkg/manager/task_plugin.go rename to internal/manager/task_plugin.go diff --git a/pkg/manager/task_scan.go b/internal/manager/task_scan.go similarity index 93% rename from pkg/manager/task_scan.go rename to internal/manager/task_scan.go index 80d874ac7..cfee44564 100644 --- a/pkg/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -10,10 +10,11 @@ import ( "github.com/remeh/sizedwaitgroup" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" ) @@ -154,7 +155,7 @@ func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, s wg := sizedwaitgroup.New(parallelTasks) for _, sp := range paths { - csFs, er := utils.IsFsPathCaseSensitive(sp.Path) + csFs, er := fsutil.IsFsPathCaseSensitive(sp.Path) if er != nil { logger.Warnf("Cannot determine fs case sensitivity: %s", er.Error()) } @@ -220,17 +221,17 @@ func (j *ScanJob) doesPathExist(path string) bool { ret := false txnErr := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { switch { - case utils.MatchExtension(path, gExt): + case fsutil.MatchExtension(path, gExt): g, _ := r.Gallery().FindByPath(path) if g != nil { ret = true } - case utils.MatchExtension(path, vidExt): + case fsutil.MatchExtension(path, vidExt): s, _ := r.Scene().FindByPath(path) if s != nil { ret = true } - case utils.MatchExtension(path, imgExt): + case fsutil.MatchExtension(path, imgExt): i, _ := r.Image().FindByPath(path) if i != nil { ret = true @@ -366,7 +367,7 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error { generatedPath := config.GetGeneratedPath() - return utils.SymWalk(s.Path, func(path string, info os.FileInfo, err error) error { + return fsutil.SymWalk(s.Path, func(path string, info os.FileInfo, err error) error { if err != nil { logger.Warnf("error scanning %s: %s", path, err.Error()) return nil @@ -374,7 +375,7 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error { if info.IsDir() { // #1102 - ignore files in generated path - if utils.IsPathInDir(generatedPath, path) { + if fsutil.IsPathInDir(generatedPath, path) { return filepath.SkipDir } @@ -388,12 +389,12 @@ func walkFilesToScan(s *models.StashConfig, f filepath.WalkFunc) error { return nil } - if !s.ExcludeVideo && utils.MatchExtension(path, vidExt) && !matchFileRegex(path, excludeVidRegex) { + if !s.ExcludeVideo && fsutil.MatchExtension(path, vidExt) && !matchFileRegex(path, excludeVidRegex) { return f(path, info, err) } if !s.ExcludeImage { - if (utils.MatchExtension(path, imgExt) || utils.MatchExtension(path, gExt)) && !matchFileRegex(path, excludeImgRegex) { + if (fsutil.MatchExtension(path, imgExt) || fsutil.MatchExtension(path, gExt)) && !matchFileRegex(path, excludeImgRegex) { return f(path, info, err) } } diff --git a/pkg/manager/task_scan_gallery.go b/internal/manager/task_scan_gallery.go similarity index 98% rename from pkg/manager/task_scan_gallery.go rename to internal/manager/task_scan_gallery.go index 751e6a0f3..9a8887db8 100644 --- a/pkg/manager/task_scan_gallery.go +++ b/internal/manager/task_scan_gallery.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/remeh/sizedwaitgroup" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/gallery" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" ) diff --git a/pkg/manager/task_scan_image.go b/internal/manager/task_scan_image.go similarity index 92% rename from pkg/manager/task_scan_image.go rename to internal/manager/task_scan_image.go index d48e5f5c7..ab35d5e92 100644 --- a/pkg/manager/task_scan_image.go +++ b/internal/manager/task_scan_image.go @@ -6,14 +6,15 @@ import ( "path/filepath" "time" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/gallery" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" - "github.com/stashapp/stash/pkg/utils" ) func (t *ScanTask) scanImage() { @@ -98,7 +99,7 @@ func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.Galler } if g == nil { - checksum := utils.MD5FromString(path) + checksum := md5.FromString(path) // create the gallery currentTime := time.Now() @@ -112,7 +113,7 @@ func (t *ScanTask) associateImageWithFolderGallery(imageID int, qb models.Galler CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, Title: sql.NullString{ - String: utils.GetNameFromPath(path, false), + String: fsutil.GetNameFromPath(path, false), Valid: true, }, } @@ -138,7 +139,7 @@ func (t *ScanTask) generateThumbnail(i *models.Image) { } thumbPath := GetInstance().Paths.Generated.GetThumbnailPath(i.Checksum, models.DefaultGthumbWidth) - exists, _ := utils.FileExists(thumbPath) + exists, _ := fsutil.FileExists(thumbPath) if exists { return } @@ -158,7 +159,7 @@ func (t *ScanTask) generateThumbnail(i *models.Image) { return } - err = utils.WriteFile(thumbPath, data) + err = fsutil.WriteFile(thumbPath, data) if err != nil { logger.Errorf("error writing thumbnail for image %s: %s", i.Path, err) } diff --git a/pkg/manager/task_scan_scene.go b/internal/manager/task_scan_scene.go similarity index 100% rename from pkg/manager/task_scan_scene.go rename to internal/manager/task_scan_scene.go diff --git a/pkg/manager/task_stash_box_tag.go b/internal/manager/task_stash_box_tag.go similarity index 98% rename from pkg/manager/task_stash_box_tag.go rename to internal/manager/task_stash_box_tag.go index f1722cff6..f13b28119 100644 --- a/pkg/manager/task_stash_box_tag.go +++ b/internal/manager/task_stash_box_tag.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper/stashbox" @@ -135,7 +136,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag(ctx context.Context) { if excluded["name"] && performer.Name != nil { value := sql.NullString{String: *performer.Name, Valid: true} partial.Name = &value - checksum := utils.MD5FromString(*performer.Name) + checksum := md5.FromString(*performer.Name) partial.Checksum = &checksum } if performer.Piercings != nil && !excluded["piercings"] { @@ -199,7 +200,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag(ctx context.Context) { Aliases: getNullString(performer.Aliases), Birthdate: getDate(performer.Birthdate), CareerLength: getNullString(performer.CareerLength), - Checksum: utils.MD5FromString(*performer.Name), + Checksum: md5.FromString(*performer.Name), Country: getNullString(performer.Country), CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, Ethnicity: getNullString(performer.Ethnicity), diff --git a/pkg/manager/task_transcode.go b/internal/manager/task_transcode.go similarity index 94% rename from pkg/manager/task_transcode.go rename to internal/manager/task_transcode.go index 4530f851d..a051274dd 100644 --- a/pkg/manager/task_transcode.go +++ b/internal/manager/task_transcode.go @@ -4,11 +4,11 @@ import ( "context" "fmt" + "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/ffmpeg" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type GenerateTranscodeTask struct { @@ -86,7 +86,7 @@ func (t *GenerateTranscodeTask) Start(ctc context.Context) { } } - if err := utils.SafeMove(outputPath, instance.Paths.Scene.GetTranscodePath(sceneHash)); err != nil { + if err := fsutil.SafeMove(outputPath, instance.Paths.Scene.GetTranscodePath(sceneHash)); err != nil { logger.Errorf("[transcode] error generating transcode: %s", err.Error()) return } diff --git a/pkg/static/embed.go b/internal/static/embed.go similarity index 100% rename from pkg/static/embed.go rename to internal/static/embed.go diff --git a/pkg/static/performer/NoName01.png b/internal/static/performer/NoName01.png similarity index 100% rename from pkg/static/performer/NoName01.png rename to internal/static/performer/NoName01.png diff --git a/pkg/static/performer/NoName02.png b/internal/static/performer/NoName02.png similarity index 100% rename from pkg/static/performer/NoName02.png rename to internal/static/performer/NoName02.png diff --git a/pkg/static/performer/NoName03.png b/internal/static/performer/NoName03.png similarity index 100% rename from pkg/static/performer/NoName03.png rename to internal/static/performer/NoName03.png diff --git a/pkg/static/performer/NoName04.png b/internal/static/performer/NoName04.png similarity index 100% rename from pkg/static/performer/NoName04.png rename to internal/static/performer/NoName04.png diff --git a/pkg/static/performer/NoName05.png b/internal/static/performer/NoName05.png similarity index 100% rename from pkg/static/performer/NoName05.png rename to internal/static/performer/NoName05.png diff --git a/pkg/static/performer/NoName06.png b/internal/static/performer/NoName06.png similarity index 100% rename from pkg/static/performer/NoName06.png rename to internal/static/performer/NoName06.png diff --git a/pkg/static/performer/NoName07.png b/internal/static/performer/NoName07.png similarity index 100% rename from pkg/static/performer/NoName07.png rename to internal/static/performer/NoName07.png diff --git a/pkg/static/performer/NoName08.png b/internal/static/performer/NoName08.png similarity index 100% rename from pkg/static/performer/NoName08.png rename to internal/static/performer/NoName08.png diff --git a/pkg/static/performer/NoName09.png b/internal/static/performer/NoName09.png similarity index 100% rename from pkg/static/performer/NoName09.png rename to internal/static/performer/NoName09.png diff --git a/pkg/static/performer/NoName10.png b/internal/static/performer/NoName10.png similarity index 100% rename from pkg/static/performer/NoName10.png rename to internal/static/performer/NoName10.png diff --git a/pkg/static/performer/NoName11.png b/internal/static/performer/NoName11.png similarity index 100% rename from pkg/static/performer/NoName11.png rename to internal/static/performer/NoName11.png diff --git a/pkg/static/performer/NoName12.png b/internal/static/performer/NoName12.png similarity index 100% rename from pkg/static/performer/NoName12.png rename to internal/static/performer/NoName12.png diff --git a/pkg/static/performer/NoName13.png b/internal/static/performer/NoName13.png similarity index 100% rename from pkg/static/performer/NoName13.png rename to internal/static/performer/NoName13.png diff --git a/pkg/static/performer/NoName14.png b/internal/static/performer/NoName14.png similarity index 100% rename from pkg/static/performer/NoName14.png rename to internal/static/performer/NoName14.png diff --git a/pkg/static/performer/NoName15.png b/internal/static/performer/NoName15.png similarity index 100% rename from pkg/static/performer/NoName15.png rename to internal/static/performer/NoName15.png diff --git a/pkg/static/performer/NoName16.png b/internal/static/performer/NoName16.png similarity index 100% rename from pkg/static/performer/NoName16.png rename to internal/static/performer/NoName16.png diff --git a/pkg/static/performer/NoName17.png b/internal/static/performer/NoName17.png similarity index 100% rename from pkg/static/performer/NoName17.png rename to internal/static/performer/NoName17.png diff --git a/pkg/static/performer/NoName18.png b/internal/static/performer/NoName18.png similarity index 100% rename from pkg/static/performer/NoName18.png rename to internal/static/performer/NoName18.png diff --git a/pkg/static/performer/NoName19.png b/internal/static/performer/NoName19.png similarity index 100% rename from pkg/static/performer/NoName19.png rename to internal/static/performer/NoName19.png diff --git a/pkg/static/performer/NoName20.png b/internal/static/performer/NoName20.png similarity index 100% rename from pkg/static/performer/NoName20.png rename to internal/static/performer/NoName20.png diff --git a/pkg/static/performer/NoName21.png b/internal/static/performer/NoName21.png similarity index 100% rename from pkg/static/performer/NoName21.png rename to internal/static/performer/NoName21.png diff --git a/pkg/static/performer/NoName22.png b/internal/static/performer/NoName22.png similarity index 100% rename from pkg/static/performer/NoName22.png rename to internal/static/performer/NoName22.png diff --git a/pkg/static/performer/NoName23.png b/internal/static/performer/NoName23.png similarity index 100% rename from pkg/static/performer/NoName23.png rename to internal/static/performer/NoName23.png diff --git a/pkg/static/performer/NoName24.png b/internal/static/performer/NoName24.png similarity index 100% rename from pkg/static/performer/NoName24.png rename to internal/static/performer/NoName24.png diff --git a/pkg/static/performer/NoName25.png b/internal/static/performer/NoName25.png similarity index 100% rename from pkg/static/performer/NoName25.png rename to internal/static/performer/NoName25.png diff --git a/pkg/static/performer/NoName26.png b/internal/static/performer/NoName26.png similarity index 100% rename from pkg/static/performer/NoName26.png rename to internal/static/performer/NoName26.png diff --git a/pkg/static/performer/NoName27.png b/internal/static/performer/NoName27.png similarity index 100% rename from pkg/static/performer/NoName27.png rename to internal/static/performer/NoName27.png diff --git a/pkg/static/performer/NoName28.png b/internal/static/performer/NoName28.png similarity index 100% rename from pkg/static/performer/NoName28.png rename to internal/static/performer/NoName28.png diff --git a/pkg/static/performer/NoName29.png b/internal/static/performer/NoName29.png similarity index 100% rename from pkg/static/performer/NoName29.png rename to internal/static/performer/NoName29.png diff --git a/pkg/static/performer/NoName30.png b/internal/static/performer/NoName30.png similarity index 100% rename from pkg/static/performer/NoName30.png rename to internal/static/performer/NoName30.png diff --git a/pkg/static/performer/NoName31.png b/internal/static/performer/NoName31.png similarity index 100% rename from pkg/static/performer/NoName31.png rename to internal/static/performer/NoName31.png diff --git a/pkg/static/performer/NoName32.png b/internal/static/performer/NoName32.png similarity index 100% rename from pkg/static/performer/NoName32.png rename to internal/static/performer/NoName32.png diff --git a/pkg/static/performer/NoName33.png b/internal/static/performer/NoName33.png similarity index 100% rename from pkg/static/performer/NoName33.png rename to internal/static/performer/NoName33.png diff --git a/pkg/static/performer/NoName34.png b/internal/static/performer/NoName34.png similarity index 100% rename from pkg/static/performer/NoName34.png rename to internal/static/performer/NoName34.png diff --git a/pkg/static/performer/NoName35.png b/internal/static/performer/NoName35.png similarity index 100% rename from pkg/static/performer/NoName35.png rename to internal/static/performer/NoName35.png diff --git a/pkg/static/performer/NoName36.png b/internal/static/performer/NoName36.png similarity index 100% rename from pkg/static/performer/NoName36.png rename to internal/static/performer/NoName36.png diff --git a/pkg/static/performer/NoName37.png b/internal/static/performer/NoName37.png similarity index 100% rename from pkg/static/performer/NoName37.png rename to internal/static/performer/NoName37.png diff --git a/pkg/static/performer/NoName38.png b/internal/static/performer/NoName38.png similarity index 100% rename from pkg/static/performer/NoName38.png rename to internal/static/performer/NoName38.png diff --git a/pkg/static/performer/NoName39.png b/internal/static/performer/NoName39.png similarity index 100% rename from pkg/static/performer/NoName39.png rename to internal/static/performer/NoName39.png diff --git a/pkg/static/performer/NoName40.png b/internal/static/performer/NoName40.png similarity index 100% rename from pkg/static/performer/NoName40.png rename to internal/static/performer/NoName40.png diff --git a/pkg/static/performer_male/noname_male_01.jpg b/internal/static/performer_male/noname_male_01.jpg similarity index 100% rename from pkg/static/performer_male/noname_male_01.jpg rename to internal/static/performer_male/noname_male_01.jpg diff --git a/pkg/static/performer_male/noname_male_02.jpg b/internal/static/performer_male/noname_male_02.jpg similarity index 100% rename from pkg/static/performer_male/noname_male_02.jpg rename to internal/static/performer_male/noname_male_02.jpg diff --git a/pkg/database/database.go b/pkg/database/database.go index 2dadcdb4b..3300b11c9 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -16,8 +16,8 @@ import ( "github.com/jmoiron/sqlx" sqlite3 "github.com/mattn/go-sqlite3" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) var DB *sqlx.DB @@ -143,7 +143,7 @@ func Reset(databasePath string) error { // remove the -shm, -wal files ( if they exist ) walFiles := []string{databasePath + "-shm", databasePath + "-wal"} for _, wf := range walFiles { - if exists, _ := utils.FileExists(wf); exists { + if exists, _ := fsutil.FileExists(wf); exists { err = os.Remove(wf) if err != nil { return errors.New("Error removing database: " + err.Error()) diff --git a/pkg/exec/command.go b/pkg/exec/command.go new file mode 100644 index 000000000..366de9215 --- /dev/null +++ b/pkg/exec/command.go @@ -0,0 +1,11 @@ +// Package exec provides functions that wrap os/exec functions. These functions prevent external commands from opening windows on the Windows platform. +package exec + +import "os/exec" + +// Command wraps the exec.Command function, preventing Windows from opening a window when starting. +func Command(name string, arg ...string) *exec.Cmd { + ret := exec.Command(name, arg...) + hideExecShell(ret) + return ret +} diff --git a/pkg/exec/shell_nonwindows.go b/pkg/exec/shell_nonwindows.go new file mode 100644 index 000000000..9d30d015b --- /dev/null +++ b/pkg/exec/shell_nonwindows.go @@ -0,0 +1,10 @@ +//go:build linux || darwin || !windows +// +build linux darwin !windows + +package exec + +import "os/exec" + +// hideExecShell does nothing on non-Windows platforms. +func hideExecShell(cmd *exec.Cmd) { +} diff --git a/pkg/exec/shell_windows.go b/pkg/exec/shell_windows.go new file mode 100644 index 000000000..9f4b551c0 --- /dev/null +++ b/pkg/exec/shell_windows.go @@ -0,0 +1,16 @@ +//go:build windows +// +build windows + +package exec + +import ( + "os/exec" + "syscall" + + "golang.org/x/sys/windows" +) + +// hideExecShell hides the windows when executing on Windows. +func hideExecShell(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.DETACHED_PROCESS} +} diff --git a/pkg/ffmpeg/downloader.go b/pkg/ffmpeg/downloader.go index bea7105c0..173dd26c8 100644 --- a/pkg/ffmpeg/downloader.go +++ b/pkg/ffmpeg/downloader.go @@ -13,9 +13,9 @@ import ( "runtime" "strings" - "github.com/stashapp/stash/pkg/desktop" + stashExec "github.com/stashapp/stash/pkg/exec" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) func GetPaths(paths []string) (string, string) { @@ -29,10 +29,10 @@ func GetPaths(paths []string) (string, string) { // Check if ffmpeg exists in the config directory if ffmpegPath == "" { - ffmpegPath = utils.FindInPaths(paths, getFFMPEGFilename()) + ffmpegPath = fsutil.FindInPaths(paths, getFFMPEGFilename()) } if ffprobePath == "" { - ffprobePath = utils.FindInPaths(paths, getFFProbeFilename()) + ffprobePath = fsutil.FindInPaths(paths, getFFProbeFilename()) } return ffmpegPath, ffprobePath @@ -40,7 +40,7 @@ func GetPaths(paths []string) (string, string) { func Download(ctx context.Context, configDirectory string) error { for _, url := range getFFMPEGURL() { - err := DownloadSingle(ctx, configDirectory, url) + err := downloadSingle(ctx, configDirectory, url) if err != nil { return err } @@ -80,7 +80,7 @@ func (r *progressReader) Read(p []byte) (int, error) { return read, err } -func DownloadSingle(ctx context.Context, configDirectory, url string) error { +func downloadSingle(ctx context.Context, configDirectory, url string) error { if url == "" { return fmt.Errorf("no ffmpeg url for this platform") } @@ -203,8 +203,7 @@ func pathBinaryHasCorrectFlags() bool { if err != nil { return false } - cmd := exec.Command(ffmpegPath) - desktop.HideExecShell(cmd) + cmd := stashExec.Command(ffmpegPath) bytes, _ := cmd.CombinedOutput() output := string(bytes) hasOpus := strings.Contains(output, "--enable-libopus") diff --git a/pkg/ffmpeg/encoder.go b/pkg/ffmpeg/encoder.go index f66583912..97676713f 100644 --- a/pkg/ffmpeg/encoder.go +++ b/pkg/ffmpeg/encoder.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/stashapp/stash/pkg/desktop" + stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" ) @@ -79,7 +79,7 @@ func KillRunningEncoders(path string) { // FFmpeg runner with progress output, used for transcodes func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, error) { - cmd := exec.Command(string(*e), args...) + cmd := stashExec.Command(string(*e), args...) stderr, err := cmd.StderrPipe() if err != nil { @@ -91,7 +91,6 @@ func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, er logger.Error("FFMPEG stdout not available: " + err.Error()) } - desktop.HideExecShell(cmd) if err = cmd.Start(); err != nil { return "", err } @@ -136,14 +135,13 @@ func (e *Encoder) runTranscode(probeResult VideoFile, args []string) (string, er } func (e *Encoder) run(sourcePath string, args []string, stdin io.Reader) (string, error) { - cmd := exec.Command(string(*e), args...) + cmd := stashExec.Command(string(*e), args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr cmd.Stdin = stdin - desktop.HideExecShell(cmd) if err := cmd.Start(); err != nil { return "", err } diff --git a/pkg/ffmpeg/encoder_scene_preview_chunk.go b/pkg/ffmpeg/encoder_scene_preview_chunk.go index 92d660b94..6adaddfdf 100644 --- a/pkg/ffmpeg/encoder_scene_preview_chunk.go +++ b/pkg/ffmpeg/encoder_scene_preview_chunk.go @@ -3,8 +3,6 @@ package ffmpeg import ( "fmt" "strconv" - - "github.com/stashapp/stash/pkg/utils" ) type ScenePreviewChunkOptions struct { @@ -97,7 +95,7 @@ func (e *Encoder) ScenePreviewVideoChunkCombine(probeResult VideoFile, concatFil args := []string{ "-v", "error", "-f", "concat", - "-i", utils.FixWindowsPath(concatFilePath), + "-i", concatFilePath, "-y", "-c", "copy", outputPath, diff --git a/pkg/ffmpeg/ffprobe.go b/pkg/ffmpeg/ffprobe.go index acb4c2165..3f454874b 100644 --- a/pkg/ffmpeg/ffprobe.go +++ b/pkg/ffmpeg/ffprobe.go @@ -5,13 +5,12 @@ import ( "fmt" "math" "os" - "os/exec" "path/filepath" "strconv" "strings" "time" - "github.com/stashapp/stash/pkg/desktop" + "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" ) @@ -100,7 +99,7 @@ func MatchContainer(format string, filePath string) Container { // match ffprobe container := FfprobeToContainer[format] if container == Matroska { - container = MagicContainer(filePath) // use magic number instead of ffprobe for matroska,webm + container = magicContainer(filePath) // use magic number instead of ffprobe for matroska,webm } if container == "" { // if format is not in our Container list leave it as ffprobes reported format_name container = Container(format) @@ -108,7 +107,7 @@ func MatchContainer(format string, filePath string) Container { // match ffprobe return container } -func IsValidCodec(codecName string, supportedCodecs []string) bool { +func isValidCodec(codecName string, supportedCodecs []string) bool { for _, c := range supportedCodecs { if c == codecName { return true @@ -117,8 +116,7 @@ func IsValidCodec(codecName string, supportedCodecs []string) bool { return false } -func IsValidAudio(audio AudioCodec, validCodecs []AudioCodec) bool { - +func isValidAudio(audio AudioCodec, validCodecs []AudioCodec) bool { // if audio codec is missing or unsupported by ffmpeg we can't do anything about it // report it as valid so that the file can at least be streamed directly if the video codec is supported if audio == MissingUnsupported { @@ -137,17 +135,17 @@ func IsValidAudio(audio AudioCodec, validCodecs []AudioCodec) bool { func IsValidAudioForContainer(audio AudioCodec, format Container) bool { switch format { case Matroska: - return IsValidAudio(audio, validAudioForMkv) + return isValidAudio(audio, validAudioForMkv) case Webm: - return IsValidAudio(audio, validAudioForWebm) + return isValidAudio(audio, validAudioForWebm) case Mp4: - return IsValidAudio(audio, validAudioForMp4) + return isValidAudio(audio, validAudioForMp4) } return false } -func IsValidForContainer(format Container, validContainers []Container) bool { +func isValidForContainer(format Container, validContainers []Container) bool { for _, fmt := range validContainers { if fmt == format { return true @@ -156,36 +154,36 @@ func IsValidForContainer(format Container, validContainers []Container) bool { return false } -// IsValidCombo checks if a codec/container combination is valid. +// isValidCombo checks if a codec/container combination is valid. // Returns true on validity, false otherwise -func IsValidCombo(codecName string, format Container, supportedVideoCodecs []string) bool { - supportMKV := IsValidCodec(Mkv, supportedVideoCodecs) - supportHEVC := IsValidCodec(Hevc, supportedVideoCodecs) +func isValidCombo(codecName string, format Container, supportedVideoCodecs []string) bool { + supportMKV := isValidCodec(Mkv, supportedVideoCodecs) + supportHEVC := isValidCodec(Hevc, supportedVideoCodecs) switch codecName { case H264: if supportMKV { - return IsValidForContainer(format, validForH264Mkv) + return isValidForContainer(format, validForH264Mkv) } - return IsValidForContainer(format, validForH264) + return isValidForContainer(format, validForH264) case H265: if supportMKV { - return IsValidForContainer(format, validForH265Mkv) + return isValidForContainer(format, validForH265Mkv) } - return IsValidForContainer(format, validForH265) + return isValidForContainer(format, validForH265) case Vp8: - return IsValidForContainer(format, validForVp8) + return isValidForContainer(format, validForVp8) case Vp9: if supportMKV { - return IsValidForContainer(format, validForVp9Mkv) + return isValidForContainer(format, validForVp9Mkv) } - return IsValidForContainer(format, validForVp9) + return isValidForContainer(format, validForVp9) case Hevc: if supportHEVC { if supportMKV { - return IsValidForContainer(format, validForHevcMkv) + return isValidForContainer(format, validForHevcMkv) } - return IsValidForContainer(format, validForHevc) + return isValidForContainer(format, validForHevc) } } return false @@ -195,7 +193,7 @@ func IsStreamable(videoCodec string, audioCodec AudioCodec, container Container) supportedVideoCodecs := DefaultSupportedCodecs // check if the video codec matches the supported codecs - return IsValidCodec(videoCodec, supportedVideoCodecs) && IsValidCombo(videoCodec, container, supportedVideoCodecs) && IsValidAudioForContainer(audioCodec, container) + return isValidCodec(videoCodec, supportedVideoCodecs) && isValidCombo(videoCodec, container, supportedVideoCodecs) && IsValidAudioForContainer(audioCodec, container) } type VideoFile struct { @@ -231,7 +229,6 @@ type FFProbe string func (f *FFProbe) NewVideoFile(videoPath string, stripExt bool) (*VideoFile, error) { args := []string{"-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", "-show_error", videoPath} cmd := exec.Command(string(*f), args...) - desktop.HideExecShell(cmd) out, err := cmd.Output() if err != nil { @@ -300,13 +297,13 @@ func parse(filePath string, probeJSON *FFProbeJSON, stripExt bool) (*VideoFile, result.StartTime, _ = strconv.ParseFloat(probeJSON.Format.StartTime, 64) result.CreationTime = probeJSON.Format.Tags.CreationTime.Time - audioStream := result.GetAudioStream() + audioStream := result.getAudioStream() if audioStream != nil { result.AudioCodec = audioStream.CodecName result.AudioStream = audioStream } - videoStream := result.GetVideoStream() + videoStream := result.getVideoStream() if videoStream != nil { result.VideoStream = videoStream result.VideoCodec = videoStream.CodecName @@ -342,7 +339,7 @@ func parse(filePath string, probeJSON *FFProbeJSON, stripExt bool) (*VideoFile, return result, nil } -func (v *VideoFile) GetAudioStream() *FFProbeStream { +func (v *VideoFile) getAudioStream() *FFProbeStream { index := v.getStreamIndex("audio", v.JSON) if index != -1 { return &v.JSON.Streams[index] @@ -350,7 +347,7 @@ func (v *VideoFile) GetAudioStream() *FFProbeStream { return nil } -func (v *VideoFile) GetVideoStream() *FFProbeStream { +func (v *VideoFile) getVideoStream() *FFProbeStream { index := v.getStreamIndex("video", v.JSON) if index != -1 { return &v.JSON.Streams[index] @@ -374,5 +371,4 @@ func (v *VideoFile) SetTitleFromPath(stripExtension bool) { ext := filepath.Ext(v.Title) v.Title = strings.TrimSuffix(v.Title, ext) } - } diff --git a/pkg/ffmpeg/media_detection.go b/pkg/ffmpeg/media_detection.go index 6d9feb59a..b563cc845 100644 --- a/pkg/ffmpeg/media_detection.go +++ b/pkg/ffmpeg/media_detection.go @@ -38,11 +38,11 @@ func containsMatroskaSignature(buf, subType []byte) bool { return buf[index-3] == 0x42 && buf[index-2] == 0x82 } -// MagicContainer returns the container type of a file path. +// magicContainer returns the container type of a file path. // Returns the zero-value on errors or no-match. Implements mkv or // webm only, as ffprobe can't distinguish between them and not all // browsers support mkv -func MagicContainer(filePath string) Container { +func magicContainer(filePath string) Container { file, err := os.Open(filePath) if err != nil { logger.Errorf("[magicfile] %v", err) diff --git a/pkg/ffmpeg/stream.go b/pkg/ffmpeg/stream.go index 41832fc89..4f4a11f24 100644 --- a/pkg/ffmpeg/stream.go +++ b/pkg/ffmpeg/stream.go @@ -4,11 +4,10 @@ import ( "io" "net/http" "os" - "os/exec" "strconv" "strings" - "github.com/stashapp/stash/pkg/desktop" + stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" ) @@ -206,7 +205,7 @@ func (e *Encoder) GetTranscodeStream(options TranscodeStreamOptions) (*Stream, e func (e *Encoder) stream(probeResult VideoFile, options TranscodeStreamOptions) (*Stream, error) { args := options.getStreamArgs() - cmd := exec.Command(string(*e), args...) + cmd := stashExec.Command(string(*e), args...) logger.Debugf("Streaming via: %s", strings.Join(cmd.Args, " ")) stdout, err := cmd.StdoutPipe() @@ -221,7 +220,6 @@ func (e *Encoder) stream(probeResult VideoFile, options TranscodeStreamOptions) return nil, err } - desktop.HideExecShell(cmd) if err = cmd.Start(); err != nil { return nil, err } diff --git a/pkg/file/hash.go b/pkg/file/hash.go index 630ffcb6f..67998a265 100644 --- a/pkg/file/hash.go +++ b/pkg/file/hash.go @@ -3,15 +3,16 @@ package file import ( "io" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/hash/md5" + "github.com/stashapp/stash/pkg/hash/oshash" ) type FSHasher struct{} func (h *FSHasher) OSHash(src io.ReadSeeker, size int64) (string, error) { - return utils.OSHashFromReader(src, size) + return oshash.FromReader(src, size) } func (h *FSHasher) MD5(src io.Reader) (string, error) { - return utils.MD5FromReader(src) + return md5.FromReader(src) } diff --git a/pkg/fsutil/dir.go b/pkg/fsutil/dir.go new file mode 100644 index 000000000..2bd796ed2 --- /dev/null +++ b/pkg/fsutil/dir.go @@ -0,0 +1,101 @@ +package fsutil + +import ( + "fmt" + "io/fs" + "os" + "os/user" + "path/filepath" + "strings" +) + +// DirExists returns true if the given path exists and is a directory +func DirExists(path string) (bool, error) { + fileInfo, err := os.Stat(path) + if err != nil { + return false, fs.ErrNotExist + } + if !fileInfo.IsDir() { + return false, fmt.Errorf("path is not a directory <%s>", path) + } + return true, nil +} + +// IsPathInDir returns true if pathToCheck is within dir. +func IsPathInDir(dir, pathToCheck string) bool { + rel, err := filepath.Rel(dir, pathToCheck) + + if err == nil { + if !strings.HasPrefix(rel, "..") { + return true + } + } + + return false +} + +// GetHomeDirectory returns the path of the user's home directory. ~ on Unix and C:\Users\UserName on Windows +func GetHomeDirectory() string { + currentUser, err := user.Current() + if err != nil { + panic(err) + } + return currentUser.HomeDir +} + +// EnsureDir will create a directory at the given path if it doesn't already exist +func EnsureDir(path string) error { + exists, err := DirExists(path) + if !exists { + err = os.Mkdir(path, 0755) + return err + } + return err +} + +// EnsureDirAll will create a directory at the given path along with any necessary parents if they don't already exist +func EnsureDirAll(path string) error { + return os.MkdirAll(path, 0755) +} + +// RemoveDir removes the given dir (if it exists) along with all of its contents +func RemoveDir(path string) error { + return os.RemoveAll(path) +} + +// EmptyDir will recursively remove the contents of a directory at the given path +func EmptyDir(path string) error { + d, err := os.Open(path) + if err != nil { + return err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + + for _, name := range names { + err = os.RemoveAll(filepath.Join(path, name)) + if err != nil { + return err + } + } + + return nil +} + +// GetIntraDir returns a string that can be added to filepath.Join to implement directory depth, "" on error +// eg given a pattern of 0af63ce3c99162e9df23a997f62621c5 and a depth of 2 length of 3 +// returns 0af/63c or 0af\63c ( dependin on os) that can be later used like this filepath.Join(directory, intradir, basename) +func GetIntraDir(pattern string, depth, length int) string { + if depth < 1 || length < 1 || (depth*length > len(pattern)) { + return "" + } + intraDir := pattern[0:length] // depth 1 , get length number of characters from pattern + for i := 1; i < depth; i++ { // for every extra depth: move to the right of the pattern length positions, get length number of chars + intraDir = filepath.Join(intraDir, pattern[length*i:length*(i+1)]) // adding each time to intradir the extra characters with a filepath join + } + return intraDir +} diff --git a/pkg/utils/file_test.go b/pkg/fsutil/dir_test.go similarity index 99% rename from pkg/utils/file_test.go rename to pkg/fsutil/dir_test.go index 4be89c3e4..577773fc1 100644 --- a/pkg/utils/file_test.go +++ b/pkg/fsutil/dir_test.go @@ -1,4 +1,4 @@ -package utils +package fsutil import ( "os" @@ -75,7 +75,5 @@ func TestDirExists(t *testing.T) { result, _ := DirExists(tc.dir) assert.Equal(tc.expected, result, "[%d] expected: %t for dir: %s;", i, tc.expected, tc.dir) } - } - } diff --git a/pkg/fsutil/file.go b/pkg/fsutil/file.go new file mode 100644 index 000000000..e958397f6 --- /dev/null +++ b/pkg/fsutil/file.go @@ -0,0 +1,118 @@ +package fsutil + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// SafeMove attempts to move the file with path src to dest using os.Rename. If this fails, then it copies src to dest, then deletes src. +func SafeMove(src, dst string) error { + err := os.Rename(src, dst) + + if err != nil { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + err = out.Close() + if err != nil { + return err + } + + err = os.Remove(src) + if err != nil { + return err + } + } + + return nil +} + +// MatchExtension returns true if the extension of the provided path +// matches any of the provided extensions. +func MatchExtension(path string, extensions []string) bool { + ext := filepath.Ext(path) + for _, e := range extensions { + if strings.EqualFold(ext, "."+e) { + return true + } + } + + return false +} + +// FindInPaths returns the path to baseName in the first path where it exists from paths. +func FindInPaths(paths []string, baseName string) string { + for _, p := range paths { + filePath := filepath.Join(p, baseName) + if exists, _ := FileExists(filePath); exists { + return filePath + } + } + + return "" +} + +// FileExists returns true if the given path exists and is a file. +// This function returns false and the error encountered if the call to os.Stat fails. +func FileExists(path string) (bool, error) { + info, err := os.Stat(path) + if err == nil { + return !info.IsDir(), nil + } + return false, err +} + +// WriteFile writes file to path creating parent directories if needed +func WriteFile(path string, file []byte) error { + pathErr := EnsureDirAll(filepath.Dir(path)) + if pathErr != nil { + return fmt.Errorf("cannot ensure path %s", pathErr) + } + + err := os.WriteFile(path, file, 0755) + if err != nil { + return fmt.Errorf("write error for thumbnail %s: %s ", path, err) + } + return nil +} + +// GetNameFromPath returns the name of a file from its path +// if stripExtension is true the extension is omitted from the name +func GetNameFromPath(path string, stripExtension bool) string { + fn := filepath.Base(path) + if stripExtension { + ext := filepath.Ext(fn) + fn = strings.TrimSuffix(fn, ext) + } + return fn +} + +// Touch creates an empty file at the given path if it doesn't already exist +func Touch(path string) error { + var _, err = os.Stat(path) + if os.IsNotExist(err) { + var file, err = os.Create(path) + if err != nil { + return err + } + defer file.Close() + } + return nil +} diff --git a/pkg/fsutil/fs.go b/pkg/fsutil/fs.go new file mode 100644 index 000000000..0b9fc6416 --- /dev/null +++ b/pkg/fsutil/fs.go @@ -0,0 +1,67 @@ +package fsutil + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "unicode" +) + +// IsFsPathCaseSensitive checks the fs of the given path to see if it is case sensitive +// if the case sensitivity can not be determined false and an error != nil are returned +func IsFsPathCaseSensitive(path string) (bool, error) { + // The case sensitivity of the fs of "path" is determined by case flipping + // the first letter rune from the base string of the path + // If the resulting flipped path exists then the fs should not be case sensitive + // ( we check the file mod time to avoid matching an existing path ) + + fi, err := os.Stat(path) + if err != nil { // path cannot be stat'd + return false, err + } + + base := filepath.Base(path) + fBase, err := flipCaseSingle(base) + if err != nil { // cannot be case flipped + return false, err + } + i := strings.LastIndex(path, base) + if i < 0 { // shouldn't happen + return false, fmt.Errorf("could not case flip path %s", path) + } + + flipped := []byte(path) + for _, c := range []byte(fBase) { // replace base of path with the flipped one ( we need to flip the base or last dir part ) + flipped[i] = c + i++ + } + + fiCase, err := os.Stat(string(flipped)) + if err != nil { // cannot stat the case flipped path + return true, nil // fs of path should be case sensitive + } + + if fiCase.ModTime() == fi.ModTime() { // file path exists and is the same + return false, nil // fs of path is not case sensitive + } + return false, fmt.Errorf("can not determine case sensitivity of path %s", path) +} + +// flipCaseSingle flips the case ( lower<->upper ) of a single char from the string s +// If the string cannot be flipped, the original string value and an error are returned +func flipCaseSingle(s string) (string, error) { + rr := []rune(s) + for i, r := range rr { + if unicode.IsLetter(r) { // look for a letter to flip + if unicode.IsUpper(r) { + rr[i] = unicode.ToLower(r) + return string(rr), nil + } + rr[i] = unicode.ToUpper(r) + return string(rr), nil + } + + } + return s, fmt.Errorf("could not case flip string %s", s) +} diff --git a/pkg/utils/symwalk.go b/pkg/fsutil/symwalk.go similarity index 99% rename from pkg/utils/symwalk.go rename to pkg/fsutil/symwalk.go index ccb0f02a7..31623f487 100644 --- a/pkg/utils/symwalk.go +++ b/pkg/fsutil/symwalk.go @@ -31,7 +31,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -package utils +package fsutil import ( "os" diff --git a/pkg/gallery/export.go b/pkg/gallery/export.go index e8347bb5c..f861918cd 100644 --- a/pkg/gallery/export.go +++ b/pkg/gallery/export.go @@ -1,8 +1,8 @@ package gallery import ( - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/gallery/export_test.go b/pkg/gallery/export_test.go index 379900dfd..81961f9b8 100644 --- a/pkg/gallery/export_test.go +++ b/pkg/gallery/export_test.go @@ -3,8 +3,8 @@ package gallery import ( "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" diff --git a/pkg/gallery/import.go b/pkg/gallery/import.go index e568d5bbd..f82cff13b 100644 --- a/pkg/gallery/import.go +++ b/pkg/gallery/import.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) type Importer struct { @@ -135,8 +135,8 @@ func (i *Importer) populatePerformers() error { pluckedNames = append(pluckedNames, performer.Name.String) } - missingPerformers := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingPerformers := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingPerformers) > 0 { @@ -191,8 +191,8 @@ func (i *Importer) populateTags() error { pluckedNames = append(pluckedNames, tag.Name) } - missingTags := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingTags := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingTags) > 0 { diff --git a/pkg/gallery/import_test.go b/pkg/gallery/import_test.go index 962bd18d7..4acc8c449 100644 --- a/pkg/gallery/import_test.go +++ b/pkg/gallery/import_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/gallery/scan.go b/pkg/gallery/scan.go index d5b195b39..2c65cd302 100644 --- a/pkg/gallery/scan.go +++ b/pkg/gallery/scan.go @@ -9,9 +9,10 @@ import ( "time" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/utils" ) @@ -121,7 +122,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler g, _ = qb.FindByChecksum(checksum) if g != nil { - exists, _ := utils.FileExists(g.Path.String) + exists, _ := fsutil.FileExists(g.Path.String) if !scanner.CaseSensitiveFs { // #1426 - if file exists but is a case-insensitive match for the // original filename, then treat it as a move @@ -151,7 +152,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler g = &models.Gallery{ Zip: true, Title: sql.NullString{ - String: utils.GetNameFromPath(path, scanner.StripFileExtension), + String: fsutil.GetNameFromPath(path, scanner.StripFileExtension), Valid: true, }, CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, @@ -161,7 +162,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler g.SetFile(*scanned) // only warn when creating the gallery - ok, err := utils.IsZipFileUncompressed(path) + ok, err := isZipFileUncompressed(path) if err == nil && !ok { logger.Warnf("%s is using above store (0) level compression.", path) } @@ -193,8 +194,26 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler return } +// IsZipFileUnmcompressed returns true if zip file in path is using 0 compression level +func isZipFileUncompressed(path string) (bool, error) { + r, err := zip.OpenReader(path) + if err != nil { + fmt.Printf("Error reading zip file %s: %s\n", path, err) + return false, err + } else { + defer r.Close() + for _, f := range r.File { + if f.FileInfo().IsDir() { // skip dirs, they always get store level compression + continue + } + return f.Method == 0, nil // check compression level of first actual file + } + } + return false, nil +} + func (scanner *Scanner) isImage(pathname string) bool { - return utils.MatchExtension(pathname, scanner.ImageExtensions) + return fsutil.MatchExtension(pathname, scanner.ImageExtensions) } func (scanner *Scanner) hasImages(path string) bool { diff --git a/pkg/gallery/update.go b/pkg/gallery/update.go index 9355282a1..4c16793ca 100644 --- a/pkg/gallery/update.go +++ b/pkg/gallery/update.go @@ -2,7 +2,7 @@ package gallery import ( "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/intslice" ) func UpdateFileModTime(qb models.GalleryWriter, id int, modTime models.NullSQLiteTimestamp) (*models.Gallery, error) { @@ -18,7 +18,7 @@ func AddImage(qb models.GalleryReaderWriter, galleryID int, imageID int) error { return err } - imageIDs = utils.IntAppendUnique(imageIDs, imageID) + imageIDs = intslice.IntAppendUnique(imageIDs, imageID) return qb.UpdateImages(galleryID, imageIDs) } @@ -29,7 +29,7 @@ func AddPerformer(qb models.GalleryReaderWriter, id int, performerID int) (bool, } oldLen := len(performerIDs) - performerIDs = utils.IntAppendUnique(performerIDs, performerID) + performerIDs = intslice.IntAppendUnique(performerIDs, performerID) if len(performerIDs) != oldLen { if err := qb.UpdatePerformers(id, performerIDs); err != nil { @@ -49,7 +49,7 @@ func AddTag(qb models.GalleryReaderWriter, id int, tagID int) (bool, error) { } oldLen := len(tagIDs) - tagIDs = utils.IntAppendUnique(tagIDs, tagID) + tagIDs = intslice.IntAppendUnique(tagIDs, tagID) if len(tagIDs) != oldLen { if err := qb.UpdateTags(id, tagIDs); err != nil { diff --git a/pkg/hash/key.go b/pkg/hash/key.go new file mode 100644 index 000000000..8012fae94 --- /dev/null +++ b/pkg/hash/key.go @@ -0,0 +1,27 @@ +// Package hash provides utility functions for generating hashes from strings and random keys. +package hash + +import ( + "crypto/rand" + "fmt" + "hash/fnv" +) + +// GenerateRandomKey generates a random string of length l. +// It returns an empty string and an error if an error occurs while generating a random number. +func GenerateRandomKey(l int) (string, error) { + b := make([]byte, l) + if _, err := rand.Read(b); err != nil { + return "", err + } + return fmt.Sprintf("%x", b), nil +} + +// IntFromString generates a uint64 from a string. +// Values returned by this function are guaranteed to be the same for equal strings. +// They are not guaranteed to be unique for different strings. +func IntFromString(str string) uint64 { + h := fnv.New64a() + h.Write([]byte(str)) + return h.Sum64() +} diff --git a/pkg/hash/md5/md5.go b/pkg/hash/md5/md5.go new file mode 100644 index 000000000..46741d9d1 --- /dev/null +++ b/pkg/hash/md5/md5.go @@ -0,0 +1,44 @@ +// Package md5 provides utility functions for generating MD5 hashes. +package md5 + +import ( + "crypto/md5" + "fmt" + "io" + "os" +) + +// FromBytes returns an MD5 checksum string from data. +func FromBytes(data []byte) string { + result := md5.Sum(data) + return fmt.Sprintf("%x", result) +} + +// FromString returns an MD5 checksum string from str. +func FromString(str string) string { + data := []byte(str) + return FromBytes(data) +} + +// FromFilePath returns an MD5 checksum string for the file at filePath. +// It returns an empty string and an error if an error occurs opening the file. +func FromFilePath(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + return "", err + } + defer f.Close() + + return FromReader(f) +} + +// FromReader returns an MD5 checksum string from data read from src. +// It returns an empty string and an error if an error occurs reading from src. +func FromReader(src io.Reader) (string, error) { + h := md5.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + checksum := h.Sum(nil) + return fmt.Sprintf("%x", checksum), nil +} diff --git a/pkg/utils/oshash.go b/pkg/hash/oshash/oshash.go similarity index 80% rename from pkg/utils/oshash.go rename to pkg/hash/oshash/oshash.go index 0057e2617..7d0687b99 100644 --- a/pkg/utils/oshash.go +++ b/pkg/hash/oshash/oshash.go @@ -1,4 +1,8 @@ -package utils +// Package oshash implements the algorithm that OpenSubtitles.org uses to generate unique hashes. +// +// Calculation is as follows: +// size + 64 bit checksum of the first and last 64k bytes of the file. +package oshash import ( "encoding/binary" @@ -42,7 +46,8 @@ func oshash(size int64, head []byte, tail []byte) (string, error) { return fmt.Sprintf("%016x", result), nil } -func OSHashFromReader(src io.ReadSeeker, fileSize int64) (string, error) { +// FromFilePath calculates the hash reading from src. +func FromReader(src io.ReadSeeker, fileSize int64) (string, error) { if fileSize == 0 { return "", nil } @@ -76,12 +81,8 @@ func OSHashFromReader(src io.ReadSeeker, fileSize int64) (string, error) { return oshash(fileSize, head, tail) } -// OSHashFromFilePath calculates the hash using the same algorithm that -// OpenSubtitles.org uses. -// -// Calculation is as follows: -// size + 64 bit checksum of the first and last 64k bytes of the file. -func OSHashFromFilePath(filePath string) (string, error) { +// Is the equivalent of opening filePath, and calling FromReader with the data and file size. +func FromFilePath(filePath string) (string, error) { f, err := os.Open(filePath) if err != nil { return "", err @@ -95,5 +96,5 @@ func OSHashFromFilePath(filePath string) (string, error) { fileSize := fi.Size() - return OSHashFromReader(f, fileSize) + return FromReader(f, fileSize) } diff --git a/pkg/utils/oshash_internal_test.go b/pkg/hash/oshash/oshash_internal_test.go similarity index 99% rename from pkg/utils/oshash_internal_test.go rename to pkg/hash/oshash/oshash_internal_test.go index d9e709444..20cdf989e 100644 --- a/pkg/utils/oshash_internal_test.go +++ b/pkg/hash/oshash/oshash_internal_test.go @@ -1,4 +1,4 @@ -package utils +package oshash import ( "math/rand" diff --git a/pkg/image/delete.go b/pkg/image/delete.go index 989cf5694..35ab3704b 100644 --- a/pkg/image/delete.go +++ b/pkg/image/delete.go @@ -2,9 +2,9 @@ package image import ( "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/paths" ) type Destroyer interface { @@ -21,7 +21,7 @@ type FileDeleter struct { // MarkGeneratedFiles marks for deletion the generated files for the provided image. func (d *FileDeleter) MarkGeneratedFiles(image *models.Image) error { thumbPath := d.Paths.Generated.GetThumbnailPath(image.Checksum, models.DefaultGthumbWidth) - exists, _ := utils.FileExists(thumbPath) + exists, _ := fsutil.FileExists(thumbPath) if exists { return d.Files([]string{thumbPath}) } diff --git a/pkg/image/export.go b/pkg/image/export.go index b70bbe7f2..647bf041d 100644 --- a/pkg/image/export.go +++ b/pkg/image/export.go @@ -1,8 +1,8 @@ package image import ( - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" ) // ToBasicJSON converts a image object into its JSON object equivalent. It diff --git a/pkg/image/export_test.go b/pkg/image/export_test.go index f47672b58..25c13f4ea 100644 --- a/pkg/image/export_test.go +++ b/pkg/image/export_test.go @@ -3,8 +3,8 @@ package image import ( "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" diff --git a/pkg/image/image.go b/pkg/image/image.go index 392c9124a..668a65513 100644 --- a/pkg/image/image.go +++ b/pkg/image/image.go @@ -13,9 +13,10 @@ import ( "time" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" _ "golang.org/x/image/webp" ) @@ -53,7 +54,7 @@ func CalculateMD5(path string) (string, error) { } defer f.Close() - return utils.MD5FromReader(f) + return md5.FromReader(f) } func FileExists(path string) bool { @@ -245,5 +246,5 @@ func GetTitle(s *models.Image) string { // If stripExt is set the file extension is omitted from the name func GetFilename(s *models.Image, stripExt bool) string { _, fn := file.ZipFilePath(s.Path) - return utils.GetNameFromPath(fn, stripExt) + return fsutil.GetNameFromPath(fn, stripExt) } diff --git a/pkg/image/import.go b/pkg/image/import.go index b6ebe5b05..78b60c4b1 100644 --- a/pkg/image/import.go +++ b/pkg/image/import.go @@ -5,9 +5,9 @@ import ( "fmt" "strings" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) type Importer struct { @@ -167,8 +167,8 @@ func (i *Importer) populatePerformers() error { pluckedNames = append(pluckedNames, performer.Name.String) } - missingPerformers := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingPerformers := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingPerformers) > 0 { @@ -315,8 +315,8 @@ func importTags(tagWriter models.TagReaderWriter, names []string, missingRefBeha pluckedNames = append(pluckedNames, tag.Name) } - missingTags := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingTags := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingTags) > 0 { diff --git a/pkg/image/import_test.go b/pkg/image/import_test.go index 8b7099823..156ec96d2 100644 --- a/pkg/image/import_test.go +++ b/pkg/image/import_test.go @@ -4,8 +4,8 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/image/scan.go b/pkg/image/scan.go index 15424f29a..c356e4697 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -9,8 +9,8 @@ import ( "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/image/update.go b/pkg/image/update.go index 95b80d697..f8c43f965 100644 --- a/pkg/image/update.go +++ b/pkg/image/update.go @@ -2,7 +2,7 @@ package image import ( "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/intslice" ) func UpdateFileModTime(qb models.ImageWriter, id int, modTime models.NullSQLiteTimestamp) (*models.Image, error) { @@ -19,7 +19,7 @@ func AddPerformer(qb models.ImageReaderWriter, id int, performerID int) (bool, e } oldLen := len(performerIDs) - performerIDs = utils.IntAppendUnique(performerIDs, performerID) + performerIDs = intslice.IntAppendUnique(performerIDs, performerID) if len(performerIDs) != oldLen { if err := qb.UpdatePerformers(id, performerIDs); err != nil { @@ -39,7 +39,7 @@ func AddTag(qb models.ImageReaderWriter, id int, tagID int) (bool, error) { } oldLen := len(tagIDs) - tagIDs = utils.IntAppendUnique(tagIDs, tagID) + tagIDs = intslice.IntAppendUnique(tagIDs, tagID) if len(tagIDs) != oldLen { if err := qb.UpdateTags(id, tagIDs); err != nil { diff --git a/pkg/image/vips.go b/pkg/image/vips.go index 951406848..39809dc18 100644 --- a/pkg/image/vips.go +++ b/pkg/image/vips.go @@ -3,10 +3,9 @@ package image import ( "bytes" "fmt" - "os/exec" "strings" - "github.com/stashapp/stash/pkg/desktop" + "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" ) @@ -33,7 +32,6 @@ func (e *vipsEncoder) run(args []string, stdin *bytes.Buffer) (string, error) { cmd.Stderr = &stderr cmd.Stdin = stdin - desktop.HideExecShell(cmd) if err := cmd.Start(); err != nil { return "", err } diff --git a/pkg/utils/context.go b/pkg/job/context.go similarity index 70% rename from pkg/utils/context.go rename to pkg/job/context.go index 06427bb5b..e53625e23 100644 --- a/pkg/utils/context.go +++ b/pkg/job/context.go @@ -1,4 +1,4 @@ -package utils +package job import ( "context" @@ -20,9 +20,3 @@ func (valueOnlyContext) Done() <-chan struct{} { func (valueOnlyContext) Err() error { return nil } - -func ValueOnlyContext(ctx context.Context) context.Context { - return valueOnlyContext{ - ctx, - } -} diff --git a/pkg/job/job.go b/pkg/job/job.go index 0ace099e1..09188eb9d 100644 --- a/pkg/job/job.go +++ b/pkg/job/job.go @@ -60,6 +60,19 @@ type Job struct { cancelFunc context.CancelFunc } +// TimeElapsed returns the total time elapsed for the job. +// If the EndTime is set, then it uses this to calculate the elapsed time, otherwise it uses time.Now. +func (j *Job) TimeElapsed() time.Duration { + var end time.Time + if j.EndTime != nil { + end = time.Now() + } else { + end = *j.EndTime + } + + return end.Sub(*j.StartTime) +} + func (j *Job) cancel() { if j.Status == StatusReady { j.Status = StatusCancelled diff --git a/pkg/job/manager.go b/pkg/job/manager.go index 95837ca2a..e5d3d239a 100644 --- a/pkg/job/manager.go +++ b/pkg/job/manager.go @@ -2,14 +2,8 @@ package job import ( "context" - "fmt" - "strconv" - "strings" "sync" "time" - - "github.com/stashapp/stash/pkg/desktop" - "github.com/stashapp/stash/pkg/utils" ) const maxGraveyardSize = 10 @@ -181,7 +175,9 @@ func (m *Manager) dispatch(j *Job) (done chan struct{}) { j.StartTime = &t j.Status = StatusRunning - ctx, cancelFunc := context.WithCancel(utils.ValueOnlyContext(j.outerCtx)) + ctx, cancelFunc := context.WithCancel(valueOnlyContext{ + j.outerCtx, + }) j.cancelFunc = cancelFunc done = make(chan struct{}) @@ -210,12 +206,6 @@ func (m *Manager) onJobFinish(job *Job) { } t := time.Now() job.EndTime = &t - cleanDesc := strings.TrimRight(job.Description, ".") - timeElapsed := job.EndTime.Sub(*job.StartTime) - hours := fmt.Sprintf("%+02s", strconv.FormatFloat(timeElapsed.Hours(), 'f', 0, 64)) - minutes := fmt.Sprintf("%+02s", strconv.FormatFloat(timeElapsed.Minutes(), 'f', 0, 64)) - seconds := fmt.Sprintf("%+02s", strconv.FormatFloat(timeElapsed.Seconds(), 'f', 0, 64)) - desktop.SendNotification("Task Finished", "Task \""+cleanDesc+"\" is finished in "+hours+":"+minutes+":"+seconds+".") } func (m *Manager) removeJob(job *Job) { diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 1a4c0d180..f97faf9f8 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -1,298 +1,146 @@ +// Package logger provides methods and interfaces used by other stash packages for logging purposes. package logger import ( - "fmt" "os" - "sync" - "time" - - "github.com/sirupsen/logrus" ) -type LogItem struct { - Time time.Time `json:"time"` - Type string `json:"type"` - Message string `json:"message"` +// LoggerImpl is the interface that groups logging methods. +// +// Progressf logs using a specific progress format. +// Trace, Debug, Info, Warn and Error log to the applicable log level. Arguments are handled in the manner of fmt.Print. +// Tracef, Debugf, Infof, Warnf, Errorf log to the applicable log level. Arguments are handled in the manner of fmt.Printf. +// Fatal and Fatalf log to the applicable log level, then call os.Exit(1). +type LoggerImpl interface { + Progressf(format string, args ...interface{}) + + Trace(args ...interface{}) + Tracef(format string, args ...interface{}) + + Debug(args ...interface{}) + Debugf(format string, args ...interface{}) + + Info(args ...interface{}) + Infof(format string, args ...interface{}) + + Warn(args ...interface{}) + Warnf(format string, args ...interface{}) + + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) } -var logger = logrus.New() -var progressLogger = logrus.New() - -var LogCache []LogItem -var mutex = &sync.Mutex{} -var logSubs []chan []LogItem -var waiting = false -var lastBroadcast = time.Now() -var logBuffer []LogItem - -// Init initialises the logger based on a logging configuration -func Init(logFile string, logOut bool, logLevel string) { - var file *os.File - customFormatter := new(logrus.TextFormatter) - customFormatter.TimestampFormat = "2006-01-02 15:04:05" - customFormatter.ForceColors = true - customFormatter.FullTimestamp = true - logger.SetOutput(os.Stderr) - logger.SetFormatter(customFormatter) - - // #1837 - trigger the console to use color-mode since it won't be - // otherwise triggered until the first log entry - // this is covers the situation where the logger is only logging to file - // and therefore does not trigger the console color-mode - resulting in - // the access log colouring not being applied - _, _ = customFormatter.Format(logrus.NewEntry(logger)) - - if logFile != "" { - var err error - file, err = os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - - if err != nil { - fmt.Printf("Could not open '%s' for log output due to error: %s\n", logFile, err.Error()) - } - } - - if file != nil { - if logOut { - // log to file separately disabling colours - fileFormatter := new(logrus.TextFormatter) - fileFormatter.TimestampFormat = customFormatter.TimestampFormat - fileFormatter.FullTimestamp = customFormatter.FullTimestamp - logger.AddHook(&fileLogHook{ - Writer: file, - Formatter: fileFormatter, - }) - } else { - // logging to file only - // turn off the colouring for the file - customFormatter.ForceColors = false - logger.Out = file - } - } - - // otherwise, output to StdErr - - SetLogLevel(logLevel) -} - -func SetLogLevel(level string) { - logger.Level = logLevelFromString(level) -} - -func logLevelFromString(level string) logrus.Level { - ret := logrus.InfoLevel - - switch level { - case "Debug": - ret = logrus.DebugLevel - case "Warning": - ret = logrus.WarnLevel - case "Error": - ret = logrus.ErrorLevel - case "Trace": - ret = logrus.TraceLevel - } - - return ret -} - -func addLogItem(l *LogItem) { - mutex.Lock() - l.Time = time.Now() - LogCache = append([]LogItem{*l}, LogCache...) - if len(LogCache) > 30 { - LogCache = LogCache[:len(LogCache)-1] - } - mutex.Unlock() - go broadcastLogItem(l) -} - -func GetLogCache() []LogItem { - mutex.Lock() - - ret := make([]LogItem, len(LogCache)) - copy(ret, LogCache) - - mutex.Unlock() - - return ret -} - -func SubscribeToLog(stop chan int) <-chan []LogItem { - ret := make(chan []LogItem, 100) - - go func() { - <-stop - unsubscribeFromLog(ret) - }() - - mutex.Lock() - logSubs = append(logSubs, ret) - mutex.Unlock() - - return ret -} - -func unsubscribeFromLog(toRemove chan []LogItem) { - mutex.Lock() - for i, c := range logSubs { - if c == toRemove { - logSubs = append(logSubs[:i], logSubs[i+1:]...) - } - } - close(toRemove) - mutex.Unlock() -} - -func doBroadcastLogItems() { - // assumes mutex held - - for _, c := range logSubs { - // don't block waiting to broadcast - select { - case c <- logBuffer: - default: - } - } - - logBuffer = nil - waiting = false - lastBroadcast = time.Now() -} - -func broadcastLogItem(l *LogItem) { - mutex.Lock() - - logBuffer = append(logBuffer, *l) - - // don't send more than once per second - if !waiting { - // if last broadcast was under a second ago, wait until a second has - // passed - timeSinceBroadcast := time.Since(lastBroadcast) - if timeSinceBroadcast.Seconds() < 1 { - waiting = true - time.AfterFunc(time.Second-timeSinceBroadcast, func() { - mutex.Lock() - doBroadcastLogItems() - mutex.Unlock() - }) - } else { - doBroadcastLogItems() - } - } - // if waiting then adding it to the buffer is sufficient - - mutex.Unlock() -} - -func init() { - progressLogger.SetFormatter(new(ProgressFormatter)) -} +// Logger is the LoggerImpl used when calling the global Logger functions. +// It is suggested to use the LoggerImpl interface directly, rather than calling global log functions. +var Logger LoggerImpl +// Progressf calls Progressf with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Progressf(format string, args ...interface{}) { - progressLogger.Infof(format, args...) - l := &LogItem{ - Type: "progress", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Progressf(format, args...) } - addLogItem(l) - } +// Trace calls Trace with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Trace(args ...interface{}) { - logger.Trace(args...) - l := &LogItem{ - Type: "trace", - Message: fmt.Sprint(args...), + if Logger != nil { + Logger.Trace(args...) } - addLogItem(l) } +// Tracef calls Tracef with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Tracef(format string, args ...interface{}) { - logger.Tracef(format, args...) - l := &LogItem{ - Type: "trace", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Tracef(format, args...) } - addLogItem(l) } +// Debug calls Debug with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Debug(args ...interface{}) { - logger.Debug(args...) - l := &LogItem{ - Type: "debug", - Message: fmt.Sprint(args...), + if Logger != nil { + Logger.Debug(args...) } - addLogItem(l) } +// Debugf calls Debugf with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Debugf(format string, args ...interface{}) { - logger.Debugf(format, args...) - l := &LogItem{ - Type: "debug", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Debugf(format, args...) } - addLogItem(l) } +// Info calls Info with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Info(args ...interface{}) { - logger.Info(args...) - l := &LogItem{ - Type: "info", - Message: fmt.Sprint(args...), + if Logger != nil { + Logger.Info(args...) } - addLogItem(l) } +// Infof calls Infof with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Infof(format string, args ...interface{}) { - logger.Infof(format, args...) - l := &LogItem{ - Type: "info", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Infof(format, args...) } - addLogItem(l) } +// Warn calls Warn with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Warn(args ...interface{}) { - logger.Warn(args...) - l := &LogItem{ - Type: "warn", - Message: fmt.Sprint(args...), + if Logger != nil { + Logger.Warn(args...) } - addLogItem(l) } +// Warnf calls Warnf with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Warnf(format string, args ...interface{}) { - logger.Warnf(format, args...) - l := &LogItem{ - Type: "warn", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Warnf(format, args...) } - addLogItem(l) } +// Error calls Error with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Error(args ...interface{}) { - logger.Error(args...) - l := &LogItem{ - Type: "error", - Message: fmt.Sprint(args...), + if Logger != nil { + Logger.Error(args...) } - addLogItem(l) } +// Errorf calls Errorf with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Errorf(format string, args ...interface{}) { - logger.Errorf(format, args...) - l := &LogItem{ - Type: "error", - Message: fmt.Sprintf(format, args...), + if Logger != nil { + Logger.Errorf(format, args...) } - addLogItem(l) } +// Fatal calls Fatal with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Fatal(args ...interface{}) { - logger.Fatal(args...) + if Logger != nil { + Logger.Fatal(args...) + } else { + os.Exit(1) + } } +// Fatalf calls Fatalf with the Logger registered using RegisterLogger. +// If no logger has been registered, then this function is a no-op. func Fatalf(format string, args ...interface{}) { - logger.Fatalf(format, args...) + if Logger != nil { + Logger.Fatalf(format, args...) + } else { + os.Exit(1) + } } diff --git a/pkg/logger/plugin.go b/pkg/logger/plugin.go index 7b2541c1b..639a2a99f 100644 --- a/pkg/logger/plugin.go +++ b/pkg/logger/plugin.go @@ -9,39 +9,40 @@ import ( "strings" ) +// PluginLogLevel represents a logging level for plugins to send log messages to stash. type PluginLogLevel struct { char byte - Name string + name string } // Valid Level values. var ( TraceLevel = PluginLogLevel{ char: 't', - Name: "trace", + name: "trace", } DebugLevel = PluginLogLevel{ char: 'd', - Name: "debug", + name: "debug", } InfoLevel = PluginLogLevel{ char: 'i', - Name: "info", + name: "info", } WarningLevel = PluginLogLevel{ char: 'w', - Name: "warning", + name: "warning", } ErrorLevel = PluginLogLevel{ char: 'e', - Name: "error", + name: "error", } ProgressLevel = PluginLogLevel{ char: 'p', - Name: "progress", + name: "progress", } NoneLevel = PluginLogLevel{ - Name: "none", + name: "none", } ) @@ -66,6 +67,8 @@ func (l PluginLogLevel) prefix() string { }) } +// Log prints the provided message to os.Stderr in a format that provides the correct LogLevel for stash. +// The message is formatted in the same way as fmt.Println. func (l PluginLogLevel) Log(args ...interface{}) { if l.char == 0 { return @@ -78,6 +81,8 @@ func (l PluginLogLevel) Log(args ...interface{}) { fmt.Fprintln(os.Stderr, argsToUse...) } +// Logf prints the provided message to os.Stderr in a format that provides the correct LogLevel for stash. +// The message is formatted in the same way as fmt.Printf. func (l PluginLogLevel) Logf(format string, args ...interface{}) { if l.char == 0 { return @@ -87,11 +92,11 @@ func (l PluginLogLevel) Logf(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, formatToUse, args...) } -// PluginLogLevelFromName returns the Level that matches the provided name or nil if +// PluginLogLevelFromName returns the PluginLogLevel that matches the provided name or nil if // the name does not match a valid value. func PluginLogLevelFromName(name string) *PluginLogLevel { for _, l := range validLevels { - if l.Name == name { + if l.name == name { return &l } } @@ -99,11 +104,11 @@ func PluginLogLevelFromName(name string) *PluginLogLevel { return nil } -// DetectLogLevel returns the Level and the logging string for a provided line +// detectLogLevel returns the Level and the logging string for a provided line // of plugin output. It parses the string for logging control characters and // determines the log level, if present. If not present, the plugin output // is returned unchanged with a nil Level. -func DetectLogLevel(line string) (*PluginLogLevel, string) { +func detectLogLevel(line string) (*PluginLogLevel, string) { if len(line) < 4 || line[0] != startLevelChar || line[2] != endLevelChar { return nil, line } @@ -127,14 +132,26 @@ func DetectLogLevel(line string) (*PluginLogLevel, string) { return level, line } +// PluginLogger interprets incoming log messages from plugins and logs to the appropriate log level. type PluginLogger struct { - Prefix string + // Logger is the LoggerImpl to forward log messages to. + Logger LoggerImpl + // Prefix is the prefix to prepend to log messages. + Prefix string + // DefaultLogLevel is the log level used if a log level prefix is not present in the received log message. DefaultLogLevel *PluginLogLevel - ProgressChan chan float64 + // ProgressChan is a channel that receives float64s indicating the current progress of an operation. + ProgressChan chan float64 } -func (log *PluginLogger) HandleStderrLine(line string) { - level, ll := DetectLogLevel(line) +func (log *PluginLogger) handleStderrLine(line string) { + if log.Logger == nil { + return + } + + logger := log.Logger + + level, ll := detectLogLevel(line) // if no log level, just output to info if level == nil { @@ -147,19 +164,19 @@ func (log *PluginLogger) HandleStderrLine(line string) { switch *level { case TraceLevel: - Trace(log.Prefix, ll) + logger.Trace(log.Prefix, ll) case DebugLevel: - Debug(log.Prefix, ll) + logger.Debug(log.Prefix, ll) case InfoLevel: - Info(log.Prefix, ll) + logger.Info(log.Prefix, ll) case WarningLevel: - Warn(log.Prefix, ll) + logger.Warn(log.Prefix, ll) case ErrorLevel: - Error(log.Prefix, ll) + logger.Error(log.Prefix, ll) case ProgressLevel: p, err := strconv.ParseFloat(ll, 64) if err != nil { - Errorf("Error parsing progress value '%s': %s", ll, err.Error()) + logger.Errorf("Error parsing progress value '%s': %s", ll, err.Error()) } else if log.ProgressChan != nil { // only pass progress through if channel present // don't block on this select { @@ -167,24 +184,28 @@ func (log *PluginLogger) HandleStderrLine(line string) { default: } } - } } -func (log *PluginLogger) HandlePluginStdErr(pluginStdErr io.ReadCloser) { +// ReadLogMessages reads plugin log messages from src, forwarding them to the PluginLoggers Logger. +// ProgressLevel messages are parsed as float64 and forwarded to ProgressChan. If ProgressChan is full, +// then the progress message is not forwarded. +// This method only returns when it reaches the end of src or encounters an error while reading src. +// This method closes src before returning. +func (log *PluginLogger) ReadLogMessages(src io.ReadCloser) { // pipe plugin stderr to our logging - scanner := bufio.NewScanner(pluginStdErr) + scanner := bufio.NewScanner(src) for scanner.Scan() { str := scanner.Text() if str != "" { - log.HandleStderrLine(str) + log.handleStderrLine(str) } } str := scanner.Text() if str != "" { - log.HandleStderrLine(str) + log.handleStderrLine(str) } - pluginStdErr.Close() + src.Close() } diff --git a/pkg/match/path.go b/pkg/match/path.go index e99ad82a5..47a7ad26e 100644 --- a/pkg/match/path.go +++ b/pkg/match/path.go @@ -11,7 +11,7 @@ import ( "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) const ( @@ -63,7 +63,7 @@ func getPathWords(path string) []string { // just use the first two characters // #2293 - need to convert to unicode runes for the substring, otherwise // the resulting string is corrupted. - ret = utils.StrAppendUnique(ret, string([]rune(w)[0:2])) + ret = stringslice.StrAppendUnique(ret, string([]rune(w)[0:2])) } } diff --git a/pkg/manager/jsonschema/gallery.go b/pkg/models/jsonschema/gallery.go similarity index 100% rename from pkg/manager/jsonschema/gallery.go rename to pkg/models/jsonschema/gallery.go diff --git a/pkg/manager/jsonschema/image.go b/pkg/models/jsonschema/image.go similarity index 100% rename from pkg/manager/jsonschema/image.go rename to pkg/models/jsonschema/image.go diff --git a/pkg/manager/jsonschema/mappings.go b/pkg/models/jsonschema/mappings.go similarity index 100% rename from pkg/manager/jsonschema/mappings.go rename to pkg/models/jsonschema/mappings.go diff --git a/pkg/manager/jsonschema/movie.go b/pkg/models/jsonschema/movie.go similarity index 100% rename from pkg/manager/jsonschema/movie.go rename to pkg/models/jsonschema/movie.go diff --git a/pkg/manager/jsonschema/performer.go b/pkg/models/jsonschema/performer.go similarity index 100% rename from pkg/manager/jsonschema/performer.go rename to pkg/models/jsonschema/performer.go diff --git a/pkg/manager/jsonschema/scene.go b/pkg/models/jsonschema/scene.go similarity index 100% rename from pkg/manager/jsonschema/scene.go rename to pkg/models/jsonschema/scene.go diff --git a/pkg/manager/jsonschema/scraped.go b/pkg/models/jsonschema/scraped.go similarity index 100% rename from pkg/manager/jsonschema/scraped.go rename to pkg/models/jsonschema/scraped.go diff --git a/pkg/manager/jsonschema/studio.go b/pkg/models/jsonschema/studio.go similarity index 100% rename from pkg/manager/jsonschema/studio.go rename to pkg/models/jsonschema/studio.go diff --git a/pkg/manager/jsonschema/tag.go b/pkg/models/jsonschema/tag.go similarity index 100% rename from pkg/manager/jsonschema/tag.go rename to pkg/models/jsonschema/tag.go diff --git a/pkg/manager/jsonschema/utils.go b/pkg/models/jsonschema/utils.go similarity index 100% rename from pkg/manager/jsonschema/utils.go rename to pkg/models/jsonschema/utils.go diff --git a/pkg/models/model_movie.go b/pkg/models/model_movie.go index 18bf49a74..b2e2631e3 100644 --- a/pkg/models/model_movie.go +++ b/pkg/models/model_movie.go @@ -4,7 +4,7 @@ import ( "database/sql" "time" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/hash/md5" ) type Movie struct { @@ -44,7 +44,7 @@ var DefaultMovieImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABkC func NewMovie(name string) *Movie { currentTime := time.Now() return &Movie{ - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Name: sql.NullString{String: name, Valid: true}, CreatedAt: SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: SQLiteTimestamp{Timestamp: currentTime}, diff --git a/pkg/models/model_performer.go b/pkg/models/model_performer.go index eefce07de..651e532d9 100644 --- a/pkg/models/model_performer.go +++ b/pkg/models/model_performer.go @@ -4,7 +4,7 @@ import ( "database/sql" "time" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/hash/md5" ) type Performer struct { @@ -68,7 +68,7 @@ type PerformerPartial struct { func NewPerformer(name string) *Performer { currentTime := time.Now() return &Performer{ - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Name: sql.NullString{String: name, Valid: true}, Favorite: sql.NullBool{Bool: false, Valid: true}, CreatedAt: SQLiteTimestamp{Timestamp: currentTime}, diff --git a/pkg/models/model_studio.go b/pkg/models/model_studio.go index 769acb8e2..34f9504d3 100644 --- a/pkg/models/model_studio.go +++ b/pkg/models/model_studio.go @@ -4,7 +4,7 @@ import ( "database/sql" "time" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/hash/md5" ) type Studio struct { @@ -36,7 +36,7 @@ var DefaultStudioImage = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGQAAABk func NewStudio(name string) *Studio { currentTime := time.Now() return &Studio{ - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Name: sql.NullString{String: name, Valid: true}, CreatedAt: SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: SQLiteTimestamp{Timestamp: currentTime}, diff --git a/pkg/manager/paths/paths.go b/pkg/models/paths/paths.go similarity index 83% rename from pkg/manager/paths/paths.go rename to pkg/models/paths/paths.go index d3216b6bc..fcd029e65 100644 --- a/pkg/manager/paths/paths.go +++ b/pkg/models/paths/paths.go @@ -3,7 +3,7 @@ package paths import ( "path/filepath" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/fsutil" ) type Paths struct { @@ -23,7 +23,7 @@ func NewPaths(generatedPath string) *Paths { } func GetStashHomeDirectory() string { - return filepath.Join(utils.GetHomeDirectory(), ".stash") + return filepath.Join(fsutil.GetHomeDirectory(), ".stash") } func GetDefaultDatabaseFilePath() string { diff --git a/pkg/manager/paths/paths_generated.go b/pkg/models/paths/paths_generated.go similarity index 85% rename from pkg/manager/paths/paths_generated.go rename to pkg/models/paths/paths_generated.go index a80f62d38..ff6d122be 100644 --- a/pkg/manager/paths/paths_generated.go +++ b/pkg/models/paths/paths_generated.go @@ -5,8 +5,8 @@ import ( "os" "path/filepath" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) const thumbDirDepth int = 2 @@ -41,15 +41,15 @@ func (gp *generatedPaths) GetTmpPath(fileName string) string { } func (gp *generatedPaths) EnsureTmpDir() error { - return utils.EnsureDir(gp.Tmp) + return fsutil.EnsureDir(gp.Tmp) } func (gp *generatedPaths) EmptyTmpDir() error { - return utils.EmptyDir(gp.Tmp) + return fsutil.EmptyDir(gp.Tmp) } func (gp *generatedPaths) RemoveTmpDir() error { - return utils.RemoveDir(gp.Tmp) + return fsutil.RemoveDir(gp.Tmp) } func (gp *generatedPaths) TempDir(pattern string) (string, error) { @@ -61,7 +61,7 @@ func (gp *generatedPaths) TempDir(pattern string) (string, error) { return "", err } - if err = utils.EmptyDir(ret); err != nil { + if err = fsutil.EmptyDir(ret); err != nil { logger.Warnf("could not recursively empty dir: %v", err) } @@ -70,5 +70,5 @@ func (gp *generatedPaths) TempDir(pattern string) (string, error) { func (gp *generatedPaths) GetThumbnailPath(checksum string, width int) string { fname := fmt.Sprintf("%s_%d.jpg", checksum, width) - return filepath.Join(gp.Thumbnails, utils.GetIntraDir(checksum, thumbDirDepth, thumbDirLength), fname) + return filepath.Join(gp.Thumbnails, fsutil.GetIntraDir(checksum, thumbDirDepth, thumbDirLength), fname) } diff --git a/pkg/manager/paths/paths_json.go b/pkg/models/paths/paths_json.go similarity index 81% rename from pkg/manager/paths/paths_json.go rename to pkg/models/paths/paths_json.go index 1bd6a5e59..486ffe71d 100644 --- a/pkg/manager/paths/paths_json.go +++ b/pkg/models/paths/paths_json.go @@ -3,8 +3,8 @@ package paths import ( "path/filepath" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) type JSONPaths struct { @@ -44,28 +44,28 @@ func GetJSONPaths(baseDir string) *JSONPaths { func EnsureJSONDirs(baseDir string) { jsonPaths := GetJSONPaths(baseDir) - if err := utils.EnsureDir(jsonPaths.Metadata); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Metadata); err != nil { logger.Warnf("couldn't create directories for Metadata: %v", err) } - if err := utils.EnsureDir(jsonPaths.Scenes); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Scenes); err != nil { logger.Warnf("couldn't create directories for Scenes: %v", err) } - if err := utils.EnsureDir(jsonPaths.Images); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Images); err != nil { logger.Warnf("couldn't create directories for Images: %v", err) } - if err := utils.EnsureDir(jsonPaths.Galleries); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Galleries); err != nil { logger.Warnf("couldn't create directories for Galleries: %v", err) } - if err := utils.EnsureDir(jsonPaths.Performers); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Performers); err != nil { logger.Warnf("couldn't create directories for Performers: %v", err) } - if err := utils.EnsureDir(jsonPaths.Studios); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Studios); err != nil { logger.Warnf("couldn't create directories for Studios: %v", err) } - if err := utils.EnsureDir(jsonPaths.Movies); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Movies); err != nil { logger.Warnf("couldn't create directories for Movies: %v", err) } - if err := utils.EnsureDir(jsonPaths.Tags); err != nil { + if err := fsutil.EnsureDir(jsonPaths.Tags); err != nil { logger.Warnf("couldn't create directories for Tags: %v", err) } } diff --git a/pkg/manager/paths/paths_scene_markers.go b/pkg/models/paths/paths_scene_markers.go similarity index 100% rename from pkg/manager/paths/paths_scene_markers.go rename to pkg/models/paths/paths_scene_markers.go diff --git a/pkg/manager/paths/paths_scenes.go b/pkg/models/paths/paths_scenes.go similarity index 94% rename from pkg/manager/paths/paths_scenes.go rename to pkg/models/paths/paths_scenes.go index 336d95466..01135ca45 100644 --- a/pkg/manager/paths/paths_scenes.go +++ b/pkg/models/paths/paths_scenes.go @@ -3,7 +3,7 @@ package paths import ( "path/filepath" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/fsutil" ) type scenePaths struct { @@ -30,7 +30,7 @@ func (sp *scenePaths) GetTranscodePath(checksum string) string { func (sp *scenePaths) GetStreamPath(scenePath string, checksum string) string { transcodePath := sp.GetTranscodePath(checksum) - transcodeExists, _ := utils.FileExists(transcodePath) + transcodeExists, _ := fsutil.FileExists(transcodePath) if transcodeExists { return transcodePath } diff --git a/pkg/movie/export.go b/pkg/movie/export.go index 74d3c892c..fa682bdc4 100644 --- a/pkg/movie/export.go +++ b/pkg/movie/export.go @@ -3,8 +3,8 @@ package movie import ( "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/movie/export_test.go b/pkg/movie/export_test.go index d9fed193a..7e5567ae8 100644 --- a/pkg/movie/export_test.go +++ b/pkg/movie/export_test.go @@ -4,8 +4,8 @@ import ( "database/sql" "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" diff --git a/pkg/movie/import.go b/pkg/movie/import.go index 97a5c7e33..6afdef8b9 100644 --- a/pkg/movie/import.go +++ b/pkg/movie/import.go @@ -4,8 +4,9 @@ import ( "database/sql" "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) @@ -29,13 +30,13 @@ func (i *Importer) PreImport() error { var err error if len(i.Input.FrontImage) > 0 { - _, i.frontImageData, err = utils.ProcessBase64Image(i.Input.FrontImage) + i.frontImageData, err = utils.ProcessBase64Image(i.Input.FrontImage) if err != nil { return fmt.Errorf("invalid front_image: %v", err) } } if len(i.Input.BackImage) > 0 { - _, i.backImageData, err = utils.ProcessBase64Image(i.Input.BackImage) + i.backImageData, err = utils.ProcessBase64Image(i.Input.BackImage) if err != nil { return fmt.Errorf("invalid back_image: %v", err) } @@ -45,7 +46,7 @@ func (i *Importer) PreImport() error { } func (i *Importer) movieJSONToMovie(movieJSON jsonschema.Movie) models.Movie { - checksum := utils.MD5FromString(movieJSON.Name) + checksum := md5.FromString(movieJSON.Name) newMovie := models.Movie{ Checksum: checksum, diff --git a/pkg/movie/import_test.go b/pkg/movie/import_test.go index e81e1a9f1..7aff71d47 100644 --- a/pkg/movie/import_test.go +++ b/pkg/movie/import_test.go @@ -4,8 +4,8 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/performer/export.go b/pkg/performer/export.go index 55e3beaf0..2b7b45ab1 100644 --- a/pkg/performer/export.go +++ b/pkg/performer/export.go @@ -3,8 +3,8 @@ package performer import ( "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/performer/export_test.go b/pkg/performer/export_test.go index 1851bd13f..83c9f0fb6 100644 --- a/pkg/performer/export_test.go +++ b/pkg/performer/export_test.go @@ -4,10 +4,10 @@ import ( "database/sql" "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" - "github.com/stashapp/stash/pkg/utils" "github.com/stretchr/testify/assert" "testing" @@ -72,7 +72,7 @@ func createFullPerformer(id int, name string) *models.Performer { return &models.Performer{ ID: id, Name: models.NullString(name), - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), URL: models.NullString(url), Aliases: models.NullString(aliases), Birthdate: birthDate, diff --git a/pkg/performer/import.go b/pkg/performer/import.go index e4589995d..071a62dda 100644 --- a/pkg/performer/import.go +++ b/pkg/performer/import.go @@ -5,8 +5,10 @@ import ( "fmt" "strings" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -32,7 +34,7 @@ func (i *Importer) PreImport() error { var err error if len(i.Input.Image) > 0 { - _, i.imageData, err = utils.ProcessBase64Image(i.Input.Image) + i.imageData, err = utils.ProcessBase64Image(i.Input.Image) if err != nil { return fmt.Errorf("invalid image: %v", err) } @@ -66,8 +68,8 @@ func importTags(tagWriter models.TagReaderWriter, names []string, missingRefBeha pluckedNames = append(pluckedNames, tag.Name) } - missingTags := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingTags := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingTags) > 0 { @@ -173,7 +175,7 @@ func (i *Importer) Update(id int) error { } func performerJSONToPerformer(performerJSON jsonschema.Performer) models.Performer { - checksum := utils.MD5FromString(performerJSON.Name) + checksum := md5.FromString(performerJSON.Name) newPerformer := models.Performer{ Checksum: checksum, diff --git a/pkg/performer/import_test.go b/pkg/performer/import_test.go index 13598f047..30ddbae5e 100644 --- a/pkg/performer/import_test.go +++ b/pkg/performer/import_test.go @@ -5,10 +5,10 @@ import ( "github.com/stretchr/testify/mock" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" - "github.com/stashapp/stash/pkg/utils" "github.com/stretchr/testify/assert" "testing" @@ -57,7 +57,7 @@ func TestImporterPreImport(t *testing.T) { assert.Nil(t, err) expectedPerformer := *createFullPerformer(0, performerName) - expectedPerformer.Checksum = utils.MD5FromString(performerName) + expectedPerformer.Checksum = md5.FromString(performerName) assert.Equal(t, expectedPerformer, i.performer) } diff --git a/pkg/plugin/common/log/log.go b/pkg/plugin/common/log/log.go index e64774a68..0e9732c2e 100644 --- a/pkg/plugin/common/log/log.go +++ b/pkg/plugin/common/log/log.go @@ -126,20 +126,3 @@ func LevelFromName(name string) *Level { return nil } - -// DetectLogLevel returns the Level and the logging string for a provided line -// of plugin output. It parses the string for logging control characters and -// determines the log level, if present. If not present, the plugin output -// is returned unchanged with a nil Level. -func DetectLogLevel(line string) (*Level, string) { - var level *logger.PluginLogLevel - level, line = logger.DetectLogLevel(line) - - if level == nil { - return nil, line - } - - return &Level{ - level, - }, line -} diff --git a/pkg/plugin/log.go b/pkg/plugin/log.go index 0b8511521..eaca8008f 100644 --- a/pkg/plugin/log.go +++ b/pkg/plugin/log.go @@ -17,10 +17,11 @@ func (t *pluginTask) handlePluginStderr(name string, pluginOutputReader io.ReadC const pluginPrefix = "[Plugin / %s] " lgr := logger.PluginLogger{ + Logger: logger.Logger, Prefix: fmt.Sprintf(pluginPrefix, name), DefaultLogLevel: logLevel, ProgressChan: t.progress, } - lgr.HandlePluginStdErr(pluginOutputReader) + lgr.ReadLogMessages(pluginOutputReader) } diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 8f88e3d06..4f635b150 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -16,16 +16,23 @@ import ( "strconv" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin/common" "github.com/stashapp/stash/pkg/session" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) +type ServerConfig interface { + GetHost() string + GetPort() int + GetConfigPath() string + HasTLSConfig() bool + GetPluginsPath() string +} + // Cache stores plugin details. type Cache struct { - config *config.Instance + config ServerConfig plugins []Config sessionStore *session.Store gqlHandler http.Handler @@ -38,7 +45,7 @@ type Cache struct { // // Does not load plugins. Plugins will need to be // loaded explicitly using ReloadPlugins. -func NewCache(config *config.Instance) *Cache { +func NewCache(config ServerConfig) *Cache { return &Cache{ config: config, } @@ -196,7 +203,7 @@ func (c Cache) executePostHooks(ctx context.Context, hookType HookTriggerEnum, h hooks := p.getHooks(hookType) // don't revisit a plugin we've already visited // only log if there's hooks that we're skipping - if len(hooks) > 0 && utils.StrInclude(visitedPlugins, p.id) { + if len(hooks) > 0 && stringslice.StrInclude(visitedPlugins, p.id) { logger.Debugf("plugin ID '%s' already triggered, not re-triggering", p.id) continue } diff --git a/pkg/plugin/raw.go b/pkg/plugin/raw.go index 6456cab4c..48f3064d7 100644 --- a/pkg/plugin/raw.go +++ b/pkg/plugin/raw.go @@ -8,7 +8,7 @@ import ( "os/exec" "sync" - "github.com/stashapp/stash/pkg/desktop" + stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/plugin/common" ) @@ -60,7 +60,7 @@ func (t *rawPluginTask) Start() error { } } - cmd := exec.Command(command[0], command[1:]...) + cmd := stashExec.Command(command[0], command[1:]...) stdin, err := cmd.StdinPipe() if err != nil { @@ -88,7 +88,6 @@ func (t *rawPluginTask) Start() error { t.waitGroup.Add(1) t.done = make(chan bool, 1) - desktop.HideExecShell(cmd) if err = cmd.Start(); err != nil { return fmt.Errorf("error running plugin: %v", err) } diff --git a/pkg/scene/delete.go b/pkg/scene/delete.go index 63672eecb..45456171b 100644 --- a/pkg/scene/delete.go +++ b/pkg/scene/delete.go @@ -4,9 +4,9 @@ import ( "path/filepath" "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/paths" ) // FileDeleter is an extension of file.Deleter that handles deletion of scene files. @@ -27,7 +27,7 @@ func (d *FileDeleter) MarkGeneratedFiles(scene *models.Scene) error { markersFolder := filepath.Join(d.Paths.Generated.Markers, sceneHash) - exists, _ := utils.FileExists(markersFolder) + exists, _ := fsutil.FileExists(markersFolder) if exists { if err := d.Dirs([]string{markersFolder}); err != nil { return err @@ -37,49 +37,49 @@ func (d *FileDeleter) MarkGeneratedFiles(scene *models.Scene) error { var files []string thumbPath := d.Paths.Scene.GetThumbnailScreenshotPath(sceneHash) - exists, _ = utils.FileExists(thumbPath) + exists, _ = fsutil.FileExists(thumbPath) if exists { files = append(files, thumbPath) } normalPath := d.Paths.Scene.GetScreenshotPath(sceneHash) - exists, _ = utils.FileExists(normalPath) + exists, _ = fsutil.FileExists(normalPath) if exists { files = append(files, normalPath) } streamPreviewPath := d.Paths.Scene.GetStreamPreviewPath(sceneHash) - exists, _ = utils.FileExists(streamPreviewPath) + exists, _ = fsutil.FileExists(streamPreviewPath) if exists { files = append(files, streamPreviewPath) } streamPreviewImagePath := d.Paths.Scene.GetStreamPreviewImagePath(sceneHash) - exists, _ = utils.FileExists(streamPreviewImagePath) + exists, _ = fsutil.FileExists(streamPreviewImagePath) if exists { files = append(files, streamPreviewImagePath) } transcodePath := d.Paths.Scene.GetTranscodePath(sceneHash) - exists, _ = utils.FileExists(transcodePath) + exists, _ = fsutil.FileExists(transcodePath) if exists { files = append(files, transcodePath) } spritePath := d.Paths.Scene.GetSpriteImageFilePath(sceneHash) - exists, _ = utils.FileExists(spritePath) + exists, _ = fsutil.FileExists(spritePath) if exists { files = append(files, spritePath) } vttPath := d.Paths.Scene.GetSpriteVttFilePath(sceneHash) - exists, _ = utils.FileExists(vttPath) + exists, _ = fsutil.FileExists(vttPath) if exists { files = append(files, vttPath) } heatmapPath := d.Paths.Scene.GetInteractiveHeatmapPath(sceneHash) - exists, _ = utils.FileExists(heatmapPath) + exists, _ = fsutil.FileExists(heatmapPath) if exists { files = append(files, heatmapPath) } @@ -96,17 +96,17 @@ func (d *FileDeleter) MarkMarkerFiles(scene *models.Scene, seconds int) error { var files []string - exists, _ := utils.FileExists(videoPath) + exists, _ := fsutil.FileExists(videoPath) if exists { files = append(files, videoPath) } - exists, _ = utils.FileExists(imagePath) + exists, _ = fsutil.FileExists(imagePath) if exists { files = append(files, imagePath) } - exists, _ = utils.FileExists(screenshotPath) + exists, _ = fsutil.FileExists(screenshotPath) if exists { files = append(files, screenshotPath) } @@ -136,8 +136,8 @@ func Destroy(scene *models.Scene, repo models.Repository, fileDeleter *FileDelet return err } - funscriptPath := utils.GetFunscriptPath(scene.Path) - funscriptExists, _ := utils.FileExists(funscriptPath) + funscriptPath := GetFunscriptPath(scene.Path) + funscriptExists, _ := fsutil.FileExists(funscriptPath) if funscriptExists { if err := fileDeleter.Files([]string{funscriptPath}); err != nil { return err diff --git a/pkg/scene/export.go b/pkg/scene/export.go index 352ce32b9..38ae5f0b1 100644 --- a/pkg/scene/export.go +++ b/pkg/scene/export.go @@ -5,8 +5,9 @@ import ( "math" "strconv" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/utils" ) @@ -175,7 +176,7 @@ func GetDependentTagIDs(tags models.TagReader, markerReader models.SceneMarkerRe } for _, tt := range t { - ret = utils.IntAppendUnique(ret, tt.ID) + ret = intslice.IntAppendUnique(ret, tt.ID) } sm, err := markerReader.FindBySceneID(scene.ID) @@ -184,14 +185,14 @@ func GetDependentTagIDs(tags models.TagReader, markerReader models.SceneMarkerRe } for _, smm := range sm { - ret = utils.IntAppendUnique(ret, smm.PrimaryTagID) + ret = intslice.IntAppendUnique(ret, smm.PrimaryTagID) smmt, err := tags.FindBySceneMarkerID(smm.ID) if err != nil { return nil, fmt.Errorf("invalid tags for scene marker: %v", err) } for _, smmtt := range smmt { - ret = utils.IntAppendUnique(ret, smmtt.ID) + ret = intslice.IntAppendUnique(ret, smmtt.ID) } } diff --git a/pkg/scene/export_test.go b/pkg/scene/export_test.go index 15aec8f4c..d87a9fadd 100644 --- a/pkg/scene/export_test.go +++ b/pkg/scene/export_test.go @@ -4,8 +4,8 @@ import ( "database/sql" "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stashapp/stash/pkg/utils" "github.com/stretchr/testify/assert" diff --git a/pkg/scene/funscript.go b/pkg/scene/funscript.go new file mode 100644 index 000000000..8a28d3e77 --- /dev/null +++ b/pkg/scene/funscript.go @@ -0,0 +1,14 @@ +package scene + +import ( + "path/filepath" + "strings" +) + +// GetFunscriptPath returns the path of a file +// with the extension changed to .funscript +func GetFunscriptPath(path string) string { + ext := filepath.Ext(path) + fn := strings.TrimSuffix(path, ext) + return fn + ".funscript" +} diff --git a/pkg/scene/import.go b/pkg/scene/import.go index abd087ea1..103be88fd 100644 --- a/pkg/scene/import.go +++ b/pkg/scene/import.go @@ -6,8 +6,9 @@ import ( "strconv" "strings" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -57,7 +58,7 @@ func (i *Importer) PreImport() error { var err error if len(i.Input.Cover) > 0 { - _, i.coverImageData, err = utils.ProcessBase64Image(i.Input.Cover) + i.coverImageData, err = utils.ProcessBase64Image(i.Input.Cover) if err != nil { return fmt.Errorf("invalid cover image: %v", err) } @@ -192,8 +193,8 @@ func (i *Importer) populateGalleries() error { pluckedChecksums = append(pluckedChecksums, gallery.Checksum) } - missingGalleries := utils.StrFilter(checksums, func(checksum string) bool { - return !utils.StrInclude(pluckedChecksums, checksum) + missingGalleries := stringslice.StrFilter(checksums, func(checksum string) bool { + return !stringslice.StrInclude(pluckedChecksums, checksum) }) if len(missingGalleries) > 0 { @@ -226,8 +227,8 @@ func (i *Importer) populatePerformers() error { pluckedNames = append(pluckedNames, performer.Name.String) } - missingPerformers := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingPerformers := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingPerformers) > 0 { @@ -458,8 +459,8 @@ func importTags(tagWriter models.TagReaderWriter, names []string, missingRefBeha pluckedNames = append(pluckedNames, tag.Name) } - missingTags := utils.StrFilter(names, func(name string) bool { - return !utils.StrInclude(pluckedNames, name) + missingTags := stringslice.StrFilter(names, func(name string) bool { + return !stringslice.StrInclude(pluckedNames, name) }) if len(missingTags) > 0 { diff --git a/pkg/scene/import_test.go b/pkg/scene/import_test.go index 2a8122551..499f27299 100644 --- a/pkg/scene/import_test.go +++ b/pkg/scene/import_test.go @@ -4,8 +4,8 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/scene/marker_import.go b/pkg/scene/marker_import.go index 2258025fe..530d025ea 100644 --- a/pkg/scene/marker_import.go +++ b/pkg/scene/marker_import.go @@ -5,8 +5,8 @@ import ( "fmt" "strconv" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" ) type MarkerImporter struct { diff --git a/pkg/scene/marker_import_test.go b/pkg/scene/marker_import_test.go index f70700a5c..0aa72a08b 100644 --- a/pkg/scene/marker_import_test.go +++ b/pkg/scene/marker_import_test.go @@ -4,8 +4,8 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/scene/migrate_hash.go b/pkg/scene/migrate_hash.go index 769b65091..ebc00bfb8 100644 --- a/pkg/scene/migrate_hash.go +++ b/pkg/scene/migrate_hash.go @@ -4,9 +4,9 @@ import ( "os" "path/filepath" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/paths" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/models/paths" ) func MigrateHash(p *paths.Paths, oldHash string, newHash string) { @@ -49,7 +49,7 @@ func MigrateHash(p *paths.Paths, oldHash string, newHash string) { } func migrateSceneFiles(oldName, newName string) { - oldExists, err := utils.FileExists(oldName) + oldExists, err := fsutil.FileExists(oldName) if err != nil && !os.IsNotExist(err) { logger.Errorf("Error checking existence of %s: %s", oldName, err.Error()) return diff --git a/pkg/scene/scan.go b/pkg/scene/scan.go index 8d5f427b4..04c7d67a7 100644 --- a/pkg/scene/scan.go +++ b/pkg/scene/scan.go @@ -11,10 +11,10 @@ import ( "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/utils" ) @@ -61,7 +61,6 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi path := scanned.New.Path interactive := getInteractive(path) - config := config.GetInstance() oldHash := s.GetHash(scanner.FileNamingAlgorithm) changed := false @@ -140,7 +139,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi } // Migrate any generated files if the hash has changed - newHash := s.GetHash(config.GetVideoFileNamingAlgorithm()) + newHash := s.GetHash(scanner.FileNamingAlgorithm) if newHash != oldHash { MigrateHash(scanner.Paths, oldHash, newHash) } @@ -203,7 +202,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e interactive := getInteractive(file.Path()) if s != nil { - exists, _ := utils.FileExists(s.Path) + exists, _ := fsutil.FileExists(s.Path) if !scanner.CaseSensitiveFs { // #1426 - if file exists but is a case-insensitive match for the // original filename, then treat it as a move @@ -299,8 +298,8 @@ func (scanner *Scanner) makeScreenshots(path string, probeResult *ffmpeg.VideoFi thumbPath := scanner.Paths.Scene.GetThumbnailScreenshotPath(checksum) normalPath := scanner.Paths.Scene.GetScreenshotPath(checksum) - thumbExists, _ := utils.FileExists(thumbPath) - normalExists, _ := utils.FileExists(normalPath) + thumbExists, _ := fsutil.FileExists(thumbPath) + normalExists, _ := fsutil.FileExists(normalPath) if thumbExists && normalExists { return @@ -331,6 +330,6 @@ func (scanner *Scanner) makeScreenshots(path string, probeResult *ffmpeg.VideoFi } func getInteractive(path string) bool { - _, err := os.Stat(utils.GetFunscriptPath(path)) + _, err := os.Stat(GetFunscriptPath(path)) return err == nil } diff --git a/pkg/scene/screenshot.go b/pkg/scene/screenshot.go index 7af8ca3e4..dca044de3 100644 --- a/pkg/scene/screenshot.go +++ b/pkg/scene/screenshot.go @@ -8,8 +8,8 @@ import ( "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/disintegration/imaging" diff --git a/pkg/scene/update.go b/pkg/scene/update.go index d351b13eb..e1155c368 100644 --- a/pkg/scene/update.go +++ b/pkg/scene/update.go @@ -7,6 +7,7 @@ import ( "time" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/utils" ) @@ -100,11 +101,11 @@ func (u UpdateSet) UpdateInput() models.SceneUpdateInput { ret := u.Partial.UpdateInput() if u.PerformerIDs != nil { - ret.PerformerIds = utils.IntSliceToStringSlice(u.PerformerIDs) + ret.PerformerIds = intslice.IntSliceToStringSlice(u.PerformerIDs) } if u.TagIDs != nil { - ret.TagIds = utils.IntSliceToStringSlice(u.TagIDs) + ret.TagIds = intslice.IntSliceToStringSlice(u.TagIDs) } if u.StashIDs != nil { @@ -167,7 +168,7 @@ func AddPerformer(qb models.SceneReaderWriter, id int, performerID int) (bool, e } oldLen := len(performerIDs) - performerIDs = utils.IntAppendUnique(performerIDs, performerID) + performerIDs = intslice.IntAppendUnique(performerIDs, performerID) if len(performerIDs) != oldLen { if err := qb.UpdatePerformers(id, performerIDs); err != nil { @@ -187,7 +188,7 @@ func AddTag(qb models.SceneReaderWriter, id int, tagID int) (bool, error) { } oldLen := len(tagIDs) - tagIDs = utils.IntAppendUnique(tagIDs, tagID) + tagIDs = intslice.IntAppendUnique(tagIDs, tagID) if len(tagIDs) != oldLen { if err := qb.UpdateTags(id, tagIDs); err != nil { @@ -207,7 +208,7 @@ func AddGallery(qb models.SceneReaderWriter, id int, galleryID int) (bool, error } oldLen := len(galleryIDs) - galleryIDs = utils.IntAppendUnique(galleryIDs, galleryID) + galleryIDs = intslice.IntAppendUnique(galleryIDs, galleryID) if len(galleryIDs) != oldLen { if err := qb.UpdateGalleries(id, galleryIDs); err != nil { diff --git a/pkg/scene/update_test.go b/pkg/scene/update_test.go index 587258b93..4619fd137 100644 --- a/pkg/scene/update_test.go +++ b/pkg/scene/update_test.go @@ -7,7 +7,7 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/mocks" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -261,9 +261,9 @@ func TestUpdateSet_UpdateInput(t *testing.T) { sceneIDStr := strconv.Itoa(sceneID) performerIDs := []int{performerID} - performerIDStrs := utils.IntSliceToStringSlice(performerIDs) + performerIDStrs := intslice.IntSliceToStringSlice(performerIDs) tagIDs := []int{tagID} - tagIDStrs := utils.IntSliceToStringSlice(tagIDs) + tagIDStrs := intslice.IntSliceToStringSlice(tagIDs) stashID := "stashID" endpoint := "endpoint" stashIDs := []models.StashID{ diff --git a/pkg/scraper/cache.go b/pkg/scraper/cache.go index d378f9c9b..e48cc51c2 100644 --- a/pkg/scraper/cache.go +++ b/pkg/scraper/cache.go @@ -11,9 +11,9 @@ import ( "strings" "time" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) const ( @@ -36,6 +36,7 @@ type GlobalConfig interface { GetScrapersPath() string GetScraperCDPPath() string GetScraperCertCheck() bool + GetScraperExcludeTagPatterns() []string } func isCDPPathHTTP(c GlobalConfig) bool { @@ -110,7 +111,7 @@ func loadScrapers(globalConfig GlobalConfig, txnManager models.TransactionManage logger.Debugf("Reading scraper configs from %s", path) scraperFiles := []string{} - err := utils.SymWalk(path, func(fp string, f os.FileInfo, err error) error { + err := fsutil.SymWalk(path, func(fp string, f os.FileInfo, err error) error { if filepath.Ext(fp) == ".yml" { c, err := loadConfigFromYAMLFile(fp) if err != nil { diff --git a/pkg/scraper/cookies.go b/pkg/scraper/cookies.go index ed854d50a..72855441f 100644 --- a/pkg/scraper/cookies.go +++ b/pkg/scraper/cookies.go @@ -3,6 +3,7 @@ package scraper import ( "context" "fmt" + "math/rand" "net/http" "net/http/cookiejar" "net/url" @@ -14,7 +15,6 @@ import ( "golang.org/x/net/publicsuffix" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/utils" ) // jar constructs a cookie jar from a configuration @@ -60,11 +60,22 @@ func (c config) jar() (*cookiejar.Jar, error) { func getCookieValue(cookie *scraperCookies) string { if cookie.ValueRandom > 0 { - return utils.RandomSequence(cookie.ValueRandom) + return randomSequence(cookie.ValueRandom) } return cookie.Value } +var characters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") + +func randomSequence(n int) string { + b := make([]rune, n) + rand.Seed(time.Now().UnixNano()) + for i := range b { + b[i] = characters[rand.Intn(len(characters))] + } + return string(b) +} + // printCookies prints all cookies from the given cookie jar func printCookies(jar *cookiejar.Jar, scraperConfig config, msg string) { driverOptions := scraperConfig.DriverOptions diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index d9363c4b2..4753bb182 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -13,7 +13,7 @@ import ( "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "gopkg.in/yaml.v2" ) @@ -680,8 +680,8 @@ func (c mappedScraperAttrConfig) concatenateResults(nodes []string) string { } func (c mappedScraperAttrConfig) cleanResults(nodes []string) []string { - cleaned := utils.StrUnique(nodes) // remove duplicate values - cleaned = utils.StrDelete(cleaned, "") // remove empty values + cleaned := stringslice.StrUnique(nodes) // remove duplicate values + cleaned = stringslice.StrDelete(cleaned, "") // remove empty values return cleaned } diff --git a/pkg/scraper/postprocessing.go b/pkg/scraper/postprocessing.go index 29c2df1c6..4be2ea0ce 100644 --- a/pkg/scraper/postprocessing.go +++ b/pkg/scraper/postprocessing.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/stashapp/stash/pkg/logger" - stash_config "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/match" "github.com/stashapp/stash/pkg/models" ) @@ -51,7 +50,7 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { tqb := r.Tag() - tags, err := postProcessTags(tqb, p.Tags) + tags, err := postProcessTags(c.globalConfig, tqb, p.Tags) if err != nil { return err } @@ -94,7 +93,7 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { tqb := r.Tag() - tags, err := postProcessTags(tqb, p.Tags) + tags, err := postProcessTags(c.globalConfig, tqb, p.Tags) if err != nil { return err } @@ -136,7 +135,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene) ( } } - tags, err := postProcessTags(tqb, scene.Tags) + tags, err := postProcessTags(c.globalConfig, tqb, scene.Tags) if err != nil { return err } @@ -175,7 +174,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery) ( } } - tags, err := postProcessTags(tqb, g.Tags) + tags, err := postProcessTags(c.globalConfig, tqb, g.Tags) if err != nil { return err } @@ -196,10 +195,10 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery) ( return g, nil } -func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) { +func postProcessTags(globalConfig GlobalConfig, tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) { var ret []*models.ScrapedTag - excludePatterns := stash_config.GetInstance().GetScraperExcludeTagPatterns() + excludePatterns := globalConfig.GetScraperExcludeTagPatterns() var excludeRegexps []*regexp.Regexp for _, excludePattern := range excludePatterns { diff --git a/pkg/scraper/script.go b/pkg/scraper/script.go index 0e1e90c59..d182b1ee3 100644 --- a/pkg/scraper/script.go +++ b/pkg/scraper/script.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - "github.com/stashapp/stash/pkg/desktop" + stashExec "github.com/stashapp/stash/pkg/exec" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" ) @@ -41,7 +41,7 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error } } - cmd := exec.Command(command[0], command[1:]...) + cmd := stashExec.Command(command[0], command[1:]...) cmd.Dir = filepath.Dir(s.config.path) stdin, err := cmd.StdinPipe() @@ -67,7 +67,6 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error logger.Error("Scraper stdout not available: " + err.Error()) } - desktop.HideExecShell(cmd) if err = cmd.Start(); err != nil { logger.Error("Error running scraper script: " + err.Error()) return errors.New("error running scraper script") @@ -241,8 +240,9 @@ func handleScraperStderr(name string, scraperOutputReader io.ReadCloser) { const scraperPrefix = "[Scrape / %s] " lgr := logger.PluginLogger{ + Logger: logger.Logger, Prefix: fmt.Sprintf(scraperPrefix, name), DefaultLogLevel: &logger.ErrorLevel, } - lgr.HandlePluginStdErr(scraperOutputReader) + lgr.ReadLogMessages(scraperOutputReader) } diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 0d761a6e9..0a48cb9f5 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -16,10 +16,13 @@ import ( "github.com/Yamashou/gqlgenc/graphqljson" "github.com/corona10/goimagehash" + "github.com/stashapp/stash/pkg/fsutil" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/match" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scraper/stashbox/graphql" + "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" "github.com/stashapp/stash/pkg/utils" ) @@ -88,7 +91,7 @@ func phashMatches(hash, other int64) bool { // scene's MD5/OSHASH checksum, or PHash, and returns results in the same order // as the input slice. func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, sceneIDs []string) ([][]*models.ScrapedScene, error) { - ids, err := utils.StringSliceToIntSlice(sceneIDs) + ids, err := stringslice.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err } @@ -155,7 +158,7 @@ func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, sceneIDs [ addScene := func(sceneIndexes []int) { for _, index := range sceneIndexes { - if !utils.IntInclude(addedTo, index) { + if !intslice.IntInclude(addedTo, index) { addedTo = append(addedTo, index) ret[index] = append(ret[index], s) } @@ -187,7 +190,7 @@ func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, sceneIDs [ // FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every // scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice. func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneIDs []string) ([]*models.ScrapedScene, error) { - ids, err := utils.StringSliceToIntSlice(sceneIDs) + ids, err := stringslice.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err } @@ -265,7 +268,7 @@ func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprin } func (c Client) SubmitStashBoxFingerprints(ctx context.Context, sceneIDs []string, endpoint string) (bool, error) { - ids, err := utils.StringSliceToIntSlice(sceneIDs) + ids, err := stringslice.StringSliceToIntSlice(sceneIDs) if err != nil { return false, err } @@ -395,7 +398,7 @@ func (c Client) queryStashBoxPerformer(ctx context.Context, queryStr string) ([] // FindStashBoxPerformersByNames queries stash-box for performers by name func (c Client) FindStashBoxPerformersByNames(ctx context.Context, performerIDs []string) ([]*models.StashBoxPerformerQueryResult, error) { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return nil, err } @@ -429,7 +432,7 @@ func (c Client) FindStashBoxPerformersByNames(ctx context.Context, performerIDs } func (c Client) FindStashBoxPerformersByPerformerNames(ctx context.Context, performerIDs []string) ([][]*models.ScrapedPerformer, error) { - ids, err := utils.StringSliceToIntSlice(performerIDs) + ids, err := stringslice.StringSliceToIntSlice(performerIDs) if err != nil { return nil, err } @@ -911,7 +914,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, sceneID int, endpoint stri } draft.Tags = tags - exists, _ := utils.FileExists(imagePath) + exists, _ := fsutil.FileExists(imagePath) if exists { file, err := os.Open(imagePath) if err == nil { diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 315f2bc8f..7f0ab25d5 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -826,6 +826,10 @@ func (mockGlobalConfig) GetScraperCertCheck() bool { return false } +func (mockGlobalConfig) GetScraperExcludeTagPatterns() []string { + return nil +} + func TestSubScrape(t *testing.T) { retHTML := `
diff --git a/pkg/session/authentication.go b/pkg/session/authentication.go index 503fe777d..3d756d7a2 100644 --- a/pkg/session/authentication.go +++ b/pkg/session/authentication.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" ) type ExternalAccessError net.IP @@ -16,7 +15,7 @@ func (e ExternalAccessError) Error() string { return fmt.Sprintf("stash accessed from external IP %s", net.IP(e).String()) } -func CheckAllowPublicWithoutAuth(c *config.Instance, r *http.Request) error { +func CheckAllowPublicWithoutAuth(c ExternalAccessConfig, r *http.Request) error { if !c.HasCredentials() && !c.GetDangerousAllowPublicWithoutAuth() && !c.IsNewSystem() { requestIPString, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { @@ -60,7 +59,7 @@ func CheckAllowPublicWithoutAuth(c *config.Instance, r *http.Request) error { return nil } -func CheckExternalAccessTripwire(c *config.Instance) *ExternalAccessError { +func CheckExternalAccessTripwire(c ExternalAccessConfig) *ExternalAccessError { if !c.HasCredentials() && !c.GetDangerousAllowPublicWithoutAuth() { if remoteIP := c.GetSecurityTripwireAccessedFromPublicInternet(); remoteIP != "" { err := ExternalAccessError(net.ParseIP(remoteIP)) diff --git a/pkg/session/authentication_test.go b/pkg/session/authentication_test.go index 6a660bc5c..ac6383f24 100644 --- a/pkg/session/authentication_test.go +++ b/pkg/session/authentication_test.go @@ -4,13 +4,33 @@ import ( "errors" "net/http" "testing" - - "github.com/stashapp/stash/pkg/manager/config" ) +type config struct { + username string + password string + dangerousAllowPublicWithoutAuth bool + securityTripwireAccessedFromPublicInternet string +} + +func (c *config) HasCredentials() bool { + return c.username != "" && c.password != "" +} + +func (c *config) GetDangerousAllowPublicWithoutAuth() bool { + return c.dangerousAllowPublicWithoutAuth +} + +func (c *config) GetSecurityTripwireAccessedFromPublicInternet() string { + return c.securityTripwireAccessedFromPublicInternet +} + +func (c *config) IsNewSystem() bool { + return false +} + func TestCheckAllowPublicWithoutAuth(t *testing.T) { - c := config.GetInstance() - _ = c.SetInitialMemoryConfig() + c := &config{} doTest := func(caseIndex int, r *http.Request, expectedErr interface{}) { t.Helper() @@ -114,18 +134,17 @@ func TestCheckAllowPublicWithoutAuth(t *testing.T) { RemoteAddr: "193.168.1.1:8080", } - c.Set(config.Username, "admin") - c.Set(config.Password, "admin") + c.username = "admin" + c.password = "admin" if err := CheckAllowPublicWithoutAuth(c, r); err != nil { t.Errorf("unexpected error: %v", err) } - c.Set(config.Username, "") - c.Set(config.Password, "") + c.username = "" + c.password = "" - // HACK - this key isn't publically exposed - c.Set("dangerous_allow_public_without_auth", true) + c.dangerousAllowPublicWithoutAuth = true if err := CheckAllowPublicWithoutAuth(c, r); err != nil { t.Errorf("unexpected error: %v", err) @@ -134,36 +153,34 @@ func TestCheckAllowPublicWithoutAuth(t *testing.T) { } func TestCheckExternalAccessTripwire(t *testing.T) { - c := config.GetInstance() - _ = c.SetInitialMemoryConfig() - - c.Set(config.SecurityTripwireAccessedFromPublicInternet, "4.4.4.4") + c := &config{} + c.securityTripwireAccessedFromPublicInternet = "4.4.4.4" // always return nil if authentication configured or dangerous key set - c.Set(config.Username, "admin") - c.Set(config.Password, "admin") + c.username = "admin" + c.password = "admin" if err := CheckExternalAccessTripwire(c); err != nil { t.Errorf("unexpected error %v", err) } - c.Set(config.Username, "") - c.Set(config.Password, "") + c.username = "" + c.password = "" // HACK - this key isn't publically exposed - c.Set("dangerous_allow_public_without_auth", true) + c.dangerousAllowPublicWithoutAuth = true if err := CheckExternalAccessTripwire(c); err != nil { t.Errorf("unexpected error %v", err) } - c.Set("dangerous_allow_public_without_auth", false) + c.dangerousAllowPublicWithoutAuth = false if err := CheckExternalAccessTripwire(c); err == nil { t.Errorf("expected error %v", ExternalAccessError("4.4.4.4")) } - c.Set(config.SecurityTripwireAccessedFromPublicInternet, "") + c.securityTripwireAccessedFromPublicInternet = "" if err := CheckExternalAccessTripwire(c); err != nil { t.Errorf("unexpected error %v", err) diff --git a/pkg/session/config.go b/pkg/session/config.go new file mode 100644 index 000000000..0bd584c51 --- /dev/null +++ b/pkg/session/config.go @@ -0,0 +1,17 @@ +package session + +type ExternalAccessConfig interface { + HasCredentials() bool + GetDangerousAllowPublicWithoutAuth() bool + GetSecurityTripwireAccessedFromPublicInternet() string + IsNewSystem() bool +} + +type SessionConfig interface { + GetUsername() string + GetAPIKey() string + + GetSessionStoreKey() []byte + GetMaxSessionAge() int + ValidateCredentials(username string, password string) bool +} diff --git a/pkg/session/session.go b/pkg/session/session.go index 9d754e63a..7fc1b7f27 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -8,8 +8,7 @@ import ( "github.com/gorilla/securecookie" "github.com/gorilla/sessions" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/manager/config" - "github.com/stashapp/stash/pkg/utils" + "github.com/stashapp/stash/pkg/sliceutil/stringslice" ) type key int @@ -40,16 +39,16 @@ var ErrUnauthorized = errors.New("unauthorized") type Store struct { sessionStore *sessions.CookieStore - config *config.Instance + config SessionConfig } -func NewStore(c *config.Instance) *Store { +func NewStore(c SessionConfig) *Store { ret := &Store{ - sessionStore: sessions.NewCookieStore(config.GetInstance().GetSessionStoreKey()), + sessionStore: sessions.NewCookieStore(c.GetSessionStoreKey()), config: c, } - ret.sessionStore.MaxAge(config.GetInstance().GetMaxSessionAge()) + ret.sessionStore.MaxAge(c.GetMaxSessionAge()) return ret } @@ -62,7 +61,7 @@ func (s *Store) Login(w http.ResponseWriter, r *http.Request) error { password := r.FormValue(passwordFormKey) // authenticate the user - if !config.GetInstance().ValidateCredentials(username, password) { + if !s.config.ValidateCredentials(username, password) { return ErrInvalidCredentials } @@ -165,7 +164,7 @@ func GetVisitedPlugins(ctx context.Context) []string { func AddVisitedPlugin(ctx context.Context, pluginID string) context.Context { curVal := GetVisitedPlugins(ctx) - curVal = utils.StrAppendUnique(curVal, pluginID) + curVal = stringslice.StrAppendUnique(curVal, pluginID) return setVisitedPlugins(ctx, curVal) } diff --git a/pkg/utils/collections.go b/pkg/sliceutil/collections.go similarity index 98% rename from pkg/utils/collections.go rename to pkg/sliceutil/collections.go index 06bc9f1f5..5a271268c 100644 --- a/pkg/utils/collections.go +++ b/pkg/sliceutil/collections.go @@ -1,4 +1,4 @@ -package utils +package sliceutil import "reflect" diff --git a/pkg/utils/collections_test.go b/pkg/sliceutil/collections_test.go similarity index 98% rename from pkg/utils/collections_test.go rename to pkg/sliceutil/collections_test.go index 359b9ad10..cfd93c86f 100644 --- a/pkg/utils/collections_test.go +++ b/pkg/sliceutil/collections_test.go @@ -1,4 +1,4 @@ -package utils +package sliceutil import "testing" diff --git a/pkg/utils/int_collections.go b/pkg/sliceutil/intslice/int_collections.go similarity index 98% rename from pkg/utils/int_collections.go rename to pkg/sliceutil/intslice/int_collections.go index 9f2d3e6bc..daf5b3d11 100644 --- a/pkg/utils/int_collections.go +++ b/pkg/sliceutil/intslice/int_collections.go @@ -1,4 +1,4 @@ -package utils +package intslice import "strconv" diff --git a/pkg/utils/string_collections.go b/pkg/sliceutil/stringslice/string_collections.go similarity index 99% rename from pkg/utils/string_collections.go rename to pkg/sliceutil/stringslice/string_collections.go index 32943344a..f466d911b 100644 --- a/pkg/utils/string_collections.go +++ b/pkg/sliceutil/stringslice/string_collections.go @@ -1,4 +1,4 @@ -package utils +package stringslice import "strconv" diff --git a/pkg/sqlite/filter.go b/pkg/sqlite/filter.go index 9d3676542..e27690eb9 100644 --- a/pkg/sqlite/filter.go +++ b/pkg/sqlite/filter.go @@ -8,9 +8,9 @@ import ( "strings" "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/utils" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) type sqlClause struct { diff --git a/pkg/sqlite/movies_test.go b/pkg/sqlite/movies_test.go index 681c9fa09..75c6cc5bf 100644 --- a/pkg/sqlite/movies_test.go +++ b/pkg/sqlite/movies_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func TestMovieFindByName(t *testing.T) { @@ -222,7 +222,7 @@ func TestMovieUpdateMovieImages(t *testing.T) { const name = "TestMovieUpdateMovieImages" movie := models.Movie{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), } created, err := mqb.Create(movie) if err != nil { @@ -289,7 +289,7 @@ func TestMovieDestroyMovieImages(t *testing.T) { const name = "TestMovieDestroyMovieImages" movie := models.Movie{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), } created, err := mqb.Create(movie) if err != nil { diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index 9bd8e05f0..e8ca56db5 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func TestPerformerFindBySceneID(t *testing.T) { @@ -266,7 +266,7 @@ func TestPerformerUpdatePerformerImage(t *testing.T) { const name = "TestPerformerUpdatePerformerImage" performer := models.Performer{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Favorite: sql.NullBool{Bool: false, Valid: true}, } created, err := qb.Create(performer) @@ -307,7 +307,7 @@ func TestPerformerDestroyPerformerImage(t *testing.T) { const name = "TestPerformerDestroyPerformerImage" performer := models.Performer{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Favorite: sql.NullBool{Bool: false, Valid: true}, } created, err := qb.Create(performer) @@ -827,7 +827,7 @@ func TestPerformerStashIDs(t *testing.T) { const name = "TestStashIDs" performer := models.Performer{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Favorite: sql.NullBool{Bool: false, Valid: true}, } created, err := qb.Create(performer) diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index 0c45a2c0e..5541fdd95 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/utils" ) func TestSceneFind(t *testing.T) { @@ -761,7 +761,7 @@ func createScene(queryBuilder models.SceneReaderWriter, width int64, height int6 Int64: height, Valid: true, }, - Checksum: sql.NullString{String: utils.MD5FromString(name), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(name), Valid: true}, } return queryBuilder.Create(scene) @@ -1597,7 +1597,7 @@ func TestSceneUpdateSceneCover(t *testing.T) { const name = "TestSceneUpdateSceneCover" scene := models.Scene{ Path: name, - Checksum: sql.NullString{String: utils.MD5FromString(name), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(name), Valid: true}, } created, err := qb.Create(scene) if err != nil { @@ -1637,7 +1637,7 @@ func TestSceneDestroySceneCover(t *testing.T) { const name = "TestSceneDestroySceneCover" scene := models.Scene{ Path: name, - Checksum: sql.NullString{String: utils.MD5FromString(name), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(name), Valid: true}, } created, err := qb.Create(scene) if err != nil { @@ -1676,7 +1676,7 @@ func TestSceneStashIDs(t *testing.T) { const name = "TestSceneStashIDs" scene := models.Scene{ Path: name, - Checksum: sql.NullString{String: utils.MD5FromString(name), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(name), Valid: true}, } created, err := qb.Create(scene) if err != nil { diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index e1aaf4f9d..4a5377e0d 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -15,10 +15,11 @@ import ( "github.com/stashapp/stash/pkg/database" "github.com/stashapp/stash/pkg/gallery" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sqlite" - "github.com/stashapp/stash/pkg/utils" ) const ( @@ -755,7 +756,7 @@ func createMovies(mqb models.MovieReaderWriter, n int, o int) error { movie := models.Movie{ Name: sql.NullString{String: name, Valid: true}, URL: getMovieNullStringValue(index, urlField), - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), } created, err := mqb.Create(movie) @@ -976,7 +977,7 @@ func getStudioNullStringValue(index int, field string) sql.NullString { func createStudio(sqb models.StudioReaderWriter, name string, parentID *int64) (*models.Studio, error) { studio := models.Studio{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), } if parentID != nil { @@ -1014,7 +1015,7 @@ func createStudios(sqb models.StudioReaderWriter, n int, o int) error { name = getStudioStringValue(index, name) studio := models.Studio{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), URL: getStudioNullStringValue(index, urlField), } created, err := createStudioFromModel(sqb, studio) @@ -1129,7 +1130,7 @@ func linkPerformerTags(qb models.PerformerReaderWriter) error { return err } - tagIDs = utils.IntAppendUnique(tagIDs, tagID) + tagIDs = intslice.IntAppendUnique(tagIDs, tagID) return qb.UpdateTags(performerID, tagIDs) }) diff --git a/pkg/studio/export.go b/pkg/studio/export.go index 41f535494..30aa46482 100644 --- a/pkg/studio/export.go +++ b/pkg/studio/export.go @@ -3,8 +3,8 @@ package studio import ( "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/studio/export_test.go b/pkg/studio/export_test.go index c18fbfa30..d1aae1948 100644 --- a/pkg/studio/export_test.go +++ b/pkg/studio/export_test.go @@ -3,8 +3,8 @@ package studio import ( "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" diff --git a/pkg/studio/import.go b/pkg/studio/import.go index 8a9dfa644..3bd21dded 100644 --- a/pkg/studio/import.go +++ b/pkg/studio/import.go @@ -5,8 +5,9 @@ import ( "errors" "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) @@ -22,7 +23,7 @@ type Importer struct { } func (i *Importer) PreImport() error { - checksum := utils.MD5FromString(i.Input.Name) + checksum := md5.FromString(i.Input.Name) i.studio = models.Studio{ Checksum: checksum, @@ -40,7 +41,7 @@ func (i *Importer) PreImport() error { var err error if len(i.Input.Image) > 0 { - _, i.imageData, err = utils.ProcessBase64Image(i.Input.Image) + i.imageData, err = utils.ProcessBase64Image(i.Input.Image) if err != nil { return fmt.Errorf("invalid image: %v", err) } diff --git a/pkg/studio/import_test.go b/pkg/studio/import_test.go index 78c788fd0..098cd2099 100644 --- a/pkg/studio/import_test.go +++ b/pkg/studio/import_test.go @@ -4,10 +4,10 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" - "github.com/stashapp/stash/pkg/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -61,7 +61,7 @@ func TestImporterPreImport(t *testing.T) { assert.Nil(t, err) expectedStudio := createFullStudio(0, 0) expectedStudio.ParentID.Valid = false - expectedStudio.Checksum = utils.MD5FromString(studioName) + expectedStudio.Checksum = md5.FromString(studioName) assert.Equal(t, expectedStudio, i.studio) } diff --git a/pkg/tag/export.go b/pkg/tag/export.go index dde39760c..5a8c6fadc 100644 --- a/pkg/tag/export.go +++ b/pkg/tag/export.go @@ -3,8 +3,8 @@ package tag import ( "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) diff --git a/pkg/tag/export_test.go b/pkg/tag/export_test.go index e37008ab4..5323c33de 100644 --- a/pkg/tag/export_test.go +++ b/pkg/tag/export_test.go @@ -3,8 +3,8 @@ package tag import ( "errors" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" diff --git a/pkg/tag/import.go b/pkg/tag/import.go index 3fd793d0e..66d608425 100644 --- a/pkg/tag/import.go +++ b/pkg/tag/import.go @@ -3,8 +3,8 @@ package tag import ( "fmt" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/utils" ) @@ -38,7 +38,7 @@ func (i *Importer) PreImport() error { var err error if len(i.Input.Image) > 0 { - _, i.imageData, err = utils.ProcessBase64Image(i.Input.Image) + i.imageData, err = utils.ProcessBase64Image(i.Input.Image) if err != nil { return fmt.Errorf("invalid image: %v", err) } diff --git a/pkg/tag/import_test.go b/pkg/tag/import_test.go index c2f29d8e5..296db7e97 100644 --- a/pkg/tag/import_test.go +++ b/pkg/tag/import_test.go @@ -4,8 +4,8 @@ import ( "errors" "testing" - "github.com/stashapp/stash/pkg/manager/jsonschema" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/jsonschema" "github.com/stashapp/stash/pkg/models/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/pkg/utils/crypto.go b/pkg/utils/crypto.go deleted file mode 100644 index 6022aeac8..000000000 --- a/pkg/utils/crypto.go +++ /dev/null @@ -1,55 +0,0 @@ -package utils - -import ( - "crypto/md5" - "crypto/rand" - "fmt" - "hash/fnv" - "io" - "os" - - "github.com/stashapp/stash/pkg/logger" -) - -func MD5FromBytes(data []byte) string { - result := md5.Sum(data) - return fmt.Sprintf("%x", result) -} - -func MD5FromString(str string) string { - data := []byte(str) - return MD5FromBytes(data) -} - -func MD5FromFilePath(filePath string) (string, error) { - f, err := os.Open(filePath) - if err != nil { - return "", err - } - defer f.Close() - - return MD5FromReader(f) -} - -func MD5FromReader(src io.Reader) (string, error) { - h := md5.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - checksum := h.Sum(nil) - return fmt.Sprintf("%x", checksum), nil -} - -func GenerateRandomKey(l int) string { - b := make([]byte, l) - if n, err := rand.Read(b); err != nil { - logger.Warnf("failure generating random key: %v (only read %v bytes)", err, n) - } - return fmt.Sprintf("%x", b) -} - -func IntFromString(str string) uint64 { - h := fnv.New64a() - h.Write([]byte(str)) - return h.Sum64() -} diff --git a/pkg/utils/file.go b/pkg/utils/file.go deleted file mode 100644 index 474f7b8a2..000000000 --- a/pkg/utils/file.go +++ /dev/null @@ -1,392 +0,0 @@ -package utils - -import ( - "archive/zip" - "fmt" - "io" - "io/fs" - "net/http" - "os" - "os/user" - "path/filepath" - "regexp" - "strings" - - "github.com/h2non/filetype" - "github.com/h2non/filetype/types" - "github.com/stashapp/stash/pkg/logger" - "golang.org/x/text/collate" -) - -// FileType uses the filetype package to determine the given file path's type -func FileType(filePath string) (types.Type, error) { - file, _ := os.Open(filePath) - - // We only have to pass the file header = first 261 bytes - head := make([]byte, 261) - _, _ = file.Read(head) - - return filetype.Match(head) -} - -// FileExists returns true if the given path exists -func FileExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - return false, err -} - -// DirExists returns true if the given path exists and is a directory -func DirExists(path string) (bool, error) { - fileInfo, err := os.Stat(path) - if err != nil { - return false, fmt.Errorf("path doesn't exist <%s>", path) - } - if !fileInfo.IsDir() { - return false, fmt.Errorf("path is not a directory <%s>", path) - } - return true, nil -} - -// Touch creates an empty file at the given path if it doesn't already exist -func Touch(path string) error { - var _, err = os.Stat(path) - if os.IsNotExist(err) { - var file, err = os.Create(path) - if err != nil { - return err - } - defer file.Close() - } - return nil -} - -// EnsureDir will create a directory at the given path if it doesn't already exist -func EnsureDir(path string) error { - exists, err := FileExists(path) - if !exists { - err = os.Mkdir(path, 0755) - return err - } - return err -} - -// EnsureDirAll will create a directory at the given path along with any necessary parents if they don't already exist -func EnsureDirAll(path string) error { - return os.MkdirAll(path, 0755) -} - -// RemoveDir removes the given dir (if it exists) along with all of its contents -func RemoveDir(path string) error { - return os.RemoveAll(path) -} - -// EmptyDir will recursively remove the contents of a directory at the given path -func EmptyDir(path string) error { - d, err := os.Open(path) - if err != nil { - return err - } - defer d.Close() - - names, err := d.Readdirnames(-1) - if err != nil { - return err - } - - for _, name := range names { - err = os.RemoveAll(filepath.Join(path, name)) - if err != nil { - return err - } - } - - return nil -} - -type dirSorter []fs.DirEntry - -func (s dirSorter) Len() int { - return len(s) -} - -func (s dirSorter) Swap(i, j int) { - s[j], s[i] = s[i], s[j] -} - -func (s dirSorter) Bytes(i int) []byte { - return []byte(s[i].Name()) -} - -// ListDir will return the contents of a given directory path as a string slice -func ListDir(col *collate.Collator, path string) ([]string, error) { - var dirPaths []string - files, err := os.ReadDir(path) - if err != nil { - path = filepath.Dir(path) - files, err = os.ReadDir(path) - if err != nil { - return dirPaths, err - } - } - - if col != nil { - col.Sort(dirSorter(files)) - } - - for _, file := range files { - if !file.IsDir() { - continue - } - dirPaths = append(dirPaths, filepath.Join(path, file.Name())) - } - return dirPaths, nil -} - -// GetHomeDirectory returns the path of the user's home directory. ~ on Unix and C:\Users\UserName on Windows -func GetHomeDirectory() string { - currentUser, err := user.Current() - if err != nil { - panic(err) - } - return currentUser.HomeDir -} - -func SafeMove(src, dst string) error { - err := os.Rename(src, dst) - - if err != nil { - logger.Errorf("[Util] unable to rename: \"%s\" due to %s. Falling back to copying.", src, err.Error()) - - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, in) - if err != nil { - return err - } - - err = out.Close() - if err != nil { - return err - } - - err = os.Remove(src) - if err != nil { - return err - } - } - - return nil -} - -// IsZipFileUnmcompressed returns true if zip file in path is using 0 compression level -func IsZipFileUncompressed(path string) (bool, error) { - r, err := zip.OpenReader(path) - if err != nil { - fmt.Printf("Error reading zip file %s: %s\n", path, err) - return false, err - } else { - defer r.Close() - for _, f := range r.File { - if f.FileInfo().IsDir() { // skip dirs, they always get store level compression - continue - } - return f.Method == 0, nil // check compression level of first actual file - } - } - return false, nil -} - -// WriteFile writes file to path creating parent directories if needed -func WriteFile(path string, file []byte) error { - pathErr := EnsureDirAll(filepath.Dir(path)) - if pathErr != nil { - return fmt.Errorf("cannot ensure path %s", pathErr) - } - - err := os.WriteFile(path, file, 0755) - if err != nil { - return fmt.Errorf("write error for thumbnail %s: %s ", path, err) - } - return nil -} - -// GetIntraDir returns a string that can be added to filepath.Join to implement directory depth, "" on error -// eg given a pattern of 0af63ce3c99162e9df23a997f62621c5 and a depth of 2 length of 3 -// returns 0af/63c or 0af\63c ( dependin on os) that can be later used like this filepath.Join(directory, intradir, basename) -func GetIntraDir(pattern string, depth, length int) string { - if depth < 1 || length < 1 || (depth*length > len(pattern)) { - return "" - } - intraDir := pattern[0:length] // depth 1 , get length number of characters from pattern - for i := 1; i < depth; i++ { // for every extra depth: move to the right of the pattern length positions, get length number of chars - intraDir = filepath.Join(intraDir, pattern[length*i:length*(i+1)]) // adding each time to intradir the extra characters with a filepath join - } - return intraDir -} - -func GetDir(path string) string { - if path == "" { - path = GetHomeDirectory() - } - - return path -} - -func GetParent(path string) *string { - isRoot := path[len(path)-1:] == "/" - if isRoot { - return nil - } else { - parentPath := filepath.Clean(path + "/..") - return &parentPath - } -} - -// ServeFileNoCache serves the provided file, ensuring that the response -// contains headers to prevent caching. -func ServeFileNoCache(w http.ResponseWriter, r *http.Request, filepath string) { - w.Header().Add("Cache-Control", "no-cache") - - http.ServeFile(w, r, filepath) -} - -// MatchEntries returns a string slice of the entries in directory dir which -// match the regexp pattern. On error an empty slice is returned -// MatchEntries isn't recursive, only the specific 'dir' is searched -// without being expanded. -func MatchEntries(dir, pattern string) ([]string, error) { - var res []string - var err error - - re, err := regexp.Compile(pattern) - if err != nil { - return nil, err - } - - f, err := os.Open(dir) - if err != nil { - return nil, err - } - defer f.Close() - - files, err := f.Readdirnames(-1) - if err != nil { - return nil, err - } - - for _, file := range files { - if re.Match([]byte(file)) { - res = append(res, filepath.Join(dir, file)) - } - } - return res, err -} - -// IsPathInDir returns true if pathToCheck is within dir. -func IsPathInDir(dir, pathToCheck string) bool { - rel, err := filepath.Rel(dir, pathToCheck) - - if err == nil { - if !strings.HasPrefix(rel, "..") { - return true - } - } - - return false -} - -// GetNameFromPath returns the name of a file from its path -// if stripExtension is true the extension is omitted from the name -func GetNameFromPath(path string, stripExtension bool) string { - fn := filepath.Base(path) - if stripExtension { - ext := filepath.Ext(fn) - fn = strings.TrimSuffix(fn, ext) - } - return fn -} - -// GetFunscriptPath returns the path of a file -// with the extension changed to .funscript -func GetFunscriptPath(path string) string { - ext := filepath.Ext(path) - fn := strings.TrimSuffix(path, ext) - return fn + ".funscript" -} - -// IsFsPathCaseSensitive checks the fs of the given path to see if it is case sensitive -// if the case sensitivity can not be determined false and an error != nil are returned -func IsFsPathCaseSensitive(path string) (bool, error) { - // The case sensitivity of the fs of "path" is determined by case flipping - // the first letter rune from the base string of the path - // If the resulting flipped path exists then the fs should not be case sensitive - // ( we check the file mod time to avoid matching an existing path ) - - fi, err := os.Stat(path) - if err != nil { // path cannot be stat'd - return false, err - } - - base := filepath.Base(path) - fBase, err := FlipCaseSingle(base) - if err != nil { // cannot be case flipped - return false, err - } - i := strings.LastIndex(path, base) - if i < 0 { // shouldn't happen - return false, fmt.Errorf("could not case flip path %s", path) - } - - flipped := []byte(path) - for _, c := range []byte(fBase) { // replace base of path with the flipped one ( we need to flip the base or last dir part ) - flipped[i] = c - i++ - } - - fiCase, err := os.Stat(string(flipped)) - if err != nil { // cannot stat the case flipped path - return true, nil // fs of path should be case sensitive - } - - if fiCase.ModTime() == fi.ModTime() { // file path exists and is the same - return false, nil // fs of path is not case sensitive - } - return false, fmt.Errorf("can not determine case sensitivity of path %s", path) -} - -func FindInPaths(paths []string, baseName string) string { - for _, p := range paths { - filePath := filepath.Join(p, baseName) - if exists, _ := FileExists(filePath); exists { - return filePath - } - } - - return "" -} - -// MatchExtension returns true if the extension of the provided path -// matches any of the provided extensions. -func MatchExtension(path string, extensions []string) bool { - ext := filepath.Ext(path) - for _, e := range extensions { - if strings.EqualFold(ext, "."+e) { - return true - } - } - - return false -} diff --git a/pkg/utils/float.go b/pkg/utils/float.go deleted file mode 100644 index 97022c7a7..000000000 --- a/pkg/utils/float.go +++ /dev/null @@ -1,8 +0,0 @@ -package utils - -import "math" - -// IsValidFloat64 ensures the given value is a valid number (not NaN) which is not equal to 0 -func IsValidFloat64(value float64) bool { - return !math.IsNaN(value) && value != 0 -} diff --git a/pkg/utils/image.go b/pkg/utils/image.go index 59435160f..061a01460 100644 --- a/pkg/utils/image.go +++ b/pkg/utils/image.go @@ -24,7 +24,7 @@ const base64RE = `^data:.+\/(.+);base64,(.*)$` func ProcessImageInput(ctx context.Context, imageInput string) ([]byte, error) { regex := regexp.MustCompile(base64RE) if regex.MatchString(imageInput) { - _, d, err := ProcessBase64Image(imageInput) + d, err := ProcessBase64Image(imageInput) return d, err } @@ -53,7 +53,7 @@ func ReadImageFromURL(ctx context.Context, url string) ([]byte, error) { if req.URL.Scheme != "" { req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/") } - req.Header.Set("User-Agent", GetUserAgent()) + req.Header.Set("User-Agent", getUserAgent()) resp, err := client.Do(req) @@ -75,11 +75,11 @@ func ReadImageFromURL(ctx context.Context, url string) ([]byte, error) { return body, nil } -// ProcessBase64Image transforms a base64 encoded string from a form post and returns the MD5 hash of the data and the -// image itself as a byte slice. -func ProcessBase64Image(imageString string) (string, []byte, error) { +// ProcessBase64Image transforms a base64 encoded string from a form post and +// returns the image itself as a byte slice. +func ProcessBase64Image(imageString string) ([]byte, error) { if imageString == "" { - return "", nil, fmt.Errorf("empty image string") + return nil, fmt.Errorf("empty image string") } regex := regexp.MustCompile(base64RE) @@ -92,10 +92,10 @@ func ProcessBase64Image(imageString string) (string, []byte, error) { } imageData, err := GetDataFromBase64String(encodedString) if err != nil { - return "", nil, err + return nil, err } - return MD5FromBytes(imageData), imageData, nil + return imageData, nil } // GetDataFromBase64String returns the given base64 encoded string as a byte slice diff --git a/pkg/utils/strings.go b/pkg/utils/strings.go index 912690069..6df220d40 100644 --- a/pkg/utils/strings.go +++ b/pkg/utils/strings.go @@ -2,41 +2,9 @@ package utils import ( "fmt" - "math/rand" "strings" - "time" - "unicode" ) -var characters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") - -func RandomSequence(n int) string { - b := make([]rune, n) - rand.Seed(time.Now().UnixNano()) - for i := range b { - b[i] = characters[rand.Intn(len(characters))] - } - return string(b) -} - -// FlipCaseSingle flips the case ( lower<->upper ) of a single char from the string s -// If the string cannot be flipped, the original string value and an error are returned -func FlipCaseSingle(s string) (string, error) { - rr := []rune(s) - for i, r := range rr { - if unicode.IsLetter(r) { // look for a letter to flip - if unicode.IsUpper(r) { - rr[i] = unicode.ToLower(r) - return string(rr), nil - } - rr[i] = unicode.ToUpper(r) - return string(rr), nil - } - - } - return s, fmt.Errorf("could not case flip string %s", s) -} - type StrFormatMap map[string]interface{} // StrFormat formats the provided format string, replacing placeholders diff --git a/pkg/utils/user_agent.go b/pkg/utils/user_agent.go index 533b6b6ab..a9bba6e8c 100644 --- a/pkg/utils/user_agent.go +++ b/pkg/utils/user_agent.go @@ -9,8 +9,8 @@ const FirefoxLinux = "Mozilla/5.0 (X11; Linux x86_64; rv:88.0) Gecko/20100101 Fi const FirefoxLinuxArm = "Mozilla/5.0 (X11; Linux armv7l; rv:86.0) Gecko/20100101 Firefox/86.0" const FirefoxLinuxArm64 = "Mozilla/5.0 (X11; Linux aarch64; rv:86.0) Gecko/20100101 Firefox/86.0" -// GetUserAgent returns a valid User Agent string that matches the running os/arch -func GetUserAgent() string { +// getUserAgent returns a valid User Agent string that matches the running os/arch +func getUserAgent() string { arch := runtime.GOARCH os := runtime.GOOS diff --git a/pkg/utils/windows.go b/pkg/utils/windows.go deleted file mode 100644 index cf63e1ae4..000000000 --- a/pkg/utils/windows.go +++ /dev/null @@ -1,14 +0,0 @@ -package utils - -import ( - "runtime" - "strings" -) - -// FixWindowsPath replaces \ with / in the given path because sometimes the \ isn't recognized as valid on windows -func FixWindowsPath(str string) string { - if runtime.GOOS == "windows" { - return strings.ReplaceAll(str, "\\", "/") - } - return str -} diff --git a/scripts/test_db_generator/makeTestDB.go b/scripts/test_db_generator/makeTestDB.go index e3aa59033..db095845b 100644 --- a/scripts/test_db_generator/makeTestDB.go +++ b/scripts/test_db_generator/makeTestDB.go @@ -1,3 +1,4 @@ +//go:build ignore // +build ignore package main @@ -14,10 +15,11 @@ import ( "time" "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sqlite" - "github.com/stashapp/stash/pkg/utils" "gopkg.in/yaml.v2" ) @@ -149,7 +151,7 @@ func makeStudios(n int) { name := names[c.Naming.Tags].generateName(rand.Intn(5) + 1) studio := models.Studio{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), } if rand.Intn(100) > 5 { @@ -182,7 +184,7 @@ func makePerformers(n int) { name := generatePerformerName() performer := models.Performer{ Name: sql.NullString{String: name, Valid: true}, - Checksum: utils.MD5FromString(name), + Checksum: md5.FromString(name), Favorite: sql.NullBool{ Bool: false, Valid: true, @@ -256,14 +258,14 @@ func getDate() string { } func generateScene(i int) models.Scene { - path := utils.MD5FromString("scene/" + strconv.Itoa(i)) + path := md5.FromString("scene/" + strconv.Itoa(i)) w, h := getResolution() return models.Scene{ Path: path, Title: sql.NullString{String: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1), Valid: true}, - Checksum: sql.NullString{String: utils.MD5FromString(path), Valid: true}, - OSHash: sql.NullString{String: utils.MD5FromString(path), Valid: true}, + Checksum: sql.NullString{String: md5.FromString(path), Valid: true}, + OSHash: sql.NullString{String: md5.FromString(path), Valid: true}, Duration: sql.NullFloat64{ Float64: rand.Float64() * 14400, Valid: true, @@ -305,14 +307,14 @@ func makeImages(n int) { } func generateImage(i int) models.Image { - path := utils.MD5FromString("image/" + strconv.Itoa(i)) + path := md5.FromString("image/" + strconv.Itoa(i)) w, h := getResolution() return models.Image{ Title: sql.NullString{String: names[c.Naming.Images].generateName(rand.Intn(7) + 1), Valid: true}, Path: path, - Checksum: utils.MD5FromString(path), + Checksum: md5.FromString(path), Height: models.NullInt64(h), Width: models.NullInt64(w), } @@ -347,12 +349,12 @@ func makeGalleries(n int) { } func generateGallery(i int) models.Gallery { - path := utils.MD5FromString("gallery/" + strconv.Itoa(i)) + path := md5.FromString("gallery/" + strconv.Itoa(i)) return models.Gallery{ Title: sql.NullString{String: names[c.Naming.Galleries].generateName(rand.Intn(7) + 1), Valid: true}, Path: sql.NullString{String: path, Valid: true}, - Checksum: utils.MD5FromString(path), + Checksum: md5.FromString(path), Date: models.SQLiteDate{ String: getDate(), Valid: true, @@ -378,7 +380,7 @@ func makeMarkers(n int) { tags := getRandomTags(r, 0, 5) // remove primary tag - tags = utils.IntExclude(tags, []int{marker.PrimaryTagID}) + tags = intslice.IntExclude(tags, []int{marker.PrimaryTagID}) if err := r.SceneMarker().UpdateTags(created.ID, tags); err != nil { return err } @@ -504,12 +506,12 @@ func getRandomPerformers(r models.Repository) []int { // } // for _, pp := range p { - // ret = utils.IntAppendUnique(ret, pp.ID) + // ret = intslice.IntAppendUnique(ret, pp.ID) // } // } for i := 0; i < n; i++ { - ret = utils.IntAppendUnique(ret, rand.Intn(c.Performers)+1) + ret = intslice.IntAppendUnique(ret, rand.Intn(c.Performers)+1) } return ret @@ -535,12 +537,12 @@ func getRandomTags(r models.Repository, min, max int) []int { // } // for _, tt := range t { - // ret = utils.IntAppendUnique(ret, tt.ID) + // ret = intslice.IntAppendUnique(ret, tt.ID) // } // } for i := 0; i < n; i++ { - ret = utils.IntAppendUnique(ret, rand.Intn(c.Tags)+1) + ret = intslice.IntAppendUnique(ret, rand.Intn(c.Tags)+1) } return ret @@ -557,12 +559,12 @@ func getRandomImages(r models.Repository) []int { // } // for _, tt := range t { - // ret = utils.IntAppendUnique(ret, tt.ID) + // ret = intslice.IntAppendUnique(ret, tt.ID) // } // } for i := 0; i < n; i++ { - ret = utils.IntAppendUnique(ret, rand.Intn(c.Images)+1) + ret = intslice.IntAppendUnique(ret, rand.Intn(c.Images)+1) } return ret diff --git a/ui/ui.go b/ui/ui.go new file mode 100644 index 000000000..87d9e7f0e --- /dev/null +++ b/ui/ui.go @@ -0,0 +1,9 @@ +package ui + +import "embed" + +//go:embed v2.5/build +var UIBox embed.FS + +//go:embed login +var LoginUIBox embed.FS