From bd45daacf3088be02d7b99d2572ac9c92ad7b66f Mon Sep 17 00:00:00 2001 From: bnkai <48220860+bnkai@users.noreply.github.com> Date: Mon, 11 May 2020 10:20:08 +0300 Subject: [PATCH] Add a cache for gallery thumbnails (#496) --- go.mod | 1 - go.sum | 2 - graphql/documents/data/config.graphql | 1 + graphql/schema/types/config.graphql | 6 +- graphql/schema/types/metadata.graphql | 4 +- pkg/api/cache_thumbs.go | 72 +++++++ pkg/api/resolver_mutation_configure.go | 7 + pkg/api/resolver_mutation_metadata.go | 2 +- pkg/api/resolver_query_configuration.go | 1 + pkg/api/routes_gallery.go | 8 +- pkg/api/server.go | 1 + pkg/manager/manager_tasks.go | 38 +++- pkg/manager/paths/paths_gallery.go | 19 ++ pkg/manager/task_clean.go | 14 +- pkg/manager/task_generate_gallery_thumbs.go | 37 ++++ pkg/manager/task_scan.go | 5 + pkg/models/model_gallery.go | 10 + pkg/models/querybuilder_scene.go | 4 +- pkg/utils/file.go | 78 ++++++- .../Settings/SettingsConfigurationPanel.tsx | 17 ++ .../SettingsTasksPanel/GenerateButton.tsx | 10 +- .../github.com/dustin/go-humanize/.travis.yml | 21 -- vendor/github.com/dustin/go-humanize/LICENSE | 21 -- .../dustin/go-humanize/README.markdown | 124 ----------- vendor/github.com/dustin/go-humanize/big.go | 31 --- .../github.com/dustin/go-humanize/bigbytes.go | 173 ---------------- vendor/github.com/dustin/go-humanize/bytes.go | 143 ------------- vendor/github.com/dustin/go-humanize/comma.go | 116 ----------- .../github.com/dustin/go-humanize/commaf.go | 40 ---- vendor/github.com/dustin/go-humanize/ftoa.go | 46 ----- .../github.com/dustin/go-humanize/humanize.go | 8 - .../github.com/dustin/go-humanize/number.go | 192 ------------------ .../github.com/dustin/go-humanize/ordinals.go | 25 --- vendor/github.com/dustin/go-humanize/si.go | 123 ----------- vendor/github.com/dustin/go-humanize/times.go | 117 ----------- vendor/modules.txt | 2 - 36 files changed, 319 insertions(+), 1200 deletions(-) create mode 100644 pkg/api/cache_thumbs.go create mode 100644 pkg/manager/task_generate_gallery_thumbs.go delete mode 100644 vendor/github.com/dustin/go-humanize/.travis.yml delete mode 100644 vendor/github.com/dustin/go-humanize/LICENSE delete mode 100644 vendor/github.com/dustin/go-humanize/README.markdown delete mode 100644 vendor/github.com/dustin/go-humanize/big.go delete mode 100644 vendor/github.com/dustin/go-humanize/bigbytes.go delete mode 100644 vendor/github.com/dustin/go-humanize/bytes.go delete mode 100644 vendor/github.com/dustin/go-humanize/comma.go delete mode 100644 vendor/github.com/dustin/go-humanize/commaf.go delete mode 100644 vendor/github.com/dustin/go-humanize/ftoa.go delete mode 100644 vendor/github.com/dustin/go-humanize/humanize.go delete mode 100644 vendor/github.com/dustin/go-humanize/number.go delete mode 100644 vendor/github.com/dustin/go-humanize/ordinals.go delete mode 100644 vendor/github.com/dustin/go-humanize/si.go delete mode 100644 vendor/github.com/dustin/go-humanize/times.go diff --git a/go.mod b/go.mod index d92f27d20..265cc12f9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/antchfx/htmlquery v1.2.3 github.com/bmatcuk/doublestar v1.1.5 github.com/disintegration/imaging v1.6.0 - github.com/dustin/go-humanize v1.0.0 github.com/go-chi/chi v4.0.2+incompatible github.com/gobuffalo/packr/v2 v2.0.2 github.com/golang-migrate/migrate/v4 v4.3.1 diff --git a/go.sum b/go.sum index a256cc38f..e8d3ea59e 100644 --- a/go.sum +++ b/go.sum @@ -325,10 +325,8 @@ github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA= github.com/golang-migrate/migrate/v4 v4.3.1 h1:3eR1NY+pplX+m6yJ1fQf5dFWX3fBgUtZfDiaS/kJVu4= github.com/golang-migrate/migrate/v4 v4.3.1/go.mod h1:mJ89KBgbXmM3P49BqOxRL3riNF/ATlg5kMhm17GA0dE= -github.com/golang-migrate/migrate/v4 v4.10.0 h1:76R6UL3BGnDTpYeittMtfpaNvGBH5zMZatO/fCzIjWo= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 2bc1929a0..48a9a437c 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -2,6 +2,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { stashes databasePath generatedPath + cachePath maxTranscodeSize maxStreamingTranscodeSize forceMkv diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index ff759a74a..b98d04aa1 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -14,6 +14,8 @@ input ConfigGeneralInput { databasePath: String """Path to generated files""" generatedPath: String + """Path to cache""" + cachePath: String """Max generated transcode size""" maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" @@ -49,7 +51,9 @@ type ConfigGeneralResult { databasePath: String! """Path to generated files""" generatedPath: String! - """Max generated transcode size""" + """Path to cache""" + cachePath: String! + """Max generated transcode size""" maxTranscodeSize: StreamingResolutionEnum """Max streaming transcode size""" maxStreamingTranscodeSize: StreamingResolutionEnum diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index a603b56b5..c069631ce 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -3,6 +3,8 @@ input GenerateMetadataInput { previews: Boolean! markers: Boolean! transcodes: Boolean! + """gallery thumbnails for cache usage""" + thumbnails: Boolean! } input ScanMetadataInput { @@ -22,4 +24,4 @@ type MetadataUpdateStatus { progress: Float! status: String! message: String! -} \ No newline at end of file +} diff --git a/pkg/api/cache_thumbs.go b/pkg/api/cache_thumbs.go new file mode 100644 index 000000000..0bcbd616c --- /dev/null +++ b/pkg/api/cache_thumbs.go @@ -0,0 +1,72 @@ +package api + +import ( + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" + "io/ioutil" +) + +type thumbBuffer struct { + path string + dir string + data []byte +} + +func newCacheThumb(dir string, path string, data []byte) *thumbBuffer { + t := thumbBuffer{dir: dir, path: path, data: data} + return &t +} + +var writeChan chan *thumbBuffer +var touchChan chan *string + +func startThumbCache() { // TODO add extra wait, close chan code if/when stash gets a stop mode + + writeChan = make(chan *thumbBuffer, 20) + go thumbnailCacheWriter() +} + +//serialize file writes to avoid race conditions +func thumbnailCacheWriter() { + + for thumb := range writeChan { + exists, _ := utils.FileExists(thumb.path) + if !exists { + err := utils.WriteFile(thumb.path, thumb.data) + if err != nil { + logger.Errorf("Write error for thumbnail %s: %s ", thumb.path, err) + } + } + } + +} + +// get thumbnail from cache, otherwise create it and store to cache +func cacheGthumb(gallery *models.Gallery, index int, width int) []byte { + thumbPath := paths.GetGthumbPath(gallery.Checksum, index, width) + exists, _ := utils.FileExists(thumbPath) + if exists { // if thumbnail exists in cache return that + content, err := ioutil.ReadFile(thumbPath) + if err == nil { + return content + } else { + logger.Errorf("Read Error for file %s : %s", thumbPath, err) + } + + } + data := gallery.GetThumbnail(index, width) + thumbDir := paths.GetGthumbDir(gallery.Checksum) + t := newCacheThumb(thumbDir, thumbPath, data) + writeChan <- t // write the file to cache + return data +} + +// create all thumbs for a given gallery +func CreateGthumbs(gallery *models.Gallery) { + count := gallery.ImageCount() + for i := 0; i < count; i++ { + cacheGthumb(gallery, i, models.DefaultGthumbWidth) + } +} diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index 6b5d1b86f..3e75d245e 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -38,6 +38,13 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co config.Set(config.Generated, input.GeneratedPath) } + if input.CachePath != nil { + if err := utils.EnsureDir(*input.CachePath); err != nil { + return makeConfigGeneralResult(), err + } + config.Set(config.Cache, input.CachePath) + } + if input.MaxTranscodeSize != nil { config.Set(config.MaxTranscodeSize, input.MaxTranscodeSize.String()) } diff --git a/pkg/api/resolver_mutation_metadata.go b/pkg/api/resolver_mutation_metadata.go index aeec1ef96..25ee96287 100644 --- a/pkg/api/resolver_mutation_metadata.go +++ b/pkg/api/resolver_mutation_metadata.go @@ -23,7 +23,7 @@ func (r *mutationResolver) MetadataExport(ctx context.Context) (string, error) { } func (r *mutationResolver) MetadataGenerate(ctx context.Context, input models.GenerateMetadataInput) (string, error) { - manager.GetInstance().Generate(input.Sprites, input.Previews, input.Markers, input.Transcodes) + manager.GetInstance().Generate(input.Sprites, input.Previews, input.Markers, input.Transcodes, input.Thumbnails) return "todo", nil } diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 25708a5ca..89913a978 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -45,6 +45,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { Stashes: config.GetStashPaths(), DatabasePath: config.GetDatabasePath(), GeneratedPath: config.GetGeneratedPath(), + CachePath: config.GetCachePath(), MaxTranscodeSize: &maxTranscodeSize, MaxStreamingTranscodeSize: &maxStreamingTranscodeSize, ForceMkv: config.GetForceMKV(), diff --git a/pkg/api/routes_gallery.go b/pkg/api/routes_gallery.go index 0c53cf35c..4b5a7f4cd 100644 --- a/pkg/api/routes_gallery.go +++ b/pkg/api/routes_gallery.go @@ -23,11 +23,15 @@ func (rs galleryRoutes) Routes() chi.Router { func (rs galleryRoutes) File(w http.ResponseWriter, r *http.Request) { gallery := r.Context().Value(galleryKey).(*models.Gallery) + if gallery == nil { + http.Error(w, http.StatusText(404), 404) + return + } fileIndex, _ := strconv.Atoi(chi.URLParam(r, "fileIndex")) thumb := r.URL.Query().Get("thumb") w.Header().Add("Cache-Control", "max-age=604800000") // 1 Week if thumb == "true" { - _, _ = w.Write(gallery.GetThumbnail(fileIndex, 200)) + _, _ = w.Write(cacheGthumb(gallery, fileIndex, models.DefaultGthumbWidth)) } else if thumb == "" { _, _ = w.Write(gallery.GetImage(fileIndex)) } else { @@ -36,7 +40,7 @@ func (rs galleryRoutes) File(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(400), 400) return } - _, _ = w.Write(gallery.GetThumbnail(fileIndex, int(width))) + _, _ = w.Write(cacheGthumb(gallery, fileIndex, int(width))) } } diff --git a/pkg/api/server.go b/pkg/api/server.go index 88740eabc..8e122de33 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -245,6 +245,7 @@ func Start() { http.Redirect(w, r, "/", 301) }) + startThumbCache() // Serve the web app r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) { ext := path.Ext(r.URL.Path) diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index ec7214091..fe041c8bf 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -171,7 +171,7 @@ func (s *singleton) Export() { }() } -func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcodes bool) { +func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcodes bool, thumbnails bool) { if s.Status.Status != Idle { return } @@ -179,6 +179,7 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod s.Status.indefiniteProgress() qb := models.NewSceneQueryBuilder() + qg := models.NewGalleryQueryBuilder() //this.job.total = await ObjectionUtils.getCount(Scene); instance.Paths.Generated.EnsureTmpDir() @@ -186,6 +187,8 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod defer s.returnToIdleState() scenes, err := qb.All() + var galleries []*models.Gallery + if err != nil { logger.Errorf("failed to get scenes for generate") return @@ -194,7 +197,16 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod delta := utils.Btoi(sprites) + utils.Btoi(previews) + utils.Btoi(markers) + utils.Btoi(transcodes) var wg sync.WaitGroup s.Status.Progress = 0 - total := len(scenes) + lenScenes := len(scenes) + total := lenScenes + if thumbnails { + galleries, err = qg.All() + if err != nil { + logger.Errorf("failed to get galleries for generate") + return + } + total += len(galleries) + } if s.Status.stopping { logger.Info("Stopping due to user request") @@ -248,6 +260,28 @@ func (s *singleton) Generate(sprites bool, previews bool, markers bool, transcod wg.Wait() } + + if thumbnails { + logger.Infof("Generating thumbnails for the galleries") + for i, gallery := range galleries { + s.Status.setProgress(lenScenes+i, total) + if s.Status.stopping { + logger.Info("Stopping due to user request") + return + } + + if gallery == nil { + logger.Errorf("nil gallery, skipping generate") + continue + } + + wg.Add(1) + task := GenerateGthumbsTask{Gallery: *gallery} + go task.Start(&wg) + wg.Wait() + } + } + logger.Infof("Generate finished") }() } diff --git a/pkg/manager/paths/paths_gallery.go b/pkg/manager/paths/paths_gallery.go index 4db60e311..9e4665c95 100644 --- a/pkg/manager/paths/paths_gallery.go +++ b/pkg/manager/paths/paths_gallery.go @@ -1,12 +1,18 @@ package paths import ( + "fmt" "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/utils" "path/filepath" ) type galleryPaths struct{} +const thumbDir = "gthumbs" +const thumbDirDepth int = 2 +const thumbDirLength int = 2 // thumbDirDepth * thumbDirLength must be smaller than the length of checksum + func newGalleryPaths() *galleryPaths { return &galleryPaths{} } @@ -15,6 +21,19 @@ func (gp *galleryPaths) GetExtractedPath(checksum string) string { return filepath.Join(config.GetCachePath(), checksum) } +func GetGthumbCache() string { + return filepath.Join(config.GetCachePath(), thumbDir) +} + +func GetGthumbDir(checksum string) string { + return filepath.Join(config.GetCachePath(), thumbDir, utils.GetIntraDir(checksum, thumbDirDepth, thumbDirLength), checksum) +} + +func GetGthumbPath(checksum string, index int, width int) string { + fname := fmt.Sprintf("%s_%d_%d.jpg", checksum, index, width) + return filepath.Join(config.GetCachePath(), thumbDir, utils.GetIntraDir(checksum, thumbDirDepth, thumbDirLength), checksum, fname) +} + func (gp *galleryPaths) GetExtractedFilePath(checksum string, fileName string) string { return filepath.Join(config.GetCachePath(), checksum, fileName) } diff --git a/pkg/manager/task_clean.go b/pkg/manager/task_clean.go index 8b05c0e77..8cca80bbd 100644 --- a/pkg/manager/task_clean.go +++ b/pkg/manager/task_clean.go @@ -10,6 +10,7 @@ import ( "github.com/stashapp/stash/pkg/database" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" + "github.com/stashapp/stash/pkg/manager/paths" "github.com/stashapp/stash/pkg/models" ) @@ -54,13 +55,13 @@ func (t *CleanTask) deleteScene(sceneID int) { err = DestroyScene(sceneID, tx) if err != nil { - logger.Infof("Error deleting scene from database: %s", err.Error()) + logger.Errorf("Error deleting scene from database: %s", err.Error()) tx.Rollback() return } if err := tx.Commit(); err != nil { - logger.Infof("Error deleting scene from database: %s", err.Error()) + logger.Errorf("Error deleting scene from database: %s", err.Error()) return } @@ -75,15 +76,20 @@ func (t *CleanTask) deleteGallery(galleryID int) { err := qb.Destroy(galleryID, tx) if err != nil { - logger.Infof("Error deleting gallery from database: %s", err.Error()) + logger.Errorf("Error deleting gallery from database: %s", err.Error()) tx.Rollback() return } if err := tx.Commit(); err != nil { - logger.Infof("Error deleting gallery from database: %s", err.Error()) + logger.Errorf("Error deleting gallery from database: %s", err.Error()) return } + + pathErr := os.RemoveAll(paths.GetGthumbDir(t.Gallery.Checksum)) // remove cache dir of gallery + if pathErr != nil { + logger.Errorf("Error deleting gallery directory from cache: %s", pathErr) + } } func (t *CleanTask) fileExists(filename string) bool { diff --git a/pkg/manager/task_generate_gallery_thumbs.go b/pkg/manager/task_generate_gallery_thumbs.go new file mode 100644 index 000000000..2079e980d --- /dev/null +++ b/pkg/manager/task_generate_gallery_thumbs.go @@ -0,0 +1,37 @@ +package manager + +import ( + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/manager/paths" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/utils" + "sync" +) + +type GenerateGthumbsTask struct { + Gallery models.Gallery +} + +func (t *GenerateGthumbsTask) Start(wg *sync.WaitGroup) { + defer wg.Done() + generated := 0 + count := t.Gallery.ImageCount() + for i := 0; i < count; i++ { + thumbPath := paths.GetGthumbPath(t.Gallery.Checksum, i, models.DefaultGthumbWidth) + exists, _ := utils.FileExists(thumbPath) + if exists { + continue + } + data := t.Gallery.GetThumbnail(i, models.DefaultGthumbWidth) + err := utils.WriteFile(thumbPath, data) + if err != nil { + logger.Errorf("error writing gallery thumbnail: %s", err) + } else { + generated++ + } + + } + if generated > 0 { + logger.Infof("Generated %d thumbnails for %s", generated, t.Gallery.Path) + } +} diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 01e327b65..3e784ae56 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -34,11 +34,16 @@ func (t *ScanTask) Start(wg *sync.WaitGroup) { func (t *ScanTask) scanGallery() { qb := models.NewGalleryQueryBuilder() gallery, _ := qb.FindByPath(t.FilePath) + if gallery != nil { // We already have this item in the database, keep going return } + ok, err := utils.IsZipFileUncompressed(t.FilePath) + if err == nil && !ok { + logger.Warnf("%s is using above store (0) level compression.", t.FilePath) + } checksum, err := t.calculateChecksum() if err != nil { logger.Error(err.Error()) diff --git a/pkg/models/model_gallery.go b/pkg/models/model_gallery.go index da45a88d4..9f8a83f5d 100644 --- a/pkg/models/model_gallery.go +++ b/pkg/models/model_gallery.go @@ -26,6 +26,8 @@ type Gallery struct { UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` } +const DefaultGthumbWidth int = 200 + func (g *Gallery) GetFiles(baseURL string) []*GalleryFilesType { var galleryFiles []*GalleryFilesType filteredFiles, readCloser, err := g.listZipContents() @@ -152,3 +154,11 @@ func reorder(a []*zip.File, toFirst int) []*zip.File { } return a } + +func (g *Gallery) ImageCount() int { + images, _, _ := g.listZipContents() + if images == nil { + return 0 + } + return len(images) +} diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index 2c2aa8a39..e9da2b0da 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" - "github.com/dustin/go-humanize" "github.com/jmoiron/sqlx" "github.com/stashapp/stash/pkg/database" + "github.com/stashapp/stash/pkg/utils" ) const sceneTable = "scenes" @@ -202,7 +202,7 @@ func (qb *SceneQueryBuilder) SizeCount() (string, error) { if err != nil { return "0", err } - return humanize.Bytes(sum), err + return utils.HumanizeBytes(sum), err } func (qb *SceneQueryBuilder) CountByStudioID(studioID int) (int, error) { diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 753958292..1e2ad1aed 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -1,10 +1,12 @@ package utils import ( + "archive/zip" "fmt" "github.com/h2non/filetype" "github.com/h2non/filetype/types" "io/ioutil" + "math" "os" "os/user" "path/filepath" @@ -66,7 +68,12 @@ func EnsureDir(path string) error { return err } -// RemoveDir removes the given file path along with all of its contents +// EnsureDirAll will create a directory at the given path along with any necessary parents if they don't already exist +func EnsureDirAll(path string) error { + return os.MkdirAll(path, 0755) +} + +// RemoveDir removes the given dir (if it exists) along with all of its contents func RemoveDir(path string) error { return os.RemoveAll(path) } @@ -125,6 +132,75 @@ func GetHomeDirectory() string { return currentUser.HomeDir } +// IsZipFileUnmcompressed returns true if zip file in path is using 0 compression level +func IsZipFileUncompressed(path string) (bool, error) { + r, err := zip.OpenReader(path) + if err != nil { + fmt.Printf("Error reading zip file %s: %s\n", path, err) + return false, err + } else { + defer r.Close() + for _, f := range r.File { + if f.FileInfo().IsDir() { // skip dirs, they always get store level compression + continue + } + return f.Method == 0, nil // check compression level of first actual file + } + } + return false, nil +} + +// humanize code taken from https://github.com/dustin/go-humanize and adjusted + +func logn(n, b float64) float64 { + return math.Log(n) / math.Log(b) +} + +// HumanizeBytes returns a human readable bytes string of a uint +func HumanizeBytes(s uint64) string { + sizes := []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"} + if s < 10 { + return fmt.Sprintf("%d B", s) + } + e := math.Floor(logn(float64(s), 1024)) + suffix := sizes[int(e)] + val := math.Floor(float64(s)/math.Pow(1024, e)*10+0.5) / 10 + f := "%.0f %s" + if val < 10 { + f = "%.1f %s" + } + + return fmt.Sprintf(f, val, suffix) +} + +// WriteFile writes file to path creating parent directories if needed +func WriteFile(path string, file []byte) error { + pathErr := EnsureDirAll(filepath.Dir(path)) + if pathErr != nil { + return fmt.Errorf("Cannot ensure path %s", pathErr) + } + + err := ioutil.WriteFile(path, file, 0755) + if err != nil { + return fmt.Errorf("Write error for thumbnail %s: %s ", path, err) + } + return nil +} + +// GetIntraDir returns a string that can be added to filepath.Join to implement directory depth, "" on error +//eg given a pattern of 0af63ce3c99162e9df23a997f62621c5 and a depth of 2 length of 3 +//returns 0af/63c or 0af\63c ( dependin on os) that can be later used like this filepath.Join(directory, intradir, basename) +func GetIntraDir(pattern string, depth, length int) string { + if depth < 1 || length < 1 || (depth*length > len(pattern)) { + return "" + } + intraDir := pattern[0:length] // depth 1 , get length number of characters from pattern + for i := 1; i < depth; i++ { // for every extra depth: move to the right of the pattern length positions, get length number of chars + intraDir = filepath.Join(intraDir, pattern[length*i:length*(i+1)]) // adding each time to intradir the extra characters with a filepath join + } + return intraDir +} + func GetDir(path string) string { if path == "" { path = GetHomeDirectory() diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index e2b50fa6b..daede0ded 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -16,6 +16,7 @@ export const SettingsConfigurationPanel: React.FC = () => { const [generatedPath, setGeneratedPath] = useState( undefined ); + const [cachePath, setCachePath] = useState(undefined); const [maxTranscodeSize, setMaxTranscodeSize] = useState< GQL.StreamingResolutionEnum | undefined >(undefined); @@ -42,6 +43,7 @@ export const SettingsConfigurationPanel: React.FC = () => { stashes, databasePath, generatedPath, + cachePath, maxTranscodeSize, maxStreamingTranscodeSize, forceMkv, @@ -65,6 +67,7 @@ export const SettingsConfigurationPanel: React.FC = () => { setStashes(conf.general.stashes ?? []); setDatabasePath(conf.general.databasePath); setGeneratedPath(conf.general.generatedPath); + setCachePath(conf.general.cachePath); setMaxTranscodeSize(conf.general.maxTranscodeSize ?? undefined); setMaxStreamingTranscodeSize( conf.general.maxStreamingTranscodeSize ?? undefined @@ -213,6 +216,20 @@ export const SettingsConfigurationPanel: React.FC = () => { + +
Cache Path
+ ) => + setCachePath(e.currentTarget.value) + } + /> + + Directory location of the cache + + +
Excluded Patterns
diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx index abd72f9ea..391e1fde0 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/GenerateButton.tsx @@ -8,7 +8,8 @@ export const GenerateButton: React.FC = () => { const [sprites, setSprites] = useState(true); const [previews, setPreviews] = useState(true); const [markers, setMarkers] = useState(true); - const [transcodes, setTranscodes] = useState(true); + const [transcodes, setTranscodes] = useState(false); + const [thumbnails, setThumbnails] = useState(false); async function onGenerate() { try { @@ -17,6 +18,7 @@ export const GenerateButton: React.FC = () => { previews, markers, transcodes, + thumbnails, }); Toast.success({ content: "Started generating" }); } catch (e) { @@ -51,6 +53,12 @@ export const GenerateButton: React.FC = () => { label="Transcodes (MP4 conversions of unsupported video formats)" onChange={() => setTranscodes(!transcodes)} /> + setThumbnails(!thumbnails)} + />