From 655d3ae9698bb190fb7838bb14ed4077aaa03b45 Mon Sep 17 00:00:00 2001 From: SmallCoccinelle <89733524+SmallCoccinelle@users.noreply.github.com> Date: Thu, 14 Oct 2021 06:32:41 +0200 Subject: [PATCH] Toward better context handling (#1835) * Use the request context The code uses context.Background() in a flow where there is a http.Request. Use the requests context instead. * Use a true context in the plugin example Let AddTag/RemoveTag take a context and use that context throughout the example. * Avoid the use of context.Background Prefer context.TODO over context.Background deep in the call chain. This marks the site as something which we need to context-handle later, and also makes it clear to the reader that the context is sort-of temporary in the code base. While here, be consistent in handling the `act` variable in each branch of the if .. { .. } .. check. * Prefer context.TODO over context.Background For the different scraping operations here, there is a context higher up the call chain, which we ought to use. Mark the call-sites as TODO for now, so we can come back later on a sweep of which parts can be context-lifted. * Thread context upwards Initialization requires context for transactions. Thread the context upward the call chain. At the intialization call, add a context.TODO since we can't break this yet. The singleton assumption prevents us from pulling it up into main for now. * make tasks context-aware Change the task interface to understand contexts. Pass the context down in some of the branches where it is needed. * Make QueryStashBoxScene context-aware This call naturally sits inside the request-context. Use it. * Introduce a context in the JS plugin code This allows us to use a context for HTTP calls inside the system. Mark the context with a TODO at top level for now. * Nitpick error formatting Use %v rather than %s for error interfaces. Do not begin an error strong with a capital letter. * Avoid the use of http.Get in FFMPEG download chain Since http.Get has no context, it isn't possible to break out or have policy induced. The call will block until the GET completes. Rewrite to use a http Request and provide a context. Thread the context through the call chain for now. provide context.TODO() at the top level of the initialization chain. * Make getRemoteCDPWSAddress aware of contexts Eliminate a call to http.Get and replace it with a context-aware variant. Push the context upwards in the call chain, but plug it before the scraper interface so we don't have to rewrite said interface yet. Plugged with context.TODO() * Scraper: make the getImage function context-aware Use a context, and pass it upwards. Plug it with context.TODO() up the chain before the rewrite gets too much out of hand for now. Minor tweaks along the way, remove a call to context.Background() deep in the call chain. * Make NOTIFY request context-aware The call sits inside a Request-handler. So it's natural to use the requests context as the context for the outgoing HTTP request. * Use a context in the url scraper code We are sitting in code which has a context, so utilize it for the request as well. * Use a context when checking versions When we check the version of stash on Github, use a context. Thread the context up to the initialization routine of the HTTP/GraphQL server and plug it with a context.TODO() for now. This paves the way for providing a context to the HTTP server code in a future patch. * Make utils func ReadImage context-aware In almost all of the cases, there is a context in the call chain which is a natural use. This is true for all the GraphQL mutations. The exception is in task_stash_box_tag, so plug that task with context.TODO() for now. * Make stash-box get context-aware Thread a context through the call chain until we hit the Client API. Plug it with context.TODO() there for now. * Enable the noctx linter The code is now free of any uncontexted HTTP request. This means we pass the noctx linter, and we can enable it in the code base. --- .golangci.yml | 2 +- pkg/api/check_version.go | 23 ++++++++-------- pkg/api/resolver.go | 2 +- pkg/api/resolver_mutation_configure.go | 4 +-- pkg/api/resolver_mutation_movie.go | 10 +++---- pkg/api/resolver_mutation_performer.go | 4 +-- pkg/api/resolver_mutation_scene.go | 8 +++--- pkg/api/resolver_mutation_studio.go | 7 +++-- pkg/api/resolver_mutation_tag.go | 4 +-- pkg/api/resolver_query_scraper.go | 4 +-- pkg/api/server.go | 2 +- pkg/database/database.go | 4 +-- pkg/dlna/dms.go | 10 +++---- pkg/ffmpeg/downloader.go | 14 +++++++--- pkg/manager/checksum.go | 4 +-- pkg/manager/manager.go | 20 ++++++++------ pkg/manager/manager_tasks.go | 4 +-- pkg/manager/post_migrate.go | 6 ++-- pkg/manager/task.go | 4 ++- pkg/manager/task_import.go | 4 +-- pkg/manager/task_scan.go | 12 ++++---- pkg/manager/task_stash_box_tag.go | 8 +++--- pkg/plugin/examples/common/graphql.go | 24 ++++++++-------- pkg/plugin/js.go | 3 +- pkg/plugin/js/gql.go | 9 +++--- pkg/scraper/action.go | 4 ++- pkg/scraper/image.go | 29 ++++++++++---------- pkg/scraper/json.go | 29 ++++++++++---------- pkg/scraper/scrapers.go | 28 +++++++++---------- pkg/scraper/stash.go | 16 +++++------ pkg/scraper/stashbox/stash_box.go | 38 ++++++++++++++------------ pkg/scraper/url.go | 23 ++++++++++------ pkg/scraper/xpath.go | 19 +++++++------ pkg/utils/image.go | 9 +++--- 34 files changed, 211 insertions(+), 180 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 85f9247a3..2900aa4df 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -31,7 +31,7 @@ linters: # - ifshort - misspell # - nakedret - # - noctx + - noctx # - paralleltest - revive - rowserrcheck diff --git a/pkg/api/check_version.go b/pkg/api/check_version.go index 44e29c52a..5586b724d 100644 --- a/pkg/api/check_version.go +++ b/pkg/api/check_version.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/json" "errors" "fmt" @@ -107,12 +108,12 @@ type githubTagResponse struct { Node_id string } -func makeGithubRequest(url string, output interface{}) error { +func makeGithubRequest(ctx context.Context, url string, output interface{}) error { client := &http.Client{ Timeout: 3 * time.Second, } - req, _ := http.NewRequest("GET", url, nil) + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) req.Header.Add("Accept", apiAcceptHeader) // gh api recommendation , send header with api version response, err := client.Do(req) @@ -147,7 +148,7 @@ func makeGithubRequest(url string, output interface{}) error { // If running a build from the "master" branch, then the latest full release // is used, otherwise it uses the release that is tagged with "latest_develop" // which is the latest pre-release build. -func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease string, err error) { +func GetLatestVersion(ctx context.Context, shortHash bool) (latestVersion string, latestRelease string, err error) { arch := runtime.GOARCH // https://en.wikipedia.org/wiki/Comparison_of_ARM_cores isARMv7 := cpu.ARM.HasNEON || cpu.ARM.HasVFPv3 || cpu.ARM.HasVFPv3D16 || cpu.ARM.HasVFPv4 // armv6 doesn't support any of these features @@ -180,14 +181,14 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin } release := githubReleasesResponse{} - err = makeGithubRequest(url, &release) + err = makeGithubRequest(ctx, url, &release) if err != nil { return "", "", err } if release.Prerelease == usePreRelease { - latestVersion = getReleaseHash(release, shortHash, usePreRelease) + latestVersion = getReleaseHash(ctx, release, shortHash, usePreRelease) if wantedRelease != "" { for _, asset := range release.Assets { @@ -205,12 +206,12 @@ func GetLatestVersion(shortHash bool) (latestVersion string, latestRelease strin return latestVersion, latestRelease, nil } -func getReleaseHash(release githubReleasesResponse, shortHash bool, usePreRelease bool) string { +func getReleaseHash(ctx context.Context, release githubReleasesResponse, shortHash bool, usePreRelease bool) string { shaLength := len(release.Target_commitish) // the /latest API call doesn't return the hash in target_commitish // also add sanity check in case Target_commitish is not 40 characters if !usePreRelease || shaLength != 40 { - return getShaFromTags(shortHash, release.Tag_name) + return getShaFromTags(ctx, shortHash, release.Tag_name) } if shortHash { @@ -225,9 +226,9 @@ func getReleaseHash(release githubReleasesResponse, shortHash bool, usePreReleas return release.Target_commitish } -func printLatestVersion() { +func printLatestVersion(ctx context.Context) { _, githash, _ = GetVersion() - latest, _, err := GetLatestVersion(true) + latest, _, err := GetLatestVersion(ctx, true) if err != nil { logger.Errorf("Couldn't find latest version: %s", err) } else { @@ -241,10 +242,10 @@ func printLatestVersion() { // get sha from the github api tags endpoint // returns the sha1 hash/shorthash or "" if something's wrong -func getShaFromTags(shortHash bool, name string) string { +func getShaFromTags(ctx context.Context, shortHash bool, name string) string { url := apiTags tags := []githubTagResponse{} - err := makeGithubRequest(url, &tags) + err := makeGithubRequest(ctx, url, &tags) if err != nil { logger.Errorf("Github Tags Api %v", err) diff --git a/pkg/api/resolver.go b/pkg/api/resolver.go index 7d3f6aa3f..85986cce1 100644 --- a/pkg/api/resolver.go +++ b/pkg/api/resolver.go @@ -160,7 +160,7 @@ func (r *queryResolver) Version(ctx context.Context) (*models.Version, error) { //Gets latest version (git shorthash commit for now) func (r *queryResolver) Latestversion(ctx context.Context) (*models.ShortVersion, error) { - ver, url, err := GetLatestVersion(true) + ver, url, err := GetLatestVersion(ctx, true) if err == nil { logger.Infof("Retrieved latest hash: %s", ver) } else { diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index e864cd616..aa9ecf710 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -14,12 +14,12 @@ import ( ) func (r *mutationResolver) Setup(ctx context.Context, input models.SetupInput) (bool, error) { - err := manager.GetInstance().Setup(input) + err := manager.GetInstance().Setup(ctx, input) return err == nil, err } func (r *mutationResolver) Migrate(ctx context.Context, input models.MigrateInput) (bool, error) { - err := manager.GetInstance().Migrate(input) + err := manager.GetInstance().Migrate(ctx, input) return err == nil, err } diff --git a/pkg/api/resolver_mutation_movie.go b/pkg/api/resolver_mutation_movie.go index e1c63974f..88769f6d3 100644 --- a/pkg/api/resolver_mutation_movie.go +++ b/pkg/api/resolver_mutation_movie.go @@ -38,7 +38,7 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input models.MovieCr // Process the base 64 encoded image string if input.FrontImage != nil { - frontimageData, err = utils.ProcessImageInput(*input.FrontImage) + frontimageData, err = utils.ProcessImageInput(ctx, *input.FrontImage) if err != nil { return nil, err } @@ -46,7 +46,7 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input models.MovieCr // Process the base 64 encoded image string if input.BackImage != nil { - backimageData, err = utils.ProcessImageInput(*input.BackImage) + backimageData, err = utils.ProcessImageInput(ctx, *input.BackImage) if err != nil { return nil, err } @@ -139,7 +139,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp var frontimageData []byte frontImageIncluded := translator.hasField("front_image") if input.FrontImage != nil { - frontimageData, err = utils.ProcessImageInput(*input.FrontImage) + frontimageData, err = utils.ProcessImageInput(ctx, *input.FrontImage) if err != nil { return nil, err } @@ -147,7 +147,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp backImageIncluded := translator.hasField("back_image") var backimageData []byte if input.BackImage != nil { - backimageData, err = utils.ProcessImageInput(*input.BackImage) + backimageData, err = utils.ProcessImageInput(ctx, *input.BackImage) if err != nil { return nil, err } @@ -202,7 +202,7 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input models.MovieUp // HACK - if front image is null and back image is not null, then set the front image // to the default image since we can't have a null front image and a non-null back image if frontimageData == nil && backimageData != nil { - frontimageData, _ = utils.ProcessImageInput(models.DefaultMovieImage) + frontimageData, _ = utils.ProcessImageInput(ctx, models.DefaultMovieImage) } if err := qb.UpdateImages(movie.ID, frontimageData, backimageData); err != nil { diff --git a/pkg/api/resolver_mutation_performer.go b/pkg/api/resolver_mutation_performer.go index 66c068aa5..90e33b78b 100644 --- a/pkg/api/resolver_mutation_performer.go +++ b/pkg/api/resolver_mutation_performer.go @@ -32,7 +32,7 @@ func (r *mutationResolver) PerformerCreate(ctx context.Context, input models.Per var err error if input.Image != nil { - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) } if err != nil { @@ -178,7 +178,7 @@ func (r *mutationResolver) PerformerUpdate(ctx context.Context, input models.Per var err error imageIncluded := translator.hasField("image") if input.Image != nil { - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err } diff --git a/pkg/api/resolver_mutation_scene.go b/pkg/api/resolver_mutation_scene.go index 090599665..82339df3c 100644 --- a/pkg/api/resolver_mutation_scene.go +++ b/pkg/api/resolver_mutation_scene.go @@ -32,7 +32,7 @@ func (r *mutationResolver) SceneUpdate(ctx context.Context, input models.SceneUp // Start the transaction and save the scene if err := r.withTxn(ctx, func(repo models.Repository) error { - ret, err = r.sceneUpdate(input, translator, repo) + ret, err = r.sceneUpdate(ctx, input, translator, repo) return err }); err != nil { return nil, err @@ -52,7 +52,7 @@ func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.Sce inputMap: inputMaps[i], } - thisScene, err := r.sceneUpdate(*scene, translator, repo) + thisScene, err := r.sceneUpdate(ctx, *scene, translator, repo) ret = append(ret, thisScene) if err != nil { @@ -85,7 +85,7 @@ func (r *mutationResolver) ScenesUpdate(ctx context.Context, input []*models.Sce return newRet, nil } -func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator changesetTranslator, repo models.Repository) (*models.Scene, error) { +func (r *mutationResolver) sceneUpdate(ctx context.Context, input models.SceneUpdateInput, translator changesetTranslator, repo models.Repository) (*models.Scene, error) { // Populate scene from the input sceneID, err := strconv.Atoi(input.ID) if err != nil { @@ -110,7 +110,7 @@ func (r *mutationResolver) sceneUpdate(input models.SceneUpdateInput, translator if input.CoverImage != nil && *input.CoverImage != "" { var err error - coverImageData, err = utils.ProcessImageInput(*input.CoverImage) + coverImageData, err = utils.ProcessImageInput(ctx, *input.CoverImage) if err != nil { return nil, err } diff --git a/pkg/api/resolver_mutation_studio.go b/pkg/api/resolver_mutation_studio.go index bdca6059f..5353c7594 100644 --- a/pkg/api/resolver_mutation_studio.go +++ b/pkg/api/resolver_mutation_studio.go @@ -3,10 +3,11 @@ package api import ( "context" "database/sql" - "github.com/stashapp/stash/pkg/studio" "strconv" "time" + "github.com/stashapp/stash/pkg/studio" + "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" @@ -33,7 +34,7 @@ func (r *mutationResolver) StudioCreate(ctx context.Context, input models.Studio // Process the base 64 encoded image string if input.Image != nil { - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err } @@ -129,7 +130,7 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input models.Studio imageIncluded := translator.hasField("image") if input.Image != nil { var err error - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err } diff --git a/pkg/api/resolver_mutation_tag.go b/pkg/api/resolver_mutation_tag.go index d1dc230e4..f65cf1fe5 100644 --- a/pkg/api/resolver_mutation_tag.go +++ b/pkg/api/resolver_mutation_tag.go @@ -36,7 +36,7 @@ func (r *mutationResolver) TagCreate(ctx context.Context, input models.TagCreate var err error if input.Image != nil { - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err @@ -121,7 +121,7 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input models.TagUpdate imageIncluded := translator.hasField("image") if input.Image != nil { - imageData, err = utils.ProcessImageInput(*input.Image) + imageData, err = utils.ProcessImageInput(ctx, *input.Image) if err != nil { return nil, err diff --git a/pkg/api/resolver_query_scraper.go b/pkg/api/resolver_query_scraper.go index 181363e24..0af2dc540 100644 --- a/pkg/api/resolver_query_scraper.go +++ b/pkg/api/resolver_query_scraper.go @@ -123,7 +123,7 @@ func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.Sta } if input.Q != nil { - return client.QueryStashBoxScene(*input.Q) + return client.QueryStashBoxScene(ctx, *input.Q) } return nil, nil @@ -197,7 +197,7 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr if input.SceneID != nil { return client.FindStashBoxScenesByFingerprintsFlat([]string{*input.SceneID}) } else if input.Query != nil { - return client.QueryStashBoxScene(*input.Query) + return client.QueryStashBoxScene(ctx, *input.Query) } return nil, errors.New("scene_id or query must be set") diff --git a/pkg/api/server.go b/pkg/api/server.go index ef181a389..24d8f4494 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -240,7 +240,7 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) { go func() { printVersion() - printLatestVersion() + printLatestVersion(context.TODO()) logger.Infof("stash is listening on " + address) if tlsConfig != nil { diff --git a/pkg/database/database.go b/pkg/database/database.go index 190db6c9b..a23a77aa8 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -165,7 +165,7 @@ func Backup(db *sqlx.DB, backupPath string) error { var err error db, err = sqlx.Connect(sqlite3Driver, "file:"+dbPath+"?_fk=true") if err != nil { - return fmt.Errorf("Open database %s failed:%s", dbPath, err) + return fmt.Errorf("open database %s failed: %v", dbPath, err) } defer db.Close() } @@ -173,7 +173,7 @@ func Backup(db *sqlx.DB, backupPath string) error { logger.Infof("Backing up database into: %s", backupPath) _, err := db.Exec(`VACUUM INTO "` + backupPath + `"`) if err != nil { - return fmt.Errorf("vacuum failed: %s", err) + return fmt.Errorf("vacuum failed: %v", err) } return nil diff --git a/pkg/dlna/dms.go b/pkg/dlna/dms.go index d7c5efa85..5773ad4f6 100644 --- a/pkg/dlna/dms.go +++ b/pkg/dlna/dms.go @@ -415,7 +415,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) { } var scene *models.Scene - err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { + err := me.txnManager.WithReadTxn(r.Context(), func(r models.ReaderRepository) error { idInt, err := strconv.Atoi(sceneId) if err != nil { return nil @@ -434,7 +434,7 @@ func (me *Server) serveIcon(w http.ResponseWriter, r *http.Request) { me.sceneServer.ServeScreenshot(scene, w, r) } -func (me *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) { +func (me *Server) contentDirectoryInitialEvent(ctx context.Context, urls []*url.URL, sid string) { body := xmlMarshalOrPanic(upnp.PropertySet{ Properties: []upnp.Property{ { @@ -465,7 +465,7 @@ func (me *Server) contentDirectoryInitialEvent(urls []*url.URL, sid string) { body = append([]byte(``+"\n"), body...) for _, _url := range urls { bodyReader := bytes.NewReader(body) - req, err := http.NewRequest("NOTIFY", _url.String(), bodyReader) + req, err := http.NewRequestWithContext(ctx, "NOTIFY", _url.String(), bodyReader) if err != nil { logger.Errorf("Could not create a request to notify %s: %s", _url.String(), err) continue @@ -526,7 +526,7 @@ func (me *Server) contentDirectoryEventSubHandler(w http.ResponseWriter, r *http w.WriteHeader(http.StatusOK) go func() { time.Sleep(100 * time.Millisecond) - me.contentDirectoryInitialEvent(urls, sid) + me.contentDirectoryInitialEvent(r.Context(), urls, sid) }() } else if r.Method == "SUBSCRIBE" { http.Error(w, "meh", http.StatusPreconditionFailed) @@ -554,7 +554,7 @@ func (me *Server) initMux(mux *http.ServeMux) { mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) { sceneId := r.URL.Query().Get("scene") var scene *models.Scene - err := me.txnManager.WithReadTxn(context.Background(), func(r models.ReaderRepository) error { + err := me.txnManager.WithReadTxn(r.Context(), func(r models.ReaderRepository) error { sceneIdInt, err := strconv.Atoi(sceneId) if err != nil { return nil diff --git a/pkg/ffmpeg/downloader.go b/pkg/ffmpeg/downloader.go index da5b2dbaf..f9c4c7242 100644 --- a/pkg/ffmpeg/downloader.go +++ b/pkg/ffmpeg/downloader.go @@ -2,6 +2,7 @@ package ffmpeg import ( "archive/zip" + "context" "fmt" "io" "net/http" @@ -36,9 +37,9 @@ func GetPaths(paths []string) (string, string) { return ffmpegPath, ffprobePath } -func Download(configDirectory string) error { +func Download(ctx context.Context, configDirectory string) error { for _, url := range getFFMPEGURL() { - err := DownloadSingle(configDirectory, url) + err := DownloadSingle(ctx, configDirectory, url) if err != nil { return err } @@ -69,7 +70,7 @@ func (r *progressReader) Read(p []byte) (int, error) { return read, err } -func DownloadSingle(configDirectory, url string) error { +func DownloadSingle(ctx context.Context, configDirectory, url string) error { if url == "" { return fmt.Errorf("no ffmpeg url for this platform") } @@ -88,7 +89,12 @@ func DownloadSingle(configDirectory, url string) error { logger.Infof("Downloading %s...", url) // Make the HTTP request - resp, err := http.Get(url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return err } diff --git a/pkg/manager/checksum.go b/pkg/manager/checksum.go index bc41ddfe1..5a36bca43 100644 --- a/pkg/manager/checksum.go +++ b/pkg/manager/checksum.go @@ -9,12 +9,12 @@ import ( "github.com/stashapp/stash/pkg/models" ) -func setInitialMD5Config(txnManager models.TransactionManager) { +func setInitialMD5Config(ctx context.Context, txnManager models.TransactionManager) { // if there are no scene files in the database, then default the // VideoFileNamingAlgorithm config setting to oshash and calculateMD5 to // false, otherwise set them to true for backwards compatibility purposes var count int - if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + if err := txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { var err error count, err = r.Scene().Count() return err diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 3ee27d445..a65eb7809 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -1,6 +1,7 @@ package manager import ( + "context" "errors" "fmt" "os" @@ -58,6 +59,7 @@ func GetInstance() *singleton { func Initialize() *singleton { once.Do(func() { + ctx := context.TODO() cfg, err := config.Initialize() if err != nil { @@ -93,7 +95,7 @@ func Initialize() *singleton { if err != nil { panic(fmt.Sprintf("error initializing configuration: %s", err.Error())) } else { - if err := instance.PostInit(); err != nil { + if err := instance.PostInit(ctx); err != nil { panic(err) } } @@ -152,6 +154,8 @@ func initProfiling(cpuProfilePath string) { } func initFFMPEG() error { + ctx := context.TODO() + // only do this if we have a config file set if instance.Config.GetConfigFile() != "" { // use same directory as config path @@ -164,7 +168,7 @@ func initFFMPEG() error { if ffmpegPath == "" || ffprobePath == "" { logger.Infof("couldn't find FFMPEG, attempting to download it") - if err := ffmpeg.Download(configDirectory); err != nil { + if err := ffmpeg.Download(ctx, configDirectory); err != nil { msg := `Unable to locate / automatically download FFMPEG Check the readme for download links. @@ -195,7 +199,7 @@ func initLog() { // PostInit initialises the paths, caches and txnManager after the initial // configuration has been set. Should only be called if the configuration // is valid. -func (s *singleton) PostInit() error { +func (s *singleton) PostInit(ctx context.Context) error { if err := s.Config.SetInitialConfig(); err != nil { logger.Warnf("could not set initial configuration: %v", err) } @@ -235,7 +239,7 @@ func (s *singleton) PostInit() error { } if database.Ready() == nil { - s.PostMigrate() + s.PostMigrate(ctx) } return nil @@ -295,7 +299,7 @@ func setSetupDefaults(input *models.SetupInput) { } } -func (s *singleton) Setup(input models.SetupInput) error { +func (s *singleton) Setup(ctx context.Context, input models.SetupInput) error { setSetupDefaults(&input) // create the config directory if it does not exist @@ -328,7 +332,7 @@ func (s *singleton) Setup(input models.SetupInput) error { } // initialise the database - if err := s.PostInit(); err != nil { + if err := s.PostInit(ctx); err != nil { return fmt.Errorf("error initializing the database: %v", err) } @@ -349,7 +353,7 @@ func (s *singleton) validateFFMPEG() error { return nil } -func (s *singleton) Migrate(input models.MigrateInput) error { +func (s *singleton) Migrate(ctx context.Context, input models.MigrateInput) error { // always backup so that we can roll back to the previous version if // migration fails backupPath := input.BackupPath @@ -377,7 +381,7 @@ func (s *singleton) Migrate(input models.MigrateInput) error { } // perform post-migration operations - s.PostMigrate() + s.PostMigrate(ctx) // if no backup path was provided, then delete the created backup if input.BackupPath == "" { diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 4bc69dde6..4f6bedf63 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -90,7 +90,7 @@ func (s *singleton) Import(ctx context.Context) (int, error) { MissingRefBehaviour: models.ImportMissingRefEnumFail, fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), } - task.Start() + task.Start(ctx) }) return s.JobManager.Add(ctx, "Importing...", j), nil @@ -122,7 +122,7 @@ func (s *singleton) RunSingleTask(ctx context.Context, t Task) int { wg.Add(1) j := job.MakeJobExec(func(ctx context.Context, progress *job.Progress) { - t.Start() + t.Start(ctx) wg.Done() }) diff --git a/pkg/manager/post_migrate.go b/pkg/manager/post_migrate.go index 400ae39a0..eaf397708 100644 --- a/pkg/manager/post_migrate.go +++ b/pkg/manager/post_migrate.go @@ -1,6 +1,8 @@ package manager +import "context" + // PostMigrate is executed after migrations have been executed. -func (s *singleton) PostMigrate() { - setInitialMD5Config(s.TxnManager) +func (s *singleton) PostMigrate(ctx context.Context) { + setInitialMD5Config(ctx, s.TxnManager) } diff --git a/pkg/manager/task.go b/pkg/manager/task.go index 9906948c8..ec2676563 100644 --- a/pkg/manager/task.go +++ b/pkg/manager/task.go @@ -1,6 +1,8 @@ package manager +import "context" + type Task interface { - Start() + Start(context.Context) GetDescription() string } diff --git a/pkg/manager/task_import.go b/pkg/manager/task_import.go index 4437043f4..70b494d7c 100644 --- a/pkg/manager/task_import.go +++ b/pkg/manager/task_import.go @@ -79,7 +79,7 @@ func (t *ImportTask) GetDescription() string { return "Importing..." } -func (t *ImportTask) Start() { +func (t *ImportTask) Start(ctx context.Context) { if t.TmpZip != "" { defer func() { err := utils.RemoveDir(t.BaseDir) @@ -126,8 +126,6 @@ func (t *ImportTask) Start() { } } - ctx := context.TODO() - t.ImportTags(ctx) t.ImportPerformers(ctx) t.ImportStudios(ctx) diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index 76bf5a646..3ab4b7844 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -120,7 +120,7 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { } go func() { - task.Start() + task.Start(ctx) wg.Done() progress.Increment() }() @@ -238,12 +238,12 @@ type ScanTask struct { CaseSensitiveFs bool } -func (t *ScanTask) Start() { +func (t *ScanTask) Start(ctx context.Context) { var s *models.Scene t.progress.ExecuteTask("Scanning "+t.FilePath, func() { if isGallery(t.FilePath) { - t.scanGallery() + t.scanGallery(ctx) } else if isVideo(t.FilePath) { s = t.scanScene() } else if isImage(t.FilePath) { @@ -318,12 +318,12 @@ func (t *ScanTask) Start() { } } -func (t *ScanTask) scanGallery() { +func (t *ScanTask) scanGallery(ctx context.Context) { var g *models.Gallery images := 0 scanImages := false - if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { var err error g, err = r.Gallery().FindByPath(t.FilePath) @@ -976,7 +976,7 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) { subTask.zipGallery = zipGallery // run the subtask and wait for it to complete - subTask.Start() + subTask.Start(context.TODO()) return nil }) if err != nil { diff --git a/pkg/manager/task_stash_box_tag.go b/pkg/manager/task_stash_box_tag.go index 7a7e7b8f7..6da960381 100644 --- a/pkg/manager/task_stash_box_tag.go +++ b/pkg/manager/task_stash_box_tag.go @@ -22,7 +22,7 @@ type StashBoxPerformerTagTask struct { } func (t *StashBoxPerformerTagTask) Start() { - t.stashBoxPerformerTag() + t.stashBoxPerformerTag(context.TODO()) } func (t *StashBoxPerformerTagTask) Description() string { @@ -36,7 +36,7 @@ func (t *StashBoxPerformerTagTask) Description() string { return fmt.Sprintf("Tagging performer %s from stash-box", name) } -func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { +func (t *StashBoxPerformerTagTask) stashBoxPerformerTag(ctx context.Context) { var performer *models.ScrapedPerformer var err error @@ -169,7 +169,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } if len(performer.Images) > 0 && !excluded["image"] { - image, err := utils.ReadImageFromURL(performer.Images[0]) + image, err := utils.ReadImageFromURL(ctx, performer.Images[0]) if err != nil { return err } @@ -232,7 +232,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } if len(performer.Images) > 0 { - image, imageErr := utils.ReadImageFromURL(performer.Images[0]) + image, imageErr := utils.ReadImageFromURL(ctx, performer.Images[0]) if imageErr != nil { return imageErr } diff --git a/pkg/plugin/examples/common/graphql.go b/pkg/plugin/examples/common/graphql.go index 212792717..299a7d2a4 100644 --- a/pkg/plugin/examples/common/graphql.go +++ b/pkg/plugin/examples/common/graphql.go @@ -66,7 +66,7 @@ type SceneUpdateInput struct { TagIds []graphql.ID `graphql:"tag_ids" json:"tag_ids"` } -func getTagID(client *graphql.Client, create bool) (*graphql.ID, error) { +func getTagID(ctx context.Context, client *graphql.Client, create bool) (*graphql.ID, error) { log.Info("Checking if tag exists already") // see if tag exists already @@ -74,7 +74,7 @@ func getTagID(client *graphql.Client, create bool) (*graphql.ID, error) { AllTags []Tag `graphql:"allTags"` } - err := client.Query(context.Background(), &q, nil) + err := client.Query(ctx, &q, nil) if err != nil { return nil, fmt.Errorf("Error getting tags: %s\n", err.Error()) } @@ -106,7 +106,7 @@ func getTagID(client *graphql.Client, create bool) (*graphql.ID, error) { log.Info("Creating new tag") - err = client.Mutate(context.Background(), &m, vars) + err = client.Mutate(ctx, &m, vars) if err != nil { return nil, fmt.Errorf("Error mutating scene: %s\n", err.Error()) } @@ -114,7 +114,7 @@ func getTagID(client *graphql.Client, create bool) (*graphql.ID, error) { return &m.TagCreate.ID, nil } -func findRandomScene(client *graphql.Client) (*Scene, error) { +func findRandomScene(ctx context.Context, client *graphql.Client) (*Scene, error) { // get a random scene var q struct { FindScenes FindScenesResultType `graphql:"findScenes(filter: $c)"` @@ -132,7 +132,7 @@ func findRandomScene(client *graphql.Client) (*Scene, error) { } log.Info("Finding a random scene") - err := client.Query(context.Background(), &q, vars) + err := client.Query(ctx, &q, vars) if err != nil { return nil, fmt.Errorf("Error getting random scene: %s\n", err.Error()) } @@ -155,14 +155,14 @@ func addTagId(tagIds []graphql.ID, tagId graphql.ID) []graphql.ID { return tagIds } -func AddTag(client *graphql.Client) error { - tagID, err := getTagID(client, true) +func AddTag(ctx context.Context, client *graphql.Client) error { + tagID, err := getTagID(ctx, client, true) if err != nil { return err } - scene, err := findRandomScene(client) + scene, err := findRandomScene(ctx, client) if err != nil { return err @@ -188,7 +188,7 @@ func AddTag(client *graphql.Client) error { } log.Infof("Adding tag to scene %v", scene.ID) - err = client.Mutate(context.Background(), &m, vars) + err = client.Mutate(ctx, &m, vars) if err != nil { return fmt.Errorf("Error mutating scene: %v", err) } @@ -196,8 +196,8 @@ func AddTag(client *graphql.Client) error { return nil } -func RemoveTag(client *graphql.Client) error { - tagID, err := getTagID(client, false) +func RemoveTag(ctx context.Context, client *graphql.Client) error { + tagID, err := getTagID(ctx, client, false) if err != nil { return err @@ -223,7 +223,7 @@ func RemoveTag(client *graphql.Client) error { log.Info("Destroying tag") - err = client.Mutate(context.Background(), &m, vars) + err = client.Mutate(ctx, &m, vars) if err != nil { return fmt.Errorf("Error destroying tag: %v", err) } diff --git a/pkg/plugin/js.go b/pkg/plugin/js.go index 4c918c686..2ebbb6ccc 100644 --- a/pkg/plugin/js.go +++ b/pkg/plugin/js.go @@ -1,6 +1,7 @@ package plugin import ( + "context" "errors" "fmt" "path/filepath" @@ -84,7 +85,7 @@ func (t *jsPluginTask) Start() error { return fmt.Errorf("error adding util API: %w", err) } - if err := js.AddGQLAPI(t.vm, t.input.ServerConnection.SessionCookie, t.gqlHandler); err != nil { + if err := js.AddGQLAPI(context.TODO(), t.vm, t.input.ServerConnection.SessionCookie, t.gqlHandler); err != nil { return fmt.Errorf("error adding GraphQL API: %w", err) } diff --git a/pkg/plugin/js/gql.go b/pkg/plugin/js/gql.go index 45f9ac9e7..9c8461177 100644 --- a/pkg/plugin/js/gql.go +++ b/pkg/plugin/js/gql.go @@ -2,6 +2,7 @@ package js import ( "bytes" + "context" "encoding/json" "fmt" "net/http" @@ -33,7 +34,7 @@ func throw(vm *otto.Otto, str string) { panic(value) } -func gqlRequestFunc(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) func(call otto.FunctionCall) otto.Value { +func gqlRequestFunc(ctx context.Context, vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) func(call otto.FunctionCall) otto.Value { return func(call otto.FunctionCall) otto.Value { if len(call.ArgumentList) == 0 { throw(vm, "missing argument") @@ -61,7 +62,7 @@ func gqlRequestFunc(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) throw(vm, err.Error()) } - r, err := http.NewRequest("POST", "/graphql", &body) + r, err := http.NewRequestWithContext(ctx, "POST", "/graphql", &body) if err != nil { throw(vm, "could not make request") } @@ -103,9 +104,9 @@ func gqlRequestFunc(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) } } -func AddGQLAPI(vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) error { +func AddGQLAPI(ctx context.Context, vm *otto.Otto, cookie *http.Cookie, gqlHandler http.Handler) error { gql, _ := vm.Object("({})") - if err := gql.Set("Do", gqlRequestFunc(vm, cookie, gqlHandler)); err != nil { + if err := gql.Set("Do", gqlRequestFunc(ctx, vm, cookie, gqlHandler)); err != nil { return fmt.Errorf("unable to set GraphQL Do function: %w", err) } diff --git a/pkg/scraper/action.go b/pkg/scraper/action.go index b5e67e712..838611252 100644 --- a/pkg/scraper/action.go +++ b/pkg/scraper/action.go @@ -1,6 +1,8 @@ package scraper -import "github.com/stashapp/stash/pkg/models" +import ( + "github.com/stashapp/stash/pkg/models" +) type scraperAction string diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index e2b3b5e1e..02376df6f 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -1,6 +1,7 @@ package scraper import ( + "context" "crypto/tls" "fmt" "io" @@ -16,13 +17,13 @@ import ( // configurable at some point. const imageGetTimeout = time.Second * 30 -func setPerformerImage(p *models.ScrapedPerformer, globalConfig GlobalConfig) error { +func setPerformerImage(ctx context.Context, p *models.ScrapedPerformer, globalConfig GlobalConfig) error { if p == nil || p.Image == nil || !strings.HasPrefix(*p.Image, "http") { // nothing to do return nil } - img, err := getImage(*p.Image, globalConfig) + img, err := getImage(ctx, *p.Image, globalConfig) if err != nil { return err } @@ -34,14 +35,14 @@ func setPerformerImage(p *models.ScrapedPerformer, globalConfig GlobalConfig) er return nil } -func setSceneImage(s *models.ScrapedScene, globalConfig GlobalConfig) error { +func setSceneImage(ctx context.Context, s *models.ScrapedScene, globalConfig GlobalConfig) error { // don't try to get the image if it doesn't appear to be a URL if s == nil || s.Image == nil || !strings.HasPrefix(*s.Image, "http") { // nothing to do return nil } - img, err := getImage(*s.Image, globalConfig) + img, err := getImage(ctx, *s.Image, globalConfig) if err != nil { return err } @@ -51,14 +52,14 @@ func setSceneImage(s *models.ScrapedScene, globalConfig GlobalConfig) error { return nil } -func setMovieFrontImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error { +func setMovieFrontImage(ctx context.Context, m *models.ScrapedMovie, globalConfig GlobalConfig) error { // don't try to get the image if it doesn't appear to be a URL if m == nil || m.FrontImage == nil || !strings.HasPrefix(*m.FrontImage, "http") { // nothing to do return nil } - img, err := getImage(*m.FrontImage, globalConfig) + img, err := getImage(ctx, *m.FrontImage, globalConfig) if err != nil { return err } @@ -68,14 +69,14 @@ func setMovieFrontImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error return nil } -func setMovieBackImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error { +func setMovieBackImage(ctx context.Context, m *models.ScrapedMovie, globalConfig GlobalConfig) error { // don't try to get the image if it doesn't appear to be a URL if m == nil || m.BackImage == nil || !strings.HasPrefix(*m.BackImage, "http") { // nothing to do return nil } - img, err := getImage(*m.BackImage, globalConfig) + img, err := getImage(ctx, *m.BackImage, globalConfig) if err != nil { return err } @@ -85,14 +86,14 @@ func setMovieBackImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error return nil } -func getImage(url string, globalConfig GlobalConfig) (*string, error) { +func getImage(ctx context.Context, url string, globalConfig GlobalConfig) (*string, error) { client := &http.Client{ Transport: &http.Transport{ // ignore insecure certificates TLSClientConfig: &tls.Config{InsecureSkipVerify: !globalConfig.GetScraperCertCheck()}}, Timeout: imageGetTimeout, } - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -136,10 +137,10 @@ func getImage(url string, globalConfig GlobalConfig) (*string, error) { return &img, nil } -func getStashPerformerImage(stashURL string, performerID string, globalConfig GlobalConfig) (*string, error) { - return getImage(stashURL+"/performer/"+performerID+"/image", globalConfig) +func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, globalConfig GlobalConfig) (*string, error) { + return getImage(ctx, stashURL+"/performer/"+performerID+"/image", globalConfig) } -func getStashSceneImage(stashURL string, sceneID string, globalConfig GlobalConfig) (*string, error) { - return getImage(stashURL+"/scene/"+sceneID+"/screenshot", globalConfig) +func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, globalConfig GlobalConfig) (*string, error) { + return getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot", globalConfig) } diff --git a/pkg/scraper/json.go b/pkg/scraper/json.go index e99a4b698..b291167a5 100644 --- a/pkg/scraper/json.go +++ b/pkg/scraper/json.go @@ -1,6 +1,7 @@ package scraper import ( + "context" "errors" "io" "net/url" @@ -31,14 +32,14 @@ func (s *jsonScraper) getJsonScraper() *mappedScraper { return s.config.JsonScrapers[s.scraper.Scraper] } -func (s *jsonScraper) scrapeURL(url string) (string, *mappedScraper, error) { +func (s *jsonScraper) scrapeURL(ctx context.Context, url string) (string, *mappedScraper, error) { scraper := s.getJsonScraper() if scraper == nil { return "", nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(ctx, url) if err != nil { return "", nil, err @@ -47,8 +48,8 @@ func (s *jsonScraper) scrapeURL(url string) (string, *mappedScraper, error) { return doc, scraper, nil } -func (s *jsonScraper) loadURL(url string) (string, error) { - r, err := loadURL(url, s.config, s.globalConfig) +func (s *jsonScraper) loadURL(ctx context.Context, url string) (string, error) { + r, err := loadURL(ctx, url, s.config, s.globalConfig) if err != nil { return "", err } @@ -72,7 +73,7 @@ func (s *jsonScraper) loadURL(url string) (string, error) { func (s *jsonScraper) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) { u := replaceURL(url, s.scraper) // allow a URL Replace for performer by URL queries - doc, scraper, err := s.scrapeURL(u) + doc, scraper, err := s.scrapeURL(context.TODO(), u) if err != nil { return nil, err } @@ -83,7 +84,7 @@ func (s *jsonScraper) scrapePerformerByURL(url string) (*models.ScrapedPerformer func (s *jsonScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error) { u := replaceURL(url, s.scraper) // allow a URL Replace for scene by URL queries - doc, scraper, err := s.scrapeURL(u) + doc, scraper, err := s.scrapeURL(context.TODO(), u) if err != nil { return nil, err } @@ -94,7 +95,7 @@ func (s *jsonScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error) func (s *jsonScraper) scrapeGalleryByURL(url string) (*models.ScrapedGallery, error) { u := replaceURL(url, s.scraper) // allow a URL Replace for gallery by URL queries - doc, scraper, err := s.scrapeURL(u) + doc, scraper, err := s.scrapeURL(context.TODO(), u) if err != nil { return nil, err } @@ -105,7 +106,7 @@ func (s *jsonScraper) scrapeGalleryByURL(url string) (*models.ScrapedGallery, er func (s *jsonScraper) scrapeMovieByURL(url string) (*models.ScrapedMovie, error) { u := replaceURL(url, s.scraper) // allow a URL Replace for movie by URL queries - doc, scraper, err := s.scrapeURL(u) + doc, scraper, err := s.scrapeURL(context.TODO(), u) if err != nil { return nil, err } @@ -129,7 +130,7 @@ func (s *jsonScraper) scrapePerformersByName(name string) ([]*models.ScrapedPerf url := s.scraper.QueryURL url = strings.Replace(url, placeholder, escapedName, -1) - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -158,7 +159,7 @@ func (s *jsonScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, e url := s.scraper.QueryURL url = strings.Replace(url, placeholder, escapedName, -1) - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -182,7 +183,7 @@ func (s *jsonScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedSc return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -206,7 +207,7 @@ func (s *jsonScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*mo return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -230,7 +231,7 @@ func (s *jsonScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.S return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -278,7 +279,7 @@ func (q *jsonQuery) runQuery(selector string) []string { } func (q *jsonQuery) subScrape(value string) mappedQuery { - doc, err := q.scraper.loadURL(value) + doc, err := q.scraper.loadURL(context.TODO(), value) if err != nil { logger.Warnf("Error getting URL '%s' for sub-scraper: %s", value, err.Error()) diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go index 697155c77..82795bb0e 100644 --- a/pkg/scraper/scrapers.go +++ b/pkg/scraper/scrapers.go @@ -202,7 +202,7 @@ func (c Cache) ScrapePerformer(scraperID string, scrapedPerformer models.Scraped } if ret != nil { - err = c.postScrapePerformer(ret) + err = c.postScrapePerformer(context.TODO(), ret) if err != nil { return nil, err } @@ -226,7 +226,7 @@ func (c Cache) ScrapePerformerURL(url string) (*models.ScrapedPerformer, error) } if ret != nil { - err = c.postScrapePerformer(ret) + err = c.postScrapePerformer(context.TODO(), ret) if err != nil { return nil, err } @@ -239,8 +239,8 @@ func (c Cache) ScrapePerformerURL(url string) (*models.ScrapedPerformer, error) return nil, nil } -func (c Cache) postScrapePerformer(ret *models.ScrapedPerformer) error { - if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { +func (c Cache) postScrapePerformer(ctx context.Context, ret *models.ScrapedPerformer) error { + if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { tqb := r.Tag() tags, err := postProcessTags(tqb, ret.Tags) @@ -255,7 +255,7 @@ func (c Cache) postScrapePerformer(ret *models.ScrapedPerformer) error { } // post-process - set the image if applicable - if err := setPerformerImage(ret, c.globalConfig); err != nil { + if err := setPerformerImage(ctx, ret, c.globalConfig); err != nil { logger.Warnf("Could not set image using URL %s: %s", *ret.Image, err.Error()) } @@ -280,8 +280,8 @@ func (c Cache) postScrapeScenePerformer(ret *models.ScrapedPerformer) error { return nil } -func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { - if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { +func (c Cache) postScrapeScene(ctx context.Context, ret *models.ScrapedScene) error { + if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { pqb := r.Performer() mqb := r.Movie() tqb := r.Tag() @@ -323,8 +323,8 @@ func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { } // post-process - set the image if applicable - if err := setSceneImage(ret, c.globalConfig); err != nil { - logger.Warnf("Could not set image using URL %s: %s", *ret.Image, err.Error()) + if err := setSceneImage(ctx, ret, c.globalConfig); err != nil { + logger.Warnf("Could not set image using URL %s: %v", *ret.Image, err) } return nil @@ -382,7 +382,7 @@ func (c Cache) ScrapeScene(scraperID string, sceneID int) (*models.ScrapedScene, } if ret != nil { - err = c.postScrapeScene(ret) + err = c.postScrapeScene(context.TODO(), ret) if err != nil { return nil, err } @@ -419,7 +419,7 @@ func (c Cache) ScrapeSceneFragment(scraperID string, scene models.ScrapedSceneIn } if ret != nil { - err = c.postScrapeScene(ret) + err = c.postScrapeScene(context.TODO(), ret) if err != nil { return nil, err } @@ -443,7 +443,7 @@ func (c Cache) ScrapeSceneURL(url string) (*models.ScrapedScene, error) { return nil, err } - err = c.postScrapeScene(ret) + err = c.postScrapeScene(context.TODO(), ret) if err != nil { return nil, err } @@ -551,10 +551,10 @@ func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error) { } // post-process - set the image if applicable - if err := setMovieFrontImage(ret, c.globalConfig); err != nil { + if err := setMovieFrontImage(context.TODO(), ret, c.globalConfig); err != nil { logger.Warnf("Could not set front image using URL %s: %s", *ret.FrontImage, err.Error()) } - if err := setMovieBackImage(ret, c.globalConfig); err != nil { + if err := setMovieBackImage(context.TODO(), ret, c.globalConfig); err != nil { logger.Warnf("Could not set back image using URL %s: %s", *ret.BackImage, err.Error()) } diff --git a/pkg/scraper/stash.go b/pkg/scraper/stash.go index 7023a6d3a..df83d5fa4 100644 --- a/pkg/scraper/stash.go +++ b/pkg/scraper/stash.go @@ -69,7 +69,7 @@ func (s *stashScraper) scrapePerformersByName(name string) ([]*models.ScrapedPer }, } - err := client.Query(context.Background(), &q, vars) + err := client.Query(context.TODO(), &q, vars) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped "f": performerID, } - err := client.Query(context.Background(), &q, vars) + err := client.Query(context.TODO(), &q, vars) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped } // get the performer image directly - ret.Image, err = getStashPerformerImage(s.config.StashServer.URL, performerID, s.globalConfig) + ret.Image, err = getStashPerformerImage(context.TODO(), s.config.StashServer.URL, performerID, s.globalConfig) if err != nil { return nil, err } @@ -164,7 +164,7 @@ func (s *stashScraper) scrapedStashSceneToScrapedScene(scene *scrapedSceneStash) } // get the performer image directly - ret.Image, err = getStashSceneImage(s.config.StashServer.URL, scene.ID, s.globalConfig) + ret.Image, err = getStashSceneImage(context.TODO(), s.config.StashServer.URL, scene.ID, s.globalConfig) if err != nil { return nil, err } @@ -190,7 +190,7 @@ func (s *stashScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, }, } - err := client.Query(context.Background(), &q, vars) + err := client.Query(context.TODO(), &q, vars) if err != nil { return nil, err } @@ -240,7 +240,7 @@ func (s *stashScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedS } client := s.getStashClient() - if err := client.Query(context.Background(), &q, vars); err != nil { + if err := client.Query(context.TODO(), &q, vars); err != nil { return nil, err } @@ -251,7 +251,7 @@ func (s *stashScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedS } // get the performer image directly - ret.Image, err = getStashSceneImage(s.config.StashServer.URL, q.FindScene.ID, s.globalConfig) + ret.Image, err = getStashSceneImage(context.TODO(), s.config.StashServer.URL, q.FindScene.ID, s.globalConfig) if err != nil { return nil, err } @@ -293,7 +293,7 @@ func (s *stashScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models. } client := s.getStashClient() - if err := client.Query(context.Background(), &q, vars); err != nil { + if err := client.Query(context.TODO(), &q, vars); err != nil { return nil, err } diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 21d2b7ec8..ff7a4b520 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -45,8 +45,8 @@ func NewClient(box models.StashBox, txnManager models.TransactionManager) *Clien } // QueryStashBoxScene queries stash-box for scenes using a query string. -func (c Client) QueryStashBoxScene(queryStr string) ([]*models.ScrapedScene, error) { - scenes, err := c.client.SearchScene(context.TODO(), queryStr) +func (c Client) QueryStashBoxScene(ctx context.Context, queryStr string) ([]*models.ScrapedScene, error) { + scenes, err := c.client.SearchScene(ctx, queryStr) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func (c Client) QueryStashBoxScene(queryStr string) ([]*models.ScrapedScene, err var ret []*models.ScrapedScene for _, s := range sceneFragments { - ss, err := sceneFragmentToScrapedScene(c.txnManager, s) + ss, err := sceneFragmentToScrapedScene(context.TODO(), c.txnManager, s) if err != nil { return nil, err } @@ -69,6 +69,8 @@ func (c Client) QueryStashBoxScene(queryStr string) ([]*models.ScrapedScene, err // scene's MD5/OSHASH checksum, or PHash, and returns results in the same order // as the input slice. func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models.ScrapedScene, error) { + ctx := context.TODO() + ids, err := utils.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err @@ -78,7 +80,7 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models // map fingerprints to their scene index fpToScene := make(map[string][]int) - if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { qb := r.Scene() for index, sceneID := range ids { @@ -113,7 +115,7 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models return nil, err } - allScenes, err := c.findStashBoxScenesByFingerprints(fingerprints) + allScenes, err := c.findStashBoxScenesByFingerprints(ctx, fingerprints) if err != nil { return nil, err } @@ -139,6 +141,8 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models // FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every // scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice. func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*models.ScrapedScene, error) { + ctx := context.TODO() + ids, err := utils.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err @@ -146,7 +150,7 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*mode var fingerprints []string - if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { qb := r.Scene() for _, sceneID := range ids { @@ -178,17 +182,17 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*mode return nil, err } - return c.findStashBoxScenesByFingerprints(fingerprints) + return c.findStashBoxScenesByFingerprints(ctx, fingerprints) } -func (c Client) findStashBoxScenesByFingerprints(fingerprints []string) ([]*models.ScrapedScene, error) { +func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprints []string) ([]*models.ScrapedScene, error) { var ret []*models.ScrapedScene for i := 0; i < len(fingerprints); i += 100 { end := i + 100 if end > len(fingerprints) { end = len(fingerprints) } - scenes, err := c.client.FindScenesByFingerprints(context.TODO(), fingerprints[i:end]) + scenes, err := c.client.FindScenesByFingerprints(ctx, fingerprints[i:end]) if err != nil { return nil, err @@ -197,7 +201,7 @@ func (c Client) findStashBoxScenesByFingerprints(fingerprints []string) ([]*mode sceneFragments := scenes.FindScenesByFingerprints for _, s := range sceneFragments { - ss, err := sceneFragmentToScrapedScene(c.txnManager, s) + ss, err := sceneFragmentToScrapedScene(ctx, c.txnManager, s) if err != nil { return nil, err } @@ -504,12 +508,12 @@ func formatBodyModifications(m []*graphql.BodyModificationFragment) *string { return &ret } -func fetchImage(url string) (*string, error) { +func fetchImage(ctx context.Context, url string) (*string, error) { client := &http.Client{ Timeout: imageGetTimeout, } - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -590,8 +594,8 @@ func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *mode return sp } -func getFirstImage(images []*graphql.ImageFragment) *string { - ret, err := fetchImage(images[0].URL) +func getFirstImage(ctx context.Context, images []*graphql.ImageFragment) *string { + ret, err := fetchImage(ctx, images[0].URL) if err != nil { logger.Warnf("Error fetching image %s: %s", images[0].URL, err.Error()) } @@ -612,7 +616,7 @@ func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint return fingerprints } -func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphql.SceneFragment) (*models.ScrapedScene, error) { +func sceneFragmentToScrapedScene(ctx context.Context, txnManager models.TransactionManager, s *graphql.SceneFragment) (*models.ScrapedScene, error) { stashID := s.ID ss := &models.ScrapedScene{ Title: s.Title, @@ -629,10 +633,10 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq if len(s.Images) > 0 { // TODO - #454 code sorts images by aspect ratio according to a wanted // orientation. I'm just grabbing the first for now - ss.Image = getFirstImage(s.Images) + ss.Image = getFirstImage(ctx, s.Images) } - if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + if err := txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error { pqb := r.Performer() tqb := r.Tag() diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go index fe33c157d..516c26387 100644 --- a/pkg/scraper/url.go +++ b/pkg/scraper/url.go @@ -28,11 +28,11 @@ import ( const scrapeGetTimeout = time.Second * 60 const scrapeDefaultSleep = time.Second * 2 -func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Reader, error) { +func loadURL(ctx context.Context, url string, scraperConfig config, globalConfig GlobalConfig) (io.Reader, error) { driverOptions := scraperConfig.DriverOptions if driverOptions != nil && driverOptions.UseCDP { // get the page using chrome dp - return urlFromCDP(url, *driverOptions, globalConfig) + return urlFromCDP(ctx, url, *driverOptions, globalConfig) } // get the page using http.Client @@ -62,7 +62,7 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re Jar: jar, } - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re // func urlFromCDP uses chrome cdp and DOM to load and process the url // if remote is set as true in the scraperConfig it will try to use localhost:9222 // else it will look for google-chrome in path -func urlFromCDP(url string, driverOptions scraperDriverOptions, globalConfig GlobalConfig) (io.Reader, error) { +func urlFromCDP(ctx context.Context, url string, driverOptions scraperDriverOptions, globalConfig GlobalConfig) (io.Reader, error) { if !driverOptions.UseCDP { return nil, fmt.Errorf("url shouldn't be fetched through CDP") @@ -117,7 +117,7 @@ func urlFromCDP(url string, driverOptions scraperDriverOptions, globalConfig Glo sleepDuration = time.Duration(driverOptions.Sleep) * time.Second } - act := context.Background() + act := context.TODO() // if scraperCDPPath is a remote address, then allocate accordingly cdpPath := globalConfig.GetScraperCDPPath() @@ -130,13 +130,13 @@ func urlFromCDP(url string, driverOptions scraperDriverOptions, globalConfig Glo // if CDPPath is http(s) then we need to get the websocket URL if isCDPPathHTTP(globalConfig) { var err error - remote, err = getRemoteCDPWSAddress(remote) + remote, err = getRemoteCDPWSAddress(ctx, remote) if err != nil { return nil, err } } - act, cancelAct = chromedp.NewRemoteAllocator(context.Background(), remote) + act, cancelAct = chromedp.NewRemoteAllocator(act, remote) } else { // use a temporary user directory for chrome dir, err := os.MkdirTemp("", "stash-chromedp") @@ -218,8 +218,13 @@ func setCDPClicks(driverOptions scraperDriverOptions) chromedp.Tasks { } // getRemoteCDPWSAddress returns the complete remote address that is required to access the cdp instance -func getRemoteCDPWSAddress(address string) (string, error) { - resp, err := http.Get(address) +func getRemoteCDPWSAddress(ctx context.Context, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return "", err } diff --git a/pkg/scraper/xpath.go b/pkg/scraper/xpath.go index 0cd955788..8a8c69d40 100644 --- a/pkg/scraper/xpath.go +++ b/pkg/scraper/xpath.go @@ -2,6 +2,7 @@ package scraper import ( "bytes" + "context" "errors" "net/url" "regexp" @@ -42,7 +43,7 @@ func (s *xpathScraper) scrapeURL(url string) (*html.Node, *mappedScraper, error) return nil, nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, nil, err @@ -110,7 +111,7 @@ func (s *xpathScraper) scrapePerformersByName(name string) ([]*models.ScrapedPer url := s.scraper.QueryURL url = strings.Replace(url, placeholder, escapedName, -1) - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -139,7 +140,7 @@ func (s *xpathScraper) scrapeScenesByName(name string) ([]*models.ScrapedScene, url := s.scraper.QueryURL url = strings.Replace(url, placeholder, escapedName, -1) - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -163,7 +164,7 @@ func (s *xpathScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedS return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -187,7 +188,7 @@ func (s *xpathScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*m return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -211,7 +212,7 @@ func (s *xpathScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models. return nil, errors.New("xpath scraper with name " + s.scraper.Scraper + " not found in config") } - doc, err := s.loadURL(url) + doc, err := s.loadURL(context.TODO(), url) if err != nil { return nil, err @@ -225,8 +226,8 @@ func (s *xpathScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInpu return nil, errors.New("scrapeGalleryByFragment not supported for xpath scraper") } -func (s *xpathScraper) loadURL(url string) (*html.Node, error) { - r, err := loadURL(url, s.config, s.globalConfig) +func (s *xpathScraper) loadURL(ctx context.Context, url string) (*html.Node, error) { + r, err := loadURL(ctx, url, s.config, s.globalConfig) if err != nil { return nil, err } @@ -298,7 +299,7 @@ func (q *xpathQuery) nodeText(n *html.Node) string { } func (q *xpathQuery) subScrape(value string) mappedQuery { - doc, err := q.scraper.loadURL(value) + doc, err := q.scraper.loadURL(context.TODO(), value) if err != nil { logger.Warnf("Error getting URL '%s' for sub-scraper: %s", value, err.Error()) diff --git a/pkg/utils/image.go b/pkg/utils/image.go index 75903fa6d..d1993af98 100644 --- a/pkg/utils/image.go +++ b/pkg/utils/image.go @@ -1,6 +1,7 @@ package utils import ( + "context" "crypto/md5" "crypto/tls" "encoding/base64" @@ -20,7 +21,7 @@ const base64RE = `^data:.+\/(.+);base64,(.*)$` // ProcessImageInput transforms an image string either from a base64 encoded // string, or from a URL, and returns the image as a byte slice -func ProcessImageInput(imageInput string) ([]byte, error) { +func ProcessImageInput(ctx context.Context, imageInput string) ([]byte, error) { regex := regexp.MustCompile(base64RE) if regex.MatchString(imageInput) { _, d, err := ProcessBase64Image(imageInput) @@ -28,11 +29,11 @@ func ProcessImageInput(imageInput string) ([]byte, error) { } // assume input is a URL. Read it. - return ReadImageFromURL(imageInput) + return ReadImageFromURL(ctx, imageInput) } // ReadImageFromURL returns image data from a URL -func ReadImageFromURL(url string) ([]byte, error) { +func ReadImageFromURL(ctx context.Context, url string) ([]byte, error) { client := &http.Client{ Transport: &http.Transport{ // ignore insecure certificates TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -41,7 +42,7 @@ func ReadImageFromURL(url string) ([]byte, error) { Timeout: imageGetTimeout, } - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err }