From 5f690d96bdeb341b50ba2e91d57898b2aef9aa0b Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:14:55 +1100 Subject: [PATCH] Fix stash scraper errors and add apikey field (#5474) * Use hasura/go-graphql-client instead of shurcooL version * Fix graphql query errors * Support setting api key for stash server --- go.mod | 3 +- go.sum | 6 +- pkg/plugin/examples/common/graphql.go | 2 +- pkg/plugin/util/client.go | 2 +- pkg/scraper/config.go | 3 +- pkg/scraper/graphql.go | 55 ++++++++ pkg/scraper/image.go | 33 ++++- pkg/scraper/stash.go | 122 +++++++++++++++--- .../src/docs/en/Manual/ScraperDevelopment.md | 3 +- 9 files changed, 195 insertions(+), 34 deletions(-) create mode 100644 pkg/scraper/graphql.go diff --git a/go.mod b/go.mod index d1bd40939..f35ee773e 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/gorilla/sessions v1.2.1 github.com/gorilla/websocket v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/hasura/go-graphql-client v0.13.1 github.com/jinzhu/copier v0.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/json-iterator/go v1.1.12 @@ -39,7 +40,6 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd - github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f github.com/sirupsen/logrus v1.9.3 github.com/spf13/cast v1.6.0 github.com/spf13/pflag v1.0.5 @@ -67,6 +67,7 @@ require ( github.com/asticode/go-astikit v0.20.0 // indirect github.com/asticode/go-astits v1.8.0 // indirect github.com/chromedp/sysutil v1.0.0 // indirect + github.com/coder/websocket v1.8.12 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect diff --git a/go.sum b/go.sum index 15516f7ff..e069fb1a1 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -394,6 +396,8 @@ github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoI github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hasura/go-graphql-client v0.13.1 h1:kKbjhxhpwz58usVl+Xvgah/TDha5K2akNTRQdsEHN6U= +github.com/hasura/go-graphql-client v0.13.1/go.mod h1:k7FF7h53C+hSNFRG3++DdVZWIuHdCaTbI7siTJ//zGQ= github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= @@ -591,8 +595,6 @@ github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDN github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk= -github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= diff --git a/pkg/plugin/examples/common/graphql.go b/pkg/plugin/examples/common/graphql.go index 8650758a8..40ac8d77e 100644 --- a/pkg/plugin/examples/common/graphql.go +++ b/pkg/plugin/examples/common/graphql.go @@ -8,7 +8,7 @@ import ( "errors" "fmt" - "github.com/shurcooL/graphql" + graphql "github.com/hasura/go-graphql-client" "github.com/stashapp/stash/pkg/plugin/common/log" ) diff --git a/pkg/plugin/util/client.go b/pkg/plugin/util/client.go index 7b33d8678..37c37bfac 100644 --- a/pkg/plugin/util/client.go +++ b/pkg/plugin/util/client.go @@ -8,7 +8,7 @@ import ( "net/url" "strconv" - "github.com/shurcooL/graphql" + graphql "github.com/hasura/go-graphql-client" "github.com/stashapp/stash/pkg/plugin/common" ) diff --git a/pkg/scraper/config.go b/pkg/scraper/config.go index 9c51b4bba..e19625f45 100644 --- a/pkg/scraper/config.go +++ b/pkg/scraper/config.go @@ -114,7 +114,8 @@ func (c config) validate() error { } type stashServer struct { - URL string `yaml:"url"` + URL string `yaml:"url"` + ApiKey string `yaml:"apiKey"` } type scraperTypeConfig struct { diff --git a/pkg/scraper/graphql.go b/pkg/scraper/graphql.go new file mode 100644 index 000000000..8f582fe84 --- /dev/null +++ b/pkg/scraper/graphql.go @@ -0,0 +1,55 @@ +package scraper + +import ( + "errors" + "strings" + + "github.com/hasura/go-graphql-client" +) + +type graphqlErrors []error + +func (e graphqlErrors) Error() string { + b := strings.Builder{} + for _, err := range e { + _, _ = b.WriteString(err.Error()) + } + return b.String() +} + +type graphqlError struct { + err graphql.Error +} + +func (e graphqlError) Error() string { + unwrapped := e.err.Unwrap() + if unwrapped != nil { + var networkErr graphql.NetworkError + if errors.As(unwrapped, &networkErr) { + if networkErr.StatusCode() == 422 { + return networkErr.Body() + } + } + } + return e.err.Error() +} + +// convertGraphqlError converts a graphql.Error or graphql.Errors into an error with a useful message. +// graphql.Error swallows important information, so we need to convert it to a more useful error type. +func convertGraphqlError(err error) error { + var gqlErrs graphql.Errors + if errors.As(err, &gqlErrs) { + ret := make(graphqlErrors, len(gqlErrs)) + for i, e := range gqlErrs { + ret[i] = convertGraphqlError(e) + } + return ret + } + + var gqlErr graphql.Error + if errors.As(err, &gqlErr) { + return graphqlError{gqlErr} + } + + return err +} diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index 193ddc517..ee82d2f21 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -122,13 +122,19 @@ func setGroupBackImage(ctx context.Context, client *http.Client, m *models.Scrap return nil } -func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) { +type imageGetter struct { + client *http.Client + globalConfig GlobalConfig + requestModifier func(req *http.Request) +} + +func (i *imageGetter) getImage(ctx context.Context, url string) (*string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } - userAgent := globalConfig.GetScraperUserAgent() + userAgent := i.globalConfig.GetScraperUserAgent() if userAgent != "" { req.Header.Set("User-Agent", userAgent) } @@ -140,7 +146,11 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/") } - resp, err := client.Do(req) + if i.requestModifier != nil { + i.requestModifier(req) + } + + resp, err := i.client.Do(req) if err != nil { return nil, err @@ -167,10 +177,19 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig return &img, nil } -func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, client *http.Client, globalConfig GlobalConfig) (*string, error) { - return getImage(ctx, stashURL+"/performer/"+performerID+"/image", client, globalConfig) +func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) { + g := imageGetter{ + client: client, + globalConfig: globalConfig, + } + + return g.getImage(ctx, url) } -func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, client *http.Client, globalConfig GlobalConfig) (*string, error) { - return getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot", client, globalConfig) +func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, imageGetter imageGetter) (*string, error) { + return imageGetter.getImage(ctx, stashURL+"/performer/"+performerID+"/image") +} + +func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, imageGetter imageGetter) (*string, error) { + return imageGetter.getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot") } diff --git a/pkg/scraper/stash.go b/pkg/scraper/stash.go index a50db8b5e..3e28a3e99 100644 --- a/pkg/scraper/stash.go +++ b/pkg/scraper/stash.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "net/http" + "strconv" + "strings" + graphql "github.com/hasura/go-graphql-client" "github.com/jinzhu/copier" - "github.com/shurcooL/graphql" "github.com/stashapp/stash/pkg/models" ) @@ -27,9 +29,21 @@ func newStashScraper(scraper scraperTypeConfig, client *http.Client, config conf } } +func setApiKeyHeader(apiKey string) func(req *http.Request) { + return func(req *http.Request) { + req.Header.Set("ApiKey", apiKey) + } +} + func (s *stashScraper) getStashClient() *graphql.Client { - url := s.config.StashServer.URL - return graphql.NewClient(url+"/graphql", nil) + url := s.config.StashServer.URL + "/graphql" + ret := graphql.NewClient(url, s.client) + + if s.config.StashServer.ApiKey != "" { + ret = ret.WithRequestModifier(setApiKeyHeader(s.config.StashServer.ApiKey)) + } + + return ret } type stashFindPerformerNamePerformer struct { @@ -58,14 +72,12 @@ type scrapedTagStash struct { type scrapedPerformerStash struct { Name *string `graphql:"name" json:"name"` Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` + URLs []string `graphql:"urls" json:"urls"` Birthdate *string `graphql:"birthdate" json:"birthdate"` Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` Country *string `graphql:"country" json:"country"` EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` + Height *int `graphql:"height_cm" json:"height_cm"` Measurements *string `graphql:"measurements" json:"measurements"` FakeTits *string `graphql:"fake_tits" json:"fake_tits"` PenisLength *string `graphql:"penis_length" json:"penis_length"` @@ -73,12 +85,25 @@ type scrapedPerformerStash struct { CareerLength *string `graphql:"career_length" json:"career_length"` Tattoos *string `graphql:"tattoos" json:"tattoos"` Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` + Aliases []string `graphql:"alias_list" json:"alias_list"` Tags []*scrapedTagStash `graphql:"tags" json:"tags"` Details *string `graphql:"details" json:"details"` DeathDate *string `graphql:"death_date" json:"death_date"` HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` + Weight *int `graphql:"weight" json:"weight"` +} + +func (s *stashScraper) imageGetter() imageGetter { + ret := imageGetter{ + client: s.client, + globalConfig: s.globalConfig, + } + + if s.config.StashServer.ApiKey != "" { + ret.requestModifier = setApiKeyHeader(s.config.StashServer.ApiKey) + } + + return ret } func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error) { @@ -102,12 +127,12 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap // get the id from the URL field vars := map[string]interface{}{ - "f": performerID, + "f": graphql.ID(performerID), } err := client.Query(ctx, &q, vars) if err != nil { - return nil, err + return nil, convertGraphqlError(err) } // need to copy back to a scraped performer @@ -117,11 +142,28 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap return nil, err } + // convert alias list to aliases + aliasStr := strings.Join(q.FindPerformer.Aliases, ", ") + ret.Aliases = &aliasStr + + // convert numeric to string + if q.FindPerformer.Height != nil { + heightStr := strconv.Itoa(*q.FindPerformer.Height) + ret.Height = &heightStr + } + if q.FindPerformer.Weight != nil { + weightStr := strconv.Itoa(*q.FindPerformer.Weight) + ret.Weight = &weightStr + } + // get the performer image directly - ret.Image, err = getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, s.client, s.globalConfig) + ig := s.imageGetter() + img, err := getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, ig) if err != nil { return nil, err } + ret.Images = []string{*img} + ret.Image = img return &ret, nil } @@ -143,8 +185,15 @@ func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scen return nil, err } - // get the performer image directly - ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, s.client, s.globalConfig) + // convert first in files to file + if len(scene.Files) > 0 { + f := scene.Files[0].SceneFileType() + ret.File = &f + } + + // get the scene image directly + ig := s.imageGetter() + ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, ig) if err != nil { return nil, err } @@ -175,7 +224,7 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC err := client.Query(ctx, &q, vars) if err != nil { - return nil, err + return nil, convertGraphqlError(err) } for _, scene := range q.FindScenes.Scenes { @@ -207,13 +256,41 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC return nil, ErrNotSupported } +type stashVideoFile struct { + Size int64 `graphql:"size" json:"size"` + Duration float64 `graphql:"duration" json:"duration"` + VideoCodec string `graphql:"video_codec" json:"video_codec"` + AudioCodec string `graphql:"audio_codec" json:"audio_codec"` + Width int `graphql:"width" json:"width"` + Height int `graphql:"height" json:"height"` + Framerate float64 `graphql:"frame_rate" json:"frame_rate"` + Bitrate int `graphql:"bit_rate" json:"bit_rate"` +} + +func (f stashVideoFile) SceneFileType() models.SceneFileType { + ret := models.SceneFileType{ + Duration: &f.Duration, + VideoCodec: &f.VideoCodec, + AudioCodec: &f.AudioCodec, + Width: &f.Width, + Height: &f.Height, + Framerate: &f.Framerate, + Bitrate: &f.Bitrate, + } + + size := strconv.FormatInt(f.Size, 10) + ret.Size = &size + + return ret +} + type scrapedSceneStash struct { ID string `graphql:"id" json:"id"` Title *string `graphql:"title" json:"title"` Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` + URLs []string `graphql:"urls" json:"urls"` Date *string `graphql:"date" json:"date"` - File *models.SceneFileType `graphql:"file" json:"file"` + Files []stashVideoFile `graphql:"files" json:"files"` Studio *scrapedStudioStash `graphql:"studio" json:"studio"` Tags []*scrapedTagStash `graphql:"tags" json:"tags"` Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"` @@ -239,12 +316,16 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce } vars := map[string]interface{}{ - "c": &input, + "c": input, } client := s.getStashClient() if err := client.Query(ctx, &q, vars); err != nil { - return nil, err + return nil, convertGraphqlError(err) + } + + if q.FindScene == nil { + return nil, nil } // need to copy back to a scraped scene @@ -254,7 +335,8 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce } // get the performer image directly - ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, s.client, s.globalConfig) + ig := s.imageGetter() + ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig) if err != nil { return nil, err } diff --git a/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md b/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md index caa3d41dc..a0ac30547 100644 --- a/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md +++ b/ui/v2.5/src/docs/en/Manual/ScraperDevelopment.md @@ -247,7 +247,7 @@ sceneByURL: A different stash server can be configured as a scraping source. This action applies only to `performerByName`, `performerByFragment`, and `sceneByFragment` types. This action requires that the top-level `stashServer` field is configured. -`stashServer` contains a single `url` field for the remote stash server. The username and password can be embedded in this string using `username:password@host`. +`stashServer` contains a single `url` field for the remote stash server. The username and password can be embedded in this string using `username:password@host`. Alternatively, the `apiKey` field can be used to authenticate with the remote stash server. An example stash scrape configuration is below: @@ -260,6 +260,7 @@ performerByFragment: sceneByFragment: action: stash stashServer: + apiKey: url: http://stashserver.com:9999 ```