Fix stash scraper errors and add apikey field (#5474)

* Use hasura/go-graphql-client instead of shurcooL version
* Fix graphql query errors
* Support setting api key for stash server
This commit is contained in:
WithoutPants 2024-11-13 10:14:55 +11:00 committed by GitHub
parent 64fed3553a
commit 5f690d96bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 195 additions and 34 deletions

3
go.mod
View File

@ -26,6 +26,7 @@ require (
github.com/gorilla/sessions v1.2.1
github.com/gorilla/websocket v1.5.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hasura/go-graphql-client v0.13.1
github.com/jinzhu/copier v0.4.0
github.com/jmoiron/sqlx v1.4.0
github.com/json-iterator/go v1.1.12
@ -39,7 +40,6 @@ require (
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/remeh/sizedwaitgroup v1.0.0
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cast v1.6.0
github.com/spf13/pflag v1.0.5
@ -67,6 +67,7 @@ require (
github.com/asticode/go-astikit v0.20.0 // indirect
github.com/asticode/go-astits v1.8.0 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coder/websocket v1.8.12 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect

6
go.sum
View File

@ -153,6 +153,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@ -394,6 +396,8 @@ github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoI
github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hasura/go-graphql-client v0.13.1 h1:kKbjhxhpwz58usVl+Xvgah/TDha5K2akNTRQdsEHN6U=
github.com/hasura/go-graphql-client v0.13.1/go.mod h1:k7FF7h53C+hSNFRG3++DdVZWIuHdCaTbI7siTJ//zGQ=
github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs=
github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E=
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
@ -591,8 +595,6 @@ github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDN
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk=
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=

View File

@ -8,7 +8,7 @@ import (
"errors"
"fmt"
"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"
"github.com/stashapp/stash/pkg/plugin/common/log"
)

View File

@ -8,7 +8,7 @@ import (
"net/url"
"strconv"
"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"
"github.com/stashapp/stash/pkg/plugin/common"
)

View File

@ -114,7 +114,8 @@ func (c config) validate() error {
}
type stashServer struct {
URL string `yaml:"url"`
URL string `yaml:"url"`
ApiKey string `yaml:"apiKey"`
}
type scraperTypeConfig struct {

55
pkg/scraper/graphql.go Normal file
View File

@ -0,0 +1,55 @@
package scraper
import (
"errors"
"strings"
"github.com/hasura/go-graphql-client"
)
type graphqlErrors []error
func (e graphqlErrors) Error() string {
b := strings.Builder{}
for _, err := range e {
_, _ = b.WriteString(err.Error())
}
return b.String()
}
type graphqlError struct {
err graphql.Error
}
func (e graphqlError) Error() string {
unwrapped := e.err.Unwrap()
if unwrapped != nil {
var networkErr graphql.NetworkError
if errors.As(unwrapped, &networkErr) {
if networkErr.StatusCode() == 422 {
return networkErr.Body()
}
}
}
return e.err.Error()
}
// convertGraphqlError converts a graphql.Error or graphql.Errors into an error with a useful message.
// graphql.Error swallows important information, so we need to convert it to a more useful error type.
func convertGraphqlError(err error) error {
var gqlErrs graphql.Errors
if errors.As(err, &gqlErrs) {
ret := make(graphqlErrors, len(gqlErrs))
for i, e := range gqlErrs {
ret[i] = convertGraphqlError(e)
}
return ret
}
var gqlErr graphql.Error
if errors.As(err, &gqlErr) {
return graphqlError{gqlErr}
}
return err
}

View File

@ -122,13 +122,19 @@ func setGroupBackImage(ctx context.Context, client *http.Client, m *models.Scrap
return nil
}
func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
type imageGetter struct {
client *http.Client
globalConfig GlobalConfig
requestModifier func(req *http.Request)
}
func (i *imageGetter) getImage(ctx context.Context, url string) (*string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
userAgent := globalConfig.GetScraperUserAgent()
userAgent := i.globalConfig.GetScraperUserAgent()
if userAgent != "" {
req.Header.Set("User-Agent", userAgent)
}
@ -140,7 +146,11 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/")
}
resp, err := client.Do(req)
if i.requestModifier != nil {
i.requestModifier(req)
}
resp, err := i.client.Do(req)
if err != nil {
return nil, err
@ -167,10 +177,19 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
return &img, nil
}
func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/performer/"+performerID+"/image", client, globalConfig)
func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
g := imageGetter{
client: client,
globalConfig: globalConfig,
}
return g.getImage(ctx, url)
}
func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot", client, globalConfig)
func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/performer/"+performerID+"/image")
}
func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot")
}

View File

@ -4,9 +4,11 @@ import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
graphql "github.com/hasura/go-graphql-client"
"github.com/jinzhu/copier"
"github.com/shurcooL/graphql"
"github.com/stashapp/stash/pkg/models"
)
@ -27,9 +29,21 @@ func newStashScraper(scraper scraperTypeConfig, client *http.Client, config conf
}
}
func setApiKeyHeader(apiKey string) func(req *http.Request) {
return func(req *http.Request) {
req.Header.Set("ApiKey", apiKey)
}
}
func (s *stashScraper) getStashClient() *graphql.Client {
url := s.config.StashServer.URL
return graphql.NewClient(url+"/graphql", nil)
url := s.config.StashServer.URL + "/graphql"
ret := graphql.NewClient(url, s.client)
if s.config.StashServer.ApiKey != "" {
ret = ret.WithRequestModifier(setApiKeyHeader(s.config.StashServer.ApiKey))
}
return ret
}
type stashFindPerformerNamePerformer struct {
@ -58,14 +72,12 @@ type scrapedTagStash struct {
type scrapedPerformerStash struct {
Name *string `graphql:"name" json:"name"`
Gender *string `graphql:"gender" json:"gender"`
URL *string `graphql:"url" json:"url"`
Twitter *string `graphql:"twitter" json:"twitter"`
Instagram *string `graphql:"instagram" json:"instagram"`
URLs []string `graphql:"urls" json:"urls"`
Birthdate *string `graphql:"birthdate" json:"birthdate"`
Ethnicity *string `graphql:"ethnicity" json:"ethnicity"`
Country *string `graphql:"country" json:"country"`
EyeColor *string `graphql:"eye_color" json:"eye_color"`
Height *string `graphql:"height" json:"height"`
Height *int `graphql:"height_cm" json:"height_cm"`
Measurements *string `graphql:"measurements" json:"measurements"`
FakeTits *string `graphql:"fake_tits" json:"fake_tits"`
PenisLength *string `graphql:"penis_length" json:"penis_length"`
@ -73,12 +85,25 @@ type scrapedPerformerStash struct {
CareerLength *string `graphql:"career_length" json:"career_length"`
Tattoos *string `graphql:"tattoos" json:"tattoos"`
Piercings *string `graphql:"piercings" json:"piercings"`
Aliases *string `graphql:"aliases" json:"aliases"`
Aliases []string `graphql:"alias_list" json:"alias_list"`
Tags []*scrapedTagStash `graphql:"tags" json:"tags"`
Details *string `graphql:"details" json:"details"`
DeathDate *string `graphql:"death_date" json:"death_date"`
HairColor *string `graphql:"hair_color" json:"hair_color"`
Weight *string `graphql:"weight" json:"weight"`
Weight *int `graphql:"weight" json:"weight"`
}
func (s *stashScraper) imageGetter() imageGetter {
ret := imageGetter{
client: s.client,
globalConfig: s.globalConfig,
}
if s.config.StashServer.ApiKey != "" {
ret.requestModifier = setApiKeyHeader(s.config.StashServer.ApiKey)
}
return ret
}
func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error) {
@ -102,12 +127,12 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
// get the id from the URL field
vars := map[string]interface{}{
"f": performerID,
"f": graphql.ID(performerID),
}
err := client.Query(ctx, &q, vars)
if err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
// need to copy back to a scraped performer
@ -117,11 +142,28 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
return nil, err
}
// convert alias list to aliases
aliasStr := strings.Join(q.FindPerformer.Aliases, ", ")
ret.Aliases = &aliasStr
// convert numeric to string
if q.FindPerformer.Height != nil {
heightStr := strconv.Itoa(*q.FindPerformer.Height)
ret.Height = &heightStr
}
if q.FindPerformer.Weight != nil {
weightStr := strconv.Itoa(*q.FindPerformer.Weight)
ret.Weight = &weightStr
}
// get the performer image directly
ret.Image, err = getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, s.client, s.globalConfig)
ig := s.imageGetter()
img, err := getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, ig)
if err != nil {
return nil, err
}
ret.Images = []string{*img}
ret.Image = img
return &ret, nil
}
@ -143,8 +185,15 @@ func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scen
return nil, err
}
// get the performer image directly
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, s.client, s.globalConfig)
// convert first in files to file
if len(scene.Files) > 0 {
f := scene.Files[0].SceneFileType()
ret.File = &f
}
// get the scene image directly
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, ig)
if err != nil {
return nil, err
}
@ -175,7 +224,7 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
err := client.Query(ctx, &q, vars)
if err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
for _, scene := range q.FindScenes.Scenes {
@ -207,13 +256,41 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
return nil, ErrNotSupported
}
type stashVideoFile struct {
Size int64 `graphql:"size" json:"size"`
Duration float64 `graphql:"duration" json:"duration"`
VideoCodec string `graphql:"video_codec" json:"video_codec"`
AudioCodec string `graphql:"audio_codec" json:"audio_codec"`
Width int `graphql:"width" json:"width"`
Height int `graphql:"height" json:"height"`
Framerate float64 `graphql:"frame_rate" json:"frame_rate"`
Bitrate int `graphql:"bit_rate" json:"bit_rate"`
}
func (f stashVideoFile) SceneFileType() models.SceneFileType {
ret := models.SceneFileType{
Duration: &f.Duration,
VideoCodec: &f.VideoCodec,
AudioCodec: &f.AudioCodec,
Width: &f.Width,
Height: &f.Height,
Framerate: &f.Framerate,
Bitrate: &f.Bitrate,
}
size := strconv.FormatInt(f.Size, 10)
ret.Size = &size
return ret
}
type scrapedSceneStash struct {
ID string `graphql:"id" json:"id"`
Title *string `graphql:"title" json:"title"`
Details *string `graphql:"details" json:"details"`
URL *string `graphql:"url" json:"url"`
URLs []string `graphql:"urls" json:"urls"`
Date *string `graphql:"date" json:"date"`
File *models.SceneFileType `graphql:"file" json:"file"`
Files []stashVideoFile `graphql:"files" json:"files"`
Studio *scrapedStudioStash `graphql:"studio" json:"studio"`
Tags []*scrapedTagStash `graphql:"tags" json:"tags"`
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
@ -239,12 +316,16 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
}
vars := map[string]interface{}{
"c": &input,
"c": input,
}
client := s.getStashClient()
if err := client.Query(ctx, &q, vars); err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
if q.FindScene == nil {
return nil, nil
}
// need to copy back to a scraped scene
@ -254,7 +335,8 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
}
// get the performer image directly
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, s.client, s.globalConfig)
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
if err != nil {
return nil, err
}

View File

@ -247,7 +247,7 @@ sceneByURL:
A different stash server can be configured as a scraping source. This action applies only to `performerByName`, `performerByFragment`, and `sceneByFragment` types. This action requires that the top-level `stashServer` field is configured.
`stashServer` contains a single `url` field for the remote stash server. The username and password can be embedded in this string using `username:password@host`.
`stashServer` contains a single `url` field for the remote stash server. The username and password can be embedded in this string using `username:password@host`. Alternatively, the `apiKey` field can be used to authenticate with the remote stash server.
An example stash scrape configuration is below:
@ -260,6 +260,7 @@ performerByFragment:
sceneByFragment:
action: stash
stashServer:
apiKey: <api key>
url: http://stashserver.com:9999
```