Rooba
/
stash
mirror of https://github.com/stashapp/stash.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix scraping error (#704)

This commit is contained in:
WithoutPants 2020-08-04 20:43:56 +10:00 committed by GitHub
parent 4373f9bf01
commit b166abfa7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/scraper/url.go
View File

@ -1,6 +1,7 @@
package scraper package scraper
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@ -65,7 +66,14 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
} }
defer resp.Body.Close() defer resp.Body.Close()
return charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
bodyReader := bytes.NewReader(body)
return charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
} }
// func urlFromCDP uses chrome cdp and DOM to load and process the url // func urlFromCDP uses chrome cdp and DOM to load and process the url