Rooba
/
stash
mirror of https://github.com/stashapp/stash.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix scraping error (#704)

This commit is contained in:
WithoutPants 2020-08-04 20:43:56 +10:00 committed by GitHub
parent 4373f9bf01
commit b166abfa7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/scraper/url.go
View File

@ -1,6 +1,7 @@
package scraper
import (
"bytes"
"context"
"errors"
"fmt"
@ -65,7 +66,14 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
}
defer resp.Body.Close()
return charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
bodyReader := bytes.NewReader(body)
return charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
}
// func urlFromCDP uses chrome cdp and DOM to load and process the url