From b166abfa7be103e99171c65da349459d36e0bdf1 Mon Sep 17 00:00:00 2001
From: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
Date: Tue, 4 Aug 2020 20:43:56 +1000
Subject: [PATCH] Fix scraping error (#704)

---
 pkg/scraper/url.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pkg/scraper/url.go b/pkg/scraper/url.go
index 391f1fd46..7e5ad3e4e 100644
--- a/pkg/scraper/url.go
+++ b/pkg/scraper/url.go
@@ -1,6 +1,7 @@
 package scraper
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -65,7 +66,14 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
 	}
 	defer resp.Body.Close()
 
-	return charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	bodyReader := bytes.NewReader(body)
+
+	return charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
 }
 
 // func urlFromCDP uses chrome cdp and DOM to load and process the url