Bypass UnicodeDecodeErrors in Webspider demo
This commit is contained in:
parent
5ee7f4573e
commit
9f75d2d42d
|
@ -29,7 +29,7 @@ def get_links_from_url(url):
|
|||
print('fetched %s' % url)
|
||||
|
||||
html = response.body if isinstance(response.body, str) \
|
||||
else response.body.decode()
|
||||
else response.body.decode(errors='ignore')
|
||||
urls = [urljoin(url, remove_fragment(new_url))
|
||||
for new_url in get_links(html)]
|
||||
except Exception as e:
|
||||
|
|
Loading…
Reference in New Issue