Bypass UnicodeDecodeErrors in Webspider demo

This commit is contained in:
EtiennePelletier 2017-10-24 23:36:58 -04:00
parent 5ee7f4573e
commit 9f75d2d42d
1 changed files with 1 additions and 1 deletions

View File

@ -29,7 +29,7 @@ def get_links_from_url(url):
print('fetched %s' % url)
html = response.body if isinstance(response.body, str) \
else response.body.decode()
else response.body.decode(errors='ignore')
urls = [urljoin(url, remove_fragment(new_url))
for new_url in get_links(html)]
except Exception as e: