diff --git a/.appveyor.yml b/.appveyor.yml index ddbfde1f7..5e1b8bc8a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -40,7 +40,7 @@ install: # Install Python modules # https://github.com/tox-dev/tox/issues/791 - "%PYTHON%\\python -m pip install virtualenv --upgrade" - - "%PYTHON%\\python -m pip install tox" + - "%PYTHON%\\python -m pip install tox coverage" # Compatibility run with Winpcap # XXX Remove me when wireshark stops using it as default @@ -57,7 +57,7 @@ for: - choco install -y wireshark # Install Python modules - "%PYTHON%\\python -m pip install virtualenv --upgrade" - - "%PYTHON%\\python -m pip install tox" + - "%PYTHON%\\python -m pip install tox coverage" test_script: # Set environment variables diff --git a/scapy/autorun.py b/scapy/autorun.py index 671f72221..138700822 100644 --- a/scapy/autorun.py +++ b/scapy/autorun.py @@ -9,7 +9,6 @@ Run commands when the Scapy interpreter starts. from __future__ import print_function import code -import importlib import logging import sys import traceback @@ -43,7 +42,8 @@ def autorun_commands(cmds, my_globals=None, verb=None): try: interp = ScapyAutorunInterpreter() if my_globals is None: - my_globals = importlib.import_module(".all", "scapy").__dict__ + from scapy.main import _scapy_builtins + my_globals = _scapy_builtins() interp.locals = my_globals try: del six.moves.builtins.__dict__["scapy_session"]["_"] diff --git a/scapy/config.py b/scapy/config.py index 077537448..522ca5099 100755 --- a/scapy/config.py +++ b/scapy/config.py @@ -272,6 +272,10 @@ class LayersList(List[Type['scapy.packet.Packet']]): result = [] # This import may feel useless, but it is required for the eval below import scapy # noqa: F401 + try: + import builtins # noqa: F401 + except ImportError: + import __builtin__ # noqa: F401 for lay in self.ldict: doc = eval(lay).__doc__ result.append((lay, doc.strip().split("\n")[0] if doc else lay)) diff --git a/scapy/layers/tls/extensions.py b/scapy/layers/tls/extensions.py index 0b1e9fc32..8617b4331 100644 --- a/scapy/layers/tls/extensions.py +++ b/scapy/layers/tls/extensions.py @@ -178,7 +178,7 @@ class ServerName(Packet): class ServerListField(PacketListField): def i2repr(self, pkt, x): res = [p.servername for p in x] - return "[%s]" % b", ".join(res) + return "[%s]" % ", ".join(repr(x) for x in res) class ServerLenField(FieldLenField): @@ -498,7 +498,7 @@ class ProtocolName(Packet): class ProtocolListField(PacketListField): def i2repr(self, pkt, x): res = [p.protocol for p in x] - return "[%s]" % b", ".join(res) + return "[%s]" % ", ".join(repr(x) for x in res) class TLS_Ext_ALPN(TLS_Ext_PrettyPacketList): # RFC 7301 @@ -732,7 +732,7 @@ class _ExtensionsLenField(FieldLenField): """ ext = pkt.get_field(self.length_of) tmp_len = ext.length_from(pkt) - if tmp_len is None: + if tmp_len is None or tmp_len < 0: v = pkt.tls_session.tls_version if v is None or v < 0x0304: return s, None diff --git a/scapy/main.py b/scapy/main.py index d9c5a567e..12313fed1 100644 --- a/scapy/main.py +++ b/scapy/main.py @@ -113,10 +113,10 @@ def _read_config_file(cf, _globals=globals(), _locals=locals(), cf) -def _validate_local(x): +def _validate_local(k): # type: (str) -> bool """Returns whether or not a variable should be imported.""" - return x[0] != "_" + return k[0] != "_" and k not in ["range", "map"] DEFAULT_PRESTART_FILE = _probe_config_file(".scapy_prestart.py") diff --git a/scapy/modules/nmap.py b/scapy/modules/nmap.py index 8394afd51..cbc623bdf 100644 --- a/scapy/modules/nmap.py +++ b/scapy/modules/nmap.py @@ -89,7 +89,7 @@ None. fdesc.close() -nmap_kdb = NmapKnowledgeBase(None) +conf.nmap_kdb = NmapKnowledgeBase(None) def nmap_tcppacket_sig(pkt): @@ -181,7 +181,7 @@ def nmap_probes2sig(tests): def nmap_search(sigs): guess = 0, [] - for osval, fprint in nmap_kdb.get_base(): + for osval, fprint in conf.nmap_kdb.get_base(): score = 0.0 for test, values in six.iteritems(fprint): if test in sigs: diff --git a/scapy/tools/UTscapy.py b/scapy/tools/UTscapy.py index b78991087..4fde0986d 100644 --- a/scapy/tools/UTscapy.py +++ b/scapy/tools/UTscapy.py @@ -69,18 +69,17 @@ class Bunch: def retry_test(func): """Retries the passed function 3 times before failing""" success = False - ex = Exception("Unknown") for _ in six.moves.range(3): try: result = func() - except Exception as e: + except Exception: + t, v, tb = sys.exc_info() time.sleep(1) - ex = e else: success = True break if not success: - raise ex + six.reraise(t, v, tb) assert success return result @@ -1143,7 +1142,8 @@ def main(): runned_campaigns = [] - scapy_ses = importlib.import_module(".all", "scapy").__dict__ + from scapy.main import _scapy_builtins + scapy_ses = _scapy_builtins() import_UTscapy_tools(scapy_ses) # Execute all files @@ -1205,6 +1205,11 @@ def main(): if threading.active_count() > 1: print("\nWARNING: UNFINISHED THREADS") print(threading.enumerate()) + import multiprocessing + processes = multiprocessing.active_children() + if processes: + print("\nWARNING: UNFINISHED PROCESSES") + print(processes) # Return state return glob_result diff --git a/test/fields.uts b/test/fields.uts index 5831ea07c..06381d5ab 100644 --- a/test/fields.uts +++ b/test/fields.uts @@ -101,7 +101,7 @@ p = Ether() / ARP(pdst="1.2.3.4") assert p.src == p.hwsrc == p[ARP].hwsrc == get_if_hwaddr(conf.iface) p = Dot3() / LLC() / SNAP() / ARP(pdst="1.2.3.4") assert p.src == p.hwsrc == p[ARP].hwsrc == get_if_hwaddr(conf.iface) -conf.route.delt(net="1.2.3.4/32") +conf.route.delt(net="1.2.3.4/32", dev=conf.iface) = IPField class ~ core field diff --git a/test/imports.uts b/test/imports.uts index 8c7e02a89..ba3e302da 100644 --- a/test/imports.uts +++ b/test/imports.uts @@ -5,6 +5,7 @@ = Prepare importing all scapy files +import os import glob import subprocess @@ -18,6 +19,10 @@ EXCEPTIONS = [ "scapy.contrib.scada*", "scapy.main", ] + +if WINDOWS: + EXCEPTIONS.append("scapy.layers.tuntap") + EXCEPTION_PACKAGES = [ "arch", "libs", @@ -26,7 +31,7 @@ EXCEPTION_PACKAGES = [ ] ALL_FILES = [ - "scapy." + re.match(r".*scapy\/(.*)\.py$", x).group(1).replace("/", ".") + "scapy." + re.match(".*scapy\\" + os.path.sep + "(.*)\\.py$", x).group(1).replace(os.path.sep, ".") for x in glob.iglob(scapy_path('/scapy/**/*.py'), recursive=True) ] ALL_FILES = [ @@ -49,10 +54,10 @@ def process_file(file): encoding="utf8") errs = "" try: - _, errs = proc.communicate(timeout=10) + _, errs = proc.communicate(timeout=30) except subprocess.TimeoutExpired: proc.kill() - errs = "Timed out !" + errs = "Timed out (>30s)!" if proc.returncode != 0: return "Importing the file '%s' failed !\\n%s" % (file, errs) return None @@ -68,8 +73,8 @@ sys.path.append(fld) pkg = importlib.import_module(os.path.splitext(file)[0]) def import_all(FILES): - with Pool(processes=8) as pool: - for err in pool.imap_unordered(pkg.process_file, FILES, 4): + with Pool(processes=4) as pool: + for err in pool.imap_unordered(pkg.process_file, FILES): if err: print(err) pool.terminate() diff --git a/test/nmap.uts b/test/nmap.uts index 45cb0faee..aca6bbb8c 100644 --- a/test/nmap.uts +++ b/test/nmap.uts @@ -26,19 +26,19 @@ try: except ImportError: from urllib2 import urlopen -for i in range(10): - try: - open('nmap-os-fingerprints', 'wb').write(urlopen('https://raw.githubusercontent.com/nmap/nmap/9efe1892/nmap-os-fingerprints').read()) - break - except: - pass +def _test(): + with open('nmap-os-fingerprints', 'wb') as fd: + fd.write(urlopen('https://raw.githubusercontent.com/nmap/nmap/9efe1892/nmap-os-fingerprints').read()) + +retry_test(_test) conf.nmap_base = 'nmap-os-fingerprints' = Database loading ~ netaccess -assert len(nmap_kdb.get_base()) > 100 +print(conf.nmap_kdb.base, conf.nmap_kdb.filename, len(conf.nmap_kdb.get_base())) +assert len(conf.nmap_kdb.get_base()) > 100 = fingerprint test: www.secdev.org ~ netaccess @@ -75,9 +75,9 @@ assert len(a["PU"]) > 0 = Nmap base not available -nmap_kdb.filename = "invalid" -nmap_kdb.reload() -assert nmap_kdb.filename == None +conf.nmap_kdb.filename = "invalid" +conf.nmap_kdb.reload() +assert conf.nmap_kdb.filename == None = Clear temp files try: diff --git a/test/pipetool.uts b/test/pipetool.uts index 2d4f1bb73..32dde758d 100644 --- a/test/pipetool.uts +++ b/test/pipetool.uts @@ -653,12 +653,14 @@ p.add(s) p.add(s2) p.start() +pkt = DNS() + s.send(IP(src="127.0.0.1")/UDP()/DNS()) -s2.send(DNS()) +s2.send(pkt) res = [c.q.get(timeout=2), c.q.get(timeout=2)] -assert b'\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00' in res -res.remove(b'\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00') +assert raw(pkt) in res +res.remove(raw(pkt)) assert DNS in res[0] and res[0][UDP].sport == 1234 p.stop() diff --git a/test/regression.uts b/test/regression.uts index fde46bb57..6106d1fb9 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -1337,15 +1337,7 @@ raw(RandASN1Object()) # RSN Information that isn't parsed properly, # causing the SSID to be overridden. # This test checks the SSID is parsed properly. -print(os.path.dirname(__file__)) -filename = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - "..", - "test", - "pcaps", - "bad_rsn_parsing_overrides_ssid.pcap" - )) +filename = scapy_path("/test/pcaps/bad_rsn_parsing_overrides_ssid.pcap") frame = rdpcap(filename)[0] beacon = frame.getlayer(5) ssid = beacon.network_stats()['ssid'] @@ -1453,7 +1445,7 @@ def _test_flood(flood_function, add_ether=False): p = IP(dst="www.google.com")/TCP(sport=RandShort(), dport=80, flags="S") if add_ether: p = Ether()/p - x = flood_function(p, timeout=2) + x = flood_function(p, timeout=0.5) conf.debug_dissector = old_debug_dissector if type(x) == tuple: x = x[0][0][1] @@ -1674,60 +1666,17 @@ s.show() s.show(2) = send() and sniff() -~ netaccess tcpdump -import time -import os +~ netaccess ss -from scapy.modules.six.moves.queue import Queue +def _test(): + sendp(Ether()/IP(src="9.0.0.0")/UDP(), count=3, iface=conf.iface) -def _send_or_sniff(pkt, timeout, flt, pid, fork, t_other=None, opened_socket=None): - assert pid != -1 - if pid == 0: - time.sleep(1) - (sendp if isinstance(pkt, (Ether, Dot3)) else send)(pkt) - if fork: - os._exit(0) - else: - return - else: - spkt = raw(pkt) - # We do not want to crash when a packet cannot be parsed - old_debug_dissector = conf.debug_dissector - conf.debug_dissector = False - pkts = sniff( - timeout=timeout, filter=flt, opened_socket=opened_socket, - stop_filter=lambda p: pkt.__class__ in p and raw(p[pkt.__class__]) == spkt - ) - conf.debug_dissector = old_debug_dissector - if fork: - os.waitpid(pid, 0) - else: - t_other.join(timeout=3) - assert raw(pkt) in (raw(p[pkt.__class__]) for p in pkts if pkt.__class__ in p) +r = sniff(timeout=3, count=1, + lfilter=lambda x: x[IP].src == "9.0.0.0", + iface=conf.iface, + started_callback=_test) -def send_and_sniff(pkt, timeout=2, flt=None, opened_socket=None): - """Send a packet, sniff, and check the packet has been seen""" - if hasattr(os, "fork"): - _send_or_sniff(pkt, timeout, flt, os.fork(), True) - else: - from threading import Thread - def run_function(pkt, timeout, flt, pid, thread, results, opened_socket): - _send_or_sniff(pkt, timeout, flt, pid, False, t_other=thread, opened_socket=opened_socket) - results.put(True) - results = Queue() - t_parent = Thread(target=run_function, args=(pkt, timeout, flt, 0, None, results, None), name="send_and_sniff 1") - t_child = Thread(target=run_function, args=(pkt, timeout, flt, 1, t_parent, results, opened_socket), name="send_and_sniff 2") - t_parent.start() - t_child.start() - t_parent.join(timeout=3) - t_child.join(timeout=3) - assert results.qsize() >= 2 - while not results.empty(): - assert results.get() - -retry_test(lambda: send_and_sniff(IP(dst="secdev.org")/ICMP())) -retry_test(lambda: send_and_sniff(IP(dst="secdev.org")/ICMP(), flt="icmp")) -retry_test(lambda: send_and_sniff(Ether()/IP(dst="secdev.org")/ICMP())) +assert r = Test SuperSocket.select ~ select diff --git a/test/scapy/layers/dns.uts b/test/scapy/layers/dns.uts index cc9ab93ab..94a8ce333 100644 --- a/test/scapy/layers/dns.uts +++ b/test/scapy/layers/dns.uts @@ -83,7 +83,7 @@ assert pkt.an.getlayer(DNSRR, type=16).rdata == [b'txtvers=1', b'vs=190.9', b'ch = DNS advanced building ~ dns -pkt = DNS(qr=1, aa=1, rd=1) +pkt = DNS(qr=1, qd=None, aa=1, rd=1) pkt.an = DNSRR(type=12, rrname='_raop._tcp.local.', rdata='140C768FFE28@Freebox Server._raop._tcp.local.')/DNSRR(rrname='140C768FFE28@Freebox Server._raop._tcp.local.', type=16, rdata=[b'txtvers=1', b'vs=190.9', b'ch=2', b'sr=44100', b'ss=16', b'pw=false', b'et=0,1', b'ek=1', b'tp=TCP,UDP', b'am=FreeboxServer1,2', b'cn=0,1,2,3', b'md=0,2', b'sf=0x44', b'ft=0xBF0A00', b'sv=false', b'da=true', b'vn=65537', b'vv=2'])/DNSRRSRV(rrname='140C768FFE28@Freebox Server._raop._tcp.local.', target='Freebox-Server-3.local.', port=5000, type=33, rclass=32769)/DNSRR(rrname='Freebox-Server-3.local.', rdata='192.168.0.254', rclass=32769, type=1, ttl=120) pkt = DNS(raw(pkt)) @@ -160,10 +160,10 @@ DNS(s) = DNS record type 16 (TXT) -p = DNS(raw(DNS(id=1,ra=1,an=DNSRR(rrname='scapy', type='TXT', rdata="niceday", ttl=1)))) +p = DNS(raw(DNS(id=1,ra=1,qd=None,an=DNSRR(rrname='scapy', type='TXT', rdata="niceday", ttl=1)))) assert p[DNS].an.rdata == [b"niceday"] -p = DNS(raw(DNS(id=1,ra=1,an=DNSRR(rrname='secdev', type='TXT', rdata=["sweet", "celestia"], ttl=1)))) +p = DNS(raw(DNS(id=1,ra=1,qd=None,an=DNSRR(rrname='secdev', type='TXT', rdata=["sweet", "celestia"], ttl=1)))) assert p[DNS].an.rdata == [b"sweet", b"celestia"] assert raw(p) == b'\x00\x01\x01\x80\x00\x00\x00\x01\x00\x00\x00\x00\x06secdev\x00\x00\x10\x00\x01\x00\x00\x00\x01\x00\x0f\x05sweet\x08celestia' @@ -173,13 +173,13 @@ _old_dbg = conf.debug_dissector conf.debug_dissector = True try: - p = IP(raw(IP()/TCP()/DNS(length=28))[:-13]) + p = IP(raw(IP()/TCP()/DNS(qd=None,length=28))[:-13]) assert False except Scapy_Exception as e: assert str(e) == "Malformed DNS message: too small!" try: - p = IP(raw(IP()/TCP()/DNS(length=28, qdcount=1))) + p = IP(raw(IP()/TCP()/DNS(qd=None,length=28, qdcount=1))) assert False except Scapy_Exception as e: assert str(e) == "Malformed DNS message: invalid length!" diff --git a/test/tls.uts b/test/tls.uts index ff7e84cd4..eaeb2026a 100644 --- a/test/tls.uts +++ b/test/tls.uts @@ -1526,6 +1526,12 @@ r1 = TLS(b"\x16\x03\x01\x02\x00\x01\x00\x01\xfc\x03\x03\xf8\xb3\xdb\xcbp\xed8\x0 r2 = TLS(b'\x16\x03\x03\x00U\x02\x00\x00Q\x03\x03 \xa5@2G~\xa3\xa9c\xb8\xa7\x00\t\x04Y\xf1\x1f\x1fJ\xd1\x89n\x1dut[~+\xdcQ\xdd\xe0 \x06\x00\xf5R\xdblQ\xb9z0\x97\x17\xff\x84{\xb6\xe8\xfe\xf1\xce&\x01TD\x13\xfd\xa7\xb6`u\xb8\x87\x00\x9d\x00\x00\t\xff\x01\x00\x01\x00\x00\x17\x00\x00\x16\x03\x03\x03n\x0b\x00\x03j\x00\x03g\x00\x03d0\x82\x03`0\x82\x02H\xa0\x03\x02\x01\x02\x02\t\x00\xebs\xb7\x1c>/\x9f\xdc0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x0b\x05\x000E1\x0b0\t\x06\x03U\x04\x06\x13\x02AU1\x130\x11\x06\x03U\x04\x08\x0c\nSome-State1!0\x1f\x06\x03U\x04\n\x0c\x18Internet Widgits Pty Ltd0\x1e\x17\r190215151403Z\x17\r290212151403Z0E1\x0b0\t\x06\x03U\x04\x06\x13\x02AU1\x130\x11\x06\x03U\x04\x08\x0c\nSome-State1!0\x1f\x06\x03U\x04\n\x0c\x18Internet Widgits Pty Ltd0\x82\x01"0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x01\x05\x00\x03\x82\x01\x0f\x000\x82\x01\n\x02\x82\x01\x01\x00\xd2\xf7\xd3k#:V\x196\x8f\xc3\xa7\xdb\x0f#d\xdcq\x98m\xd4\xee\xbc\xbe\xe8[x>\x13\x9c\xfe\xb0\xa8\r\xe5\x01G\xc96\xaa\x84#\x0e/\xa2\xeb\x91\xef\x177A\x03\x87\xb92D\n\xc7\xcf\xda\xff~\xca,yMq<\x13\xf8\x0c\xd5?\x84z\xa1\x96\xd0\xad\xc0D\x94y\nb\x8e2\x7fKS\xd0[\x83\x02\\>\xa5A\x19_\x95<\xe6\xfc7\xed\xcch\xa8\xfdn\xcab\x1f8\xbc\x08\xbc-\x8dr\xcf\xcd\xf8\\h\xf9\xf4\xf4H[2\x13zh_ <\r\xb8\xe0\xff\x1d\x1aY\x91\xd2\xf0X\xf4\x8f \xb1\n_\xb0\xdf\'\xa1\xf9\x87L\xc0\xfe\x8dn\xbfw\xe9\xa7\xba8I\x0e\x9dc$\x1a\x0f\xb3\xfdw\x01\xff;\x13\x0c\x9a\xa7\xaaww\x02\x80\xb7\x00<\x1b\xb5\xe0xL4\xaa\xcbt\xce\x81\x14\x96\x0eP\xee\xe0F\x02\xa7\xab \xe5\xc8x\x02\x8eB\x92\xe9\x0e@\xfdc\x1f\xee\x16\x03\x03\x00\x04\x0e\x00\x00\x00', tls_session=r1.tls_session.mirror()) assert r2.tls_session.tls_version == 0x303 += PR/Issue 3295 + +pkt = TLSServerHello(b"\x02\x00\x00\x28\x03\x03ABCDEFGHIJKLMNOPQRSTUVWXYZ012345\x00\x00\x39\x00\x00\x00") +assert pkt.extlen == 0 +assert pkt.ext is None + ############################################################################### ############################ Automaton behaviour ############################## ###############################################################################