Repair unit tests (#3303)

* Fix TLS extensions

* Fix minor DNS test failure

* Add qd=None to some DNS tests

* Increase import timeout

* Fix import.uts on Windows

* Stabilize nmap test

* Improve loading of nmap os fingerprints

* Use reraise in retry_test

* Disable some imports

* Re-use _scapy_builtins

* Do not use __file__ in regression.uts

* Fix appveyor build

* Simplify send() and sniff() by a lot...

* Add test for issue 3295

* Reduce flooding delay

* Show unfinished processes
This commit is contained in:
gpotter2 2021-07-21 14:27:09 +02:00 committed by GitHub
parent 556d2a0a60
commit cbb147daa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 72 additions and 101 deletions

View File

@ -40,7 +40,7 @@ install:
# Install Python modules
# https://github.com/tox-dev/tox/issues/791
- "%PYTHON%\\python -m pip install virtualenv --upgrade"
- "%PYTHON%\\python -m pip install tox"
- "%PYTHON%\\python -m pip install tox coverage"
# Compatibility run with Winpcap
# XXX Remove me when wireshark stops using it as default
@ -57,7 +57,7 @@ for:
- choco install -y wireshark
# Install Python modules
- "%PYTHON%\\python -m pip install virtualenv --upgrade"
- "%PYTHON%\\python -m pip install tox"
- "%PYTHON%\\python -m pip install tox coverage"
test_script:
# Set environment variables

View File

@ -9,7 +9,6 @@ Run commands when the Scapy interpreter starts.
from __future__ import print_function
import code
import importlib
import logging
import sys
import traceback
@ -43,7 +42,8 @@ def autorun_commands(cmds, my_globals=None, verb=None):
try:
interp = ScapyAutorunInterpreter()
if my_globals is None:
my_globals = importlib.import_module(".all", "scapy").__dict__
from scapy.main import _scapy_builtins
my_globals = _scapy_builtins()
interp.locals = my_globals
try:
del six.moves.builtins.__dict__["scapy_session"]["_"]

View File

@ -272,6 +272,10 @@ class LayersList(List[Type['scapy.packet.Packet']]):
result = []
# This import may feel useless, but it is required for the eval below
import scapy # noqa: F401
try:
import builtins # noqa: F401
except ImportError:
import __builtin__ # noqa: F401
for lay in self.ldict:
doc = eval(lay).__doc__
result.append((lay, doc.strip().split("\n")[0] if doc else lay))

View File

@ -178,7 +178,7 @@ class ServerName(Packet):
class ServerListField(PacketListField):
def i2repr(self, pkt, x):
res = [p.servername for p in x]
return "[%s]" % b", ".join(res)
return "[%s]" % ", ".join(repr(x) for x in res)
class ServerLenField(FieldLenField):
@ -498,7 +498,7 @@ class ProtocolName(Packet):
class ProtocolListField(PacketListField):
def i2repr(self, pkt, x):
res = [p.protocol for p in x]
return "[%s]" % b", ".join(res)
return "[%s]" % ", ".join(repr(x) for x in res)
class TLS_Ext_ALPN(TLS_Ext_PrettyPacketList): # RFC 7301
@ -732,7 +732,7 @@ class _ExtensionsLenField(FieldLenField):
"""
ext = pkt.get_field(self.length_of)
tmp_len = ext.length_from(pkt)
if tmp_len is None:
if tmp_len is None or tmp_len < 0:
v = pkt.tls_session.tls_version
if v is None or v < 0x0304:
return s, None

View File

@ -113,10 +113,10 @@ def _read_config_file(cf, _globals=globals(), _locals=locals(),
cf)
def _validate_local(x):
def _validate_local(k):
# type: (str) -> bool
"""Returns whether or not a variable should be imported."""
return x[0] != "_"
return k[0] != "_" and k not in ["range", "map"]
DEFAULT_PRESTART_FILE = _probe_config_file(".scapy_prestart.py")

View File

@ -89,7 +89,7 @@ None.
fdesc.close()
nmap_kdb = NmapKnowledgeBase(None)
conf.nmap_kdb = NmapKnowledgeBase(None)
def nmap_tcppacket_sig(pkt):
@ -181,7 +181,7 @@ def nmap_probes2sig(tests):
def nmap_search(sigs):
guess = 0, []
for osval, fprint in nmap_kdb.get_base():
for osval, fprint in conf.nmap_kdb.get_base():
score = 0.0
for test, values in six.iteritems(fprint):
if test in sigs:

View File

@ -69,18 +69,17 @@ class Bunch:
def retry_test(func):
"""Retries the passed function 3 times before failing"""
success = False
ex = Exception("Unknown")
for _ in six.moves.range(3):
try:
result = func()
except Exception as e:
except Exception:
t, v, tb = sys.exc_info()
time.sleep(1)
ex = e
else:
success = True
break
if not success:
raise ex
six.reraise(t, v, tb)
assert success
return result
@ -1143,7 +1142,8 @@ def main():
runned_campaigns = []
scapy_ses = importlib.import_module(".all", "scapy").__dict__
from scapy.main import _scapy_builtins
scapy_ses = _scapy_builtins()
import_UTscapy_tools(scapy_ses)
# Execute all files
@ -1205,6 +1205,11 @@ def main():
if threading.active_count() > 1:
print("\nWARNING: UNFINISHED THREADS")
print(threading.enumerate())
import multiprocessing
processes = multiprocessing.active_children()
if processes:
print("\nWARNING: UNFINISHED PROCESSES")
print(processes)
# Return state
return glob_result

View File

@ -101,7 +101,7 @@ p = Ether() / ARP(pdst="1.2.3.4")
assert p.src == p.hwsrc == p[ARP].hwsrc == get_if_hwaddr(conf.iface)
p = Dot3() / LLC() / SNAP() / ARP(pdst="1.2.3.4")
assert p.src == p.hwsrc == p[ARP].hwsrc == get_if_hwaddr(conf.iface)
conf.route.delt(net="1.2.3.4/32")
conf.route.delt(net="1.2.3.4/32", dev=conf.iface)
= IPField class
~ core field

View File

@ -5,6 +5,7 @@
= Prepare importing all scapy files
import os
import glob
import subprocess
@ -18,6 +19,10 @@ EXCEPTIONS = [
"scapy.contrib.scada*",
"scapy.main",
]
if WINDOWS:
EXCEPTIONS.append("scapy.layers.tuntap")
EXCEPTION_PACKAGES = [
"arch",
"libs",
@ -26,7 +31,7 @@ EXCEPTION_PACKAGES = [
]
ALL_FILES = [
"scapy." + re.match(r".*scapy\/(.*)\.py$", x).group(1).replace("/", ".")
"scapy." + re.match(".*scapy\\" + os.path.sep + "(.*)\\.py$", x).group(1).replace(os.path.sep, ".")
for x in glob.iglob(scapy_path('/scapy/**/*.py'), recursive=True)
]
ALL_FILES = [
@ -49,10 +54,10 @@ def process_file(file):
encoding="utf8")
errs = ""
try:
_, errs = proc.communicate(timeout=10)
_, errs = proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
proc.kill()
errs = "Timed out !"
errs = "Timed out (>30s)!"
if proc.returncode != 0:
return "Importing the file '%s' failed !\\n%s" % (file, errs)
return None
@ -68,8 +73,8 @@ sys.path.append(fld)
pkg = importlib.import_module(os.path.splitext(file)[0])
def import_all(FILES):
with Pool(processes=8) as pool:
for err in pool.imap_unordered(pkg.process_file, FILES, 4):
with Pool(processes=4) as pool:
for err in pool.imap_unordered(pkg.process_file, FILES):
if err:
print(err)
pool.terminate()

View File

@ -26,19 +26,19 @@ try:
except ImportError:
from urllib2 import urlopen
for i in range(10):
try:
open('nmap-os-fingerprints', 'wb').write(urlopen('https://raw.githubusercontent.com/nmap/nmap/9efe1892/nmap-os-fingerprints').read())
break
except:
pass
def _test():
with open('nmap-os-fingerprints', 'wb') as fd:
fd.write(urlopen('https://raw.githubusercontent.com/nmap/nmap/9efe1892/nmap-os-fingerprints').read())
retry_test(_test)
conf.nmap_base = 'nmap-os-fingerprints'
= Database loading
~ netaccess
assert len(nmap_kdb.get_base()) > 100
print(conf.nmap_kdb.base, conf.nmap_kdb.filename, len(conf.nmap_kdb.get_base()))
assert len(conf.nmap_kdb.get_base()) > 100
= fingerprint test: www.secdev.org
~ netaccess
@ -75,9 +75,9 @@ assert len(a["PU"]) > 0
= Nmap base not available
nmap_kdb.filename = "invalid"
nmap_kdb.reload()
assert nmap_kdb.filename == None
conf.nmap_kdb.filename = "invalid"
conf.nmap_kdb.reload()
assert conf.nmap_kdb.filename == None
= Clear temp files
try:

View File

@ -653,12 +653,14 @@ p.add(s)
p.add(s2)
p.start()
pkt = DNS()
s.send(IP(src="127.0.0.1")/UDP()/DNS())
s2.send(DNS())
s2.send(pkt)
res = [c.q.get(timeout=2), c.q.get(timeout=2)]
assert b'\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00' in res
res.remove(b'\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
assert raw(pkt) in res
res.remove(raw(pkt))
assert DNS in res[0] and res[0][UDP].sport == 1234
p.stop()

View File

@ -1337,15 +1337,7 @@ raw(RandASN1Object())
# RSN Information that isn't parsed properly,
# causing the SSID to be overridden.
# This test checks the SSID is parsed properly.
print(os.path.dirname(__file__))
filename = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"..",
"test",
"pcaps",
"bad_rsn_parsing_overrides_ssid.pcap"
))
filename = scapy_path("/test/pcaps/bad_rsn_parsing_overrides_ssid.pcap")
frame = rdpcap(filename)[0]
beacon = frame.getlayer(5)
ssid = beacon.network_stats()['ssid']
@ -1453,7 +1445,7 @@ def _test_flood(flood_function, add_ether=False):
p = IP(dst="www.google.com")/TCP(sport=RandShort(), dport=80, flags="S")
if add_ether:
p = Ether()/p
x = flood_function(p, timeout=2)
x = flood_function(p, timeout=0.5)
conf.debug_dissector = old_debug_dissector
if type(x) == tuple:
x = x[0][0][1]
@ -1674,60 +1666,17 @@ s.show()
s.show(2)
= send() and sniff()
~ netaccess tcpdump
import time
import os
~ netaccess ss
from scapy.modules.six.moves.queue import Queue
def _test():
sendp(Ether()/IP(src="9.0.0.0")/UDP(), count=3, iface=conf.iface)
def _send_or_sniff(pkt, timeout, flt, pid, fork, t_other=None, opened_socket=None):
assert pid != -1
if pid == 0:
time.sleep(1)
(sendp if isinstance(pkt, (Ether, Dot3)) else send)(pkt)
if fork:
os._exit(0)
else:
return
else:
spkt = raw(pkt)
# We do not want to crash when a packet cannot be parsed
old_debug_dissector = conf.debug_dissector
conf.debug_dissector = False
pkts = sniff(
timeout=timeout, filter=flt, opened_socket=opened_socket,
stop_filter=lambda p: pkt.__class__ in p and raw(p[pkt.__class__]) == spkt
)
conf.debug_dissector = old_debug_dissector
if fork:
os.waitpid(pid, 0)
else:
t_other.join(timeout=3)
assert raw(pkt) in (raw(p[pkt.__class__]) for p in pkts if pkt.__class__ in p)
r = sniff(timeout=3, count=1,
lfilter=lambda x: x[IP].src == "9.0.0.0",
iface=conf.iface,
started_callback=_test)
def send_and_sniff(pkt, timeout=2, flt=None, opened_socket=None):
"""Send a packet, sniff, and check the packet has been seen"""
if hasattr(os, "fork"):
_send_or_sniff(pkt, timeout, flt, os.fork(), True)
else:
from threading import Thread
def run_function(pkt, timeout, flt, pid, thread, results, opened_socket):
_send_or_sniff(pkt, timeout, flt, pid, False, t_other=thread, opened_socket=opened_socket)
results.put(True)
results = Queue()
t_parent = Thread(target=run_function, args=(pkt, timeout, flt, 0, None, results, None), name="send_and_sniff 1")
t_child = Thread(target=run_function, args=(pkt, timeout, flt, 1, t_parent, results, opened_socket), name="send_and_sniff 2")
t_parent.start()
t_child.start()
t_parent.join(timeout=3)
t_child.join(timeout=3)
assert results.qsize() >= 2
while not results.empty():
assert results.get()
retry_test(lambda: send_and_sniff(IP(dst="secdev.org")/ICMP()))
retry_test(lambda: send_and_sniff(IP(dst="secdev.org")/ICMP(), flt="icmp"))
retry_test(lambda: send_and_sniff(Ether()/IP(dst="secdev.org")/ICMP()))
assert r
= Test SuperSocket.select
~ select

View File

@ -83,7 +83,7 @@ assert pkt.an.getlayer(DNSRR, type=16).rdata == [b'txtvers=1', b'vs=190.9', b'ch
= DNS advanced building
~ dns
pkt = DNS(qr=1, aa=1, rd=1)
pkt = DNS(qr=1, qd=None, aa=1, rd=1)
pkt.an = DNSRR(type=12, rrname='_raop._tcp.local.', rdata='140C768FFE28@Freebox Server._raop._tcp.local.')/DNSRR(rrname='140C768FFE28@Freebox Server._raop._tcp.local.', type=16, rdata=[b'txtvers=1', b'vs=190.9', b'ch=2', b'sr=44100', b'ss=16', b'pw=false', b'et=0,1', b'ek=1', b'tp=TCP,UDP', b'am=FreeboxServer1,2', b'cn=0,1,2,3', b'md=0,2', b'sf=0x44', b'ft=0xBF0A00', b'sv=false', b'da=true', b'vn=65537', b'vv=2'])/DNSRRSRV(rrname='140C768FFE28@Freebox Server._raop._tcp.local.', target='Freebox-Server-3.local.', port=5000, type=33, rclass=32769)/DNSRR(rrname='Freebox-Server-3.local.', rdata='192.168.0.254', rclass=32769, type=1, ttl=120)
pkt = DNS(raw(pkt))
@ -160,10 +160,10 @@ DNS(s)
= DNS record type 16 (TXT)
p = DNS(raw(DNS(id=1,ra=1,an=DNSRR(rrname='scapy', type='TXT', rdata="niceday", ttl=1))))
p = DNS(raw(DNS(id=1,ra=1,qd=None,an=DNSRR(rrname='scapy', type='TXT', rdata="niceday", ttl=1))))
assert p[DNS].an.rdata == [b"niceday"]
p = DNS(raw(DNS(id=1,ra=1,an=DNSRR(rrname='secdev', type='TXT', rdata=["sweet", "celestia"], ttl=1))))
p = DNS(raw(DNS(id=1,ra=1,qd=None,an=DNSRR(rrname='secdev', type='TXT', rdata=["sweet", "celestia"], ttl=1))))
assert p[DNS].an.rdata == [b"sweet", b"celestia"]
assert raw(p) == b'\x00\x01\x01\x80\x00\x00\x00\x01\x00\x00\x00\x00\x06secdev\x00\x00\x10\x00\x01\x00\x00\x00\x01\x00\x0f\x05sweet\x08celestia'
@ -173,13 +173,13 @@ _old_dbg = conf.debug_dissector
conf.debug_dissector = True
try:
p = IP(raw(IP()/TCP()/DNS(length=28))[:-13])
p = IP(raw(IP()/TCP()/DNS(qd=None,length=28))[:-13])
assert False
except Scapy_Exception as e:
assert str(e) == "Malformed DNS message: too small!"
try:
p = IP(raw(IP()/TCP()/DNS(length=28, qdcount=1)))
p = IP(raw(IP()/TCP()/DNS(qd=None,length=28, qdcount=1)))
assert False
except Scapy_Exception as e:
assert str(e) == "Malformed DNS message: invalid length!"

View File

@ -1526,6 +1526,12 @@ r1 = TLS(b"\x16\x03\x01\x02\x00\x01\x00\x01\xfc\x03\x03\xf8\xb3\xdb\xcbp\xed8\x0
r2 = TLS(b'\x16\x03\x03\x00U\x02\x00\x00Q\x03\x03 \xa5@2G~\xa3\xa9c\xb8\xa7\x00\t\x04Y\xf1\x1f\x1fJ\xd1\x89n\x1dut[~+\xdcQ\xdd\xe0 \x06\x00\xf5R\xdblQ\xb9z0\x97\x17\xff\x84{\xb6\xe8\xfe\xf1\xce&\x01TD\x13\xfd\xa7\xb6`u\xb8\x87\x00\x9d\x00\x00\t\xff\x01\x00\x01\x00\x00\x17\x00\x00\x16\x03\x03\x03n\x0b\x00\x03j\x00\x03g\x00\x03d0\x82\x03`0\x82\x02H\xa0\x03\x02\x01\x02\x02\t\x00\xebs\xb7\x1c>/\x9f\xdc0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x0b\x05\x000E1\x0b0\t\x06\x03U\x04\x06\x13\x02AU1\x130\x11\x06\x03U\x04\x08\x0c\nSome-State1!0\x1f\x06\x03U\x04\n\x0c\x18Internet Widgits Pty Ltd0\x1e\x17\r190215151403Z\x17\r290212151403Z0E1\x0b0\t\x06\x03U\x04\x06\x13\x02AU1\x130\x11\x06\x03U\x04\x08\x0c\nSome-State1!0\x1f\x06\x03U\x04\n\x0c\x18Internet Widgits Pty Ltd0\x82\x01"0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x01\x05\x00\x03\x82\x01\x0f\x000\x82\x01\n\x02\x82\x01\x01\x00\xd2\xf7\xd3k#:V\x196\x8f\xc3\xa7\xdb\x0f#d\xdcq\x98m\xd4\xee\xbc\xbe\xe8[x>\x13\x9c\xfe\xb0\xa8\r\xe5\x01G\xc96\xaa\x84#\x0e/\xa2\xeb\x91\xef\x177A\x03\x87\xb92D\n\xc7\xcf\xda<Yf\xee\xf8\x8fh\x8b\xcb\xeej]\r\xfa\xa0u\xb7}\xfe\x83o,\xb3\x187_\xf5\xbe+5\xf6\xd6-\xd7\xc4\xb1G"Og\xd5=\x95\xc7\xfc\xc1\x1c\xda{\xc6"6\x9bMJhVU\x16\x9f\xb2\x8ff\xe5\x11rO\x0c\x9a\xf2\xf7N\xa4\xcd\xf0\x9b\x92\xf9\x17$jX/g\xfa\xde>\xff~\xca,yMq<\x13\xf8\x0c\xd5?\x84z\xa1\x96\xd0\xad\xc0D\x94y\nb\x8e2\x7fKS\xd0[\x83\x02\\>\xa5A\x19_\x95<\xe6\xfc7\xed\xcch\xa8\xfdn\xcab\x1f8\xbc\x08\xbc-\x8dr\xcf\xcd\xf8\\h\xf9\xf4\xf4H[2\x13<c)\x9f\x85\xff\xd6+\xdeZ\x9dX^Z\x89o\x081\x94H\'\x7f\x19\xe8m]hx\xbcSv\x8b.\xf9\xb3!\x02\x03\x01\x00\x01\xa3S0Q0\x1d\x06\x03U\x1d\x0e\x04\x16\x04\x14\x82\x97\xfb\x8c\xf3\xd9\xdd\xf2\xf6\x0e\xc9\x0f\x92\x81\\(\xe3\xf3\xff50\x1f\x06\x03U\x1d#\x04\x180\x16\x80\x14\x82\x97\xfb\x8c\xf3\xd9\xdd\xf2\xf6\x0e\xc9\x0f\x92\x81\\(\xe3\xf3\xff50\x0f\x06\x03U\x1d\x13\x01\x01\xff\x04\x050\x03\x01\x01\xff0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x0b\x05\x00\x03\x82\x01\x01\x00<\x00\x1f\x0f]\x10`r\xe0\xbe\xed\xfaG\x89S)\xd4b4"\x08\x0ef\xca\xa4\xbe\xb4\xc3\xd9\xb6\x86\x8f\xa1\x07F}a`\xe5\x12\xed\xe7\xcb\xbe\xfd\xeb\x17\xc0\x9b\x9fYO\x86\xf1\xa9\x92,\x98,\xcf\x96U\xd4\xd6~\xda\x06\x1ff~\xeb%\xfer\x03\xbe\xa0\x0e@\xf1P\xa4\x90\xa7\x8c\xa9c\xd8|\x18Z\xde\x8b|)OPR\xfb\x1e\x1e\xdb4\x03\x83\x1e3\xe4\x02l\x8eVq|\xb7{\xc3#!\x85\x8b\xe3zw\xfe\x7fU\x11\xef\x8c \x13\xc8n\'0\xc5\xb26hu\x13\x1c\xae\x0cv\x16\xfd\xc7\xc8`V\x96\x13>zh_ <\r\xb8\xe0\xff\x1d\x1aY\x91\xd2\xf0X\xf4\x8f \xb1\n_\xb0\xdf\'\xa1\xf9\x87L\xc0\xfe\x8dn\xbfw\xe9\xa7\xba8I\x0e\x9dc$\x1a\x0f\xb3\xfdw\x01\xff;\x13\x0c\x9a\xa7\xaaww\x02\x80\xb7\x00<\x1b\xb5\xe0xL4\xaa\xcbt\xce\x81\x14\x96\x0eP\xee\xe0F\x02\xa7\xab \xe5\xc8x\x02\x8eB\x92\xe9\x0e@\xfdc\x1f\xee\x16\x03\x03\x00\x04\x0e\x00\x00\x00', tls_session=r1.tls_session.mirror())
assert r2.tls_session.tls_version == 0x303
= PR/Issue 3295
pkt = TLSServerHello(b"\x02\x00\x00\x28\x03\x03ABCDEFGHIJKLMNOPQRSTUVWXYZ012345\x00\x00\x39\x00\x00\x00")
assert pkt.extlen == 0
assert pkt.ext is None
###############################################################################
############################ Automaton behaviour ##############################
###############################################################################