From 598bdae8c8f0a8c1786bb84718c55d38191d2bdb Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Fri, 23 Dec 2016 17:13:22 +0100 Subject: [PATCH 1/3] Introduce tcpdump() function One can now use tcpdump or tshark to dissect packets from Scapy. This can help, for example, to check Scapy's dissectors --- .travis/test.sh | 3 +- appveyor.yml | 4 +- scapy/arch/windows/__init__.py | 1 + scapy/config.py | 1 + scapy/utils.py | 70 ++++++++++++++++++++++++++++++++++ test/regression.uts | 16 ++++++++ 6 files changed, 92 insertions(+), 3 deletions(-) diff --git a/.travis/test.sh b/.travis/test.sh index b82d16c8b..f9cf530a1 100644 --- a/.travis/test.sh +++ b/.travis/test.sh @@ -39,8 +39,9 @@ then fi fi -# Do we have tcpdump? +# Do we have tcpdump or thsark? which tcpdump >/dev/null 2>&1 || UT_FLAGS+=" -K tcpdump" +which tshark >/dev/null 2>&1 || UT_FLAGS+=" -K tshark" # Dump Environment (so that we can check PATH, UT_FLAGS, etc.) set diff --git a/appveyor.yml b/appveyor.yml index 9b8537ef5..f290b714c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,7 @@ install: # Installing WinPcap directly does not work, # see http://help.appveyor.com/discussions/problems/2280-winpcap-installation-issue # - choco install -y nmap - - choco install -y winpcap + - choco install -y winpcap wireshark - ps: wget http://www.winpcap.org/windump/install/bin/windump_3_9_5/WinDump.exe -UseBasicParsing -OutFile C:\Windows\System32\windump.exe - refreshenv @@ -24,7 +24,7 @@ install: test_script: # Set environment variables - set PYTHONPATH=%APPVEYOR_BUILD_FOLDER% - - set PATH=%APPVEYOR_BUILD_FOLDER%;%PATH% + - set PATH="%APPVEYOR_BUILD_FOLDER%;C:\Program Files\Wireshark\;%PATH%" # Main unit tests - "%PYTHON%\\python bin\\UTscapy -f text -t test\\regression.uts -F -K automaton -K mock_read_routes6_bsd || exit /b 42" diff --git a/scapy/arch/windows/__init__.py b/scapy/arch/windows/__init__.py index 5ff08a023..ddb5487ff 100755 --- a/scapy/arch/windows/__init__.py +++ b/scapy/arch/windows/__init__.py @@ -187,6 +187,7 @@ class WinProgPath(ConfClass): psreader = win_find_exe("gsview32.exe", "Ghostgum/gsview") dot = win_find_exe("dot", "ATT/Graphviz/bin") tcpdump = win_find_exe("windump") + tshark = win_find_exe("tshark") tcpreplay = win_find_exe("tcpreplay") display = _default hexedit = win_find_exe("hexer") diff --git a/scapy/config.py b/scapy/config.py index 568dada8a..b97ee0bd0 100755 --- a/scapy/config.py +++ b/scapy/config.py @@ -64,6 +64,7 @@ class ProgPath(ConfClass): tcpdump = "tcpdump" tcpreplay = "tcpreplay" hexedit = "hexer" + tshark = "tshark" wireshark = "wireshark" ifconfig = "ifconfig" diff --git a/scapy/utils.py b/scapy/utils.py index af8eda4b4..3f6bb4b7d 100644 --- a/scapy/utils.py +++ b/scapy/utils.py @@ -1088,6 +1088,76 @@ def wireshark(pktlist): wrpcap(f, pktlist) subprocess.Popen([conf.prog.wireshark, "-r", f]) +@conf.commands.register +def tcpdump(pktlist, dump=False, getfd=False, args=None, + prog=None): + """Run tcpdump or tshark on a list of packets + +pktlist: a Packet instance, a PacketList instance or a list of Packet + instances. Can also be a filename (as a string) or an open + file-like object that must be a file format readable by + tshark (Pcap, PcapNg, etc.) + +dump: when set to True, returns a string instead of displaying it. +getfd: when set to True, returns a file-like object to read data + from tcpdump or tshark from. +args: arguments (as a list) to pass to tshark (example for tshark: + args=["-T", "json"]). Defaults to ["-n"]. +prog: program to use (defaults to tcpdump, will work with tshark) + +Examples: + +>>> tcpdump([IP()/TCP(), IP()/UDP()]) +reading from file -, link-type RAW (Raw IP) +16:46:00.474515 IP 127.0.0.1.20 > 127.0.0.1.80: Flags [S], seq 0, win 8192, length 0 +16:46:00.475019 IP 127.0.0.1.53 > 127.0.0.1.53: [|domain] + +>>> tcpdump([IP()/TCP(), IP()/UDP()], prog=conf.prog.tshark) + 1 0.000000 127.0.0.1 -> 127.0.0.1 TCP 40 20->80 [SYN] Seq=0 Win=8192 Len=0 + 2 0.000459 127.0.0.1 -> 127.0.0.1 UDP 28 53->53 Len=0 + +To get a JSON representation of a tshark-parsed PacketList(), one can: +>>> import json, pprint +>>> json_data = json.load(tcpdump(IP(src="217.25.178.5", dst="45.33.32.156"), +... prog=conf.prog.tshark, args=["-T", "json"], +... getfd=True)) +>>> pprint.pprint(json_data) +[{u'_index': u'packets-2016-12-23', + u'_score': None, + u'_source': {u'layers': {u'frame': {u'frame.cap_len': u'20', + u'frame.encap_type': u'7', +[...] + u'frame.time_relative': u'0.000000000'}, + u'ip': {u'ip.addr': u'45.33.32.156', + u'ip.checksum': u'0x0000a20d', +[...] + u'ip.ttl': u'64', + u'ip.version': u'4'}, + u'raw': u'Raw packet data'}}, + u'_type': u'pcap_file'}] +>>> json_data[0]['_source']['layers']['ip']['ip.ttl'] +u'64' + + """ + proc = subprocess.Popen( + [conf.prog.tcpdump if prog is None else prog, "-r", + pktlist if isinstance(pktlist, basestring) else "-"] + + (["-n"] if args is None else args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE if dump or getfd else None, + ) + try: + proc.stdin.writelines(iter(lambda: pktlist.read(1048576), "")) + except AttributeError: + wrpcap(proc.stdin, pktlist) + else: + proc.stdin.close() + if dump: + return "".join(iter(lambda: proc.stdout.read(1048576), "")) + if getfd: + return proc.stdout + proc.wait() + @conf.commands.register def hexedit(x): x = str(x) diff --git a/test/regression.uts b/test/regression.uts index 7c379a502..b68261954 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -4706,6 +4706,22 @@ assert isinstance(pkt, Padding) and pkt.load == '\xeay$\xf6' pkt = pkt.payload assert isinstance(pkt, NoPayload) += Check tcpdump() +~ tcpdump +* No very specific tests because we do not want to depend on tcpdump output +pcapfile = cStringIO.StringIO('\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00') +data = tcpdump(pcapfile, dump=True, args=['-n']).split('\n') +print data +assert 'IP 127.0.0.1.20 > 127.0.0.1.80:' in data[0] +assert 'IP 127.0.0.1.53 > 127.0.0.1.53:' in data[1] +assert 'IP 127.0.0.1 > 127.0.0.1:' in data[2] + += Check tcpdump() command with tshark +~ tshark +pcapfile = cStringIO.StringIO('\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00') +values = [tuple(int(val) for val in line[:-1].split('\t')) for line in tcpdump(pcapfile, prog=conf.prog.tshark, getfd=True, args=['-T', 'fields', '-e', 'ip.ttl', '-e', 'ip.proto'])] +assert values == [(64, 6), (64, 17), (64, 1)] + ############ ############ From 767de9353793268968f3508fc915f47ddac71690 Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Tue, 3 Jan 2017 13:45:59 +0100 Subject: [PATCH 2/3] Fix tcpdump with MacOS (-r - won't work, @guedou @gpotter2) --- scapy/utils.py | 54 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/scapy/utils.py b/scapy/utils.py index 3f6bb4b7d..ad5230e03 100644 --- a/scapy/utils.py +++ b/scapy/utils.py @@ -12,6 +12,7 @@ import random,time import gzip,zlib,cPickle import re,struct,array import subprocess +import tempfile import warnings warnings.filterwarnings("ignore","tempnam",RuntimeWarning, __name__) @@ -1139,19 +1140,48 @@ To get a JSON representation of a tshark-parsed PacketList(), one can: u'64' """ - proc = subprocess.Popen( - [conf.prog.tcpdump if prog is None else prog, "-r", - pktlist if isinstance(pktlist, basestring) else "-"] - + (["-n"] if args is None else args), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE if dump or getfd else None, - ) - try: - proc.stdin.writelines(iter(lambda: pktlist.read(1048576), "")) - except AttributeError: - wrpcap(proc.stdin, pktlist) + if isinstance(pktlist, basestring): + proc = subprocess.Popen( + [conf.prog.tcpdump if prog is None else prog, "-r", pktlist] + + (["-n"] if args is None else args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE if dump or getfd else None, + stderr=open(os.devnull), + ) + elif sys.platform.startswith("darwin"): + # Tcpdump cannot read from stdin, see + # + tmpfile = tempfile.NamedTemporaryFile(delete=False) + try: + tmpfile.writelines(iter(lambda: pktlist.read(1048576), "")) + except AttributeError: + wrpcap(tmpfile, pktlist) + else: + tmpfile.close() + proc = subprocess.Popen( + [conf.prog.tcpdump if prog is None else prog, "-r", + tmpfile.name] + (["-n"] if args is None else args), + stdout=subprocess.PIPE if dump or getfd else None, + stderr=open(os.devnull), + ) + def newdel(self, *args, **kargs): + os.unlink(tmpfile.name) + proc.__del__(self, *args, **kargs) + proc.__del__ = newdel else: - proc.stdin.close() + proc = subprocess.Popen( + [conf.prog.tcpdump if prog is None else prog, "-r", "-"] + + (["-n"] if args is None else args), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE if dump or getfd else None, + stderr=open(os.devnull), + ) + try: + proc.stdin.writelines(iter(lambda: pktlist.read(1048576), "")) + except AttributeError: + wrpcap(proc.stdin, pktlist) + else: + proc.stdin.close() if dump: return "".join(iter(lambda: proc.stdout.read(1048576), "")) if getfd: From 6c7a893d13823764bad039ca1b29042c846ca311 Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Tue, 3 Jan 2017 14:36:26 +0100 Subject: [PATCH 3/3] Use new tcpdump() function to handle sniff() with filter set --- scapy/arch/windows/compatibility.py | 22 +++------------------- scapy/sendrecv.py | 29 +++++------------------------ scapy/utils.py | 5 +---- 3 files changed, 9 insertions(+), 47 deletions(-) diff --git a/scapy/arch/windows/compatibility.py b/scapy/arch/windows/compatibility.py index 573d64c41..6c0b3aefc 100644 --- a/scapy/arch/windows/compatibility.py +++ b/scapy/arch/windows/compatibility.py @@ -19,7 +19,7 @@ from scapy.arch.consts import LOOPBACK_NAME from scapy.config import conf,ConfClass from scapy.base_classes import Gen, SetGen import scapy.plist as plist -from scapy.utils import PcapReader +from scapy.utils import PcapReader, tcpdump from scapy.arch.pcapdnet import PcapTimeoutElapsed from scapy.error import log_runtime from scapy.data import MTU, ETH_P_ARP,ETH_P_ALL @@ -197,24 +197,8 @@ L2socket: use the provided L2socket s = L2socket(type=ETH_P_ALL, *arg, **karg) else: flt = karg.get('filter') - if flt is not None: - if isinstance(offline, basestring): - s = PcapReader( - subprocess.Popen( - [conf.prog.tcpdump, "-r", offline, "-w", "-", flt], - stdout=subprocess.PIPE - ).stdout - ) - else: - s = PcapReader( - subprocess.Popen( - [conf.prog.tcpdump, "-r", "-", "-w", "-", flt], - stdin=offline, - stdout=subprocess.PIPE - ).stdout - ) - else: - s = PcapReader(offline) + s = PcapReader(offline if flt is None else + tcpdump(offline, args=["-w", "-", flt], getfd=True)) lst = [] if timeout is not None: stoptime = time.time()+timeout diff --git a/scapy/sendrecv.py b/scapy/sendrecv.py index 9ca7d4805..308c114d2 100644 --- a/scapy/sendrecv.py +++ b/scapy/sendrecv.py @@ -16,7 +16,7 @@ from scapy.arch.consts import DARWIN, FREEBSD, OPENBSD from scapy.data import * from scapy.config import conf from scapy.packet import Gen -from scapy.utils import warning,get_temp_file,PcapReader,wrpcap +from scapy.utils import warning, get_temp_file, PcapReader, tcpdump, wrpcap from scapy import plist from scapy.error import log_runtime,log_interactive from scapy.base_classes import SetGen @@ -607,29 +607,10 @@ interfaces) **karg)] else: flt = karg.get('filter') - if flt is not None: - if isinstance(offline, basestring): - sniff_sockets = [ - PcapReader( - subprocess.Popen( - [conf.prog.tcpdump, "-r", offline, "-w", "-", - flt], - stdout=subprocess.PIPE - ).stdout - ) - ] - else: - sniff_sockets = [ - PcapReader( - subprocess.Popen( - [conf.prog.tcpdump, "-r", "-", "-w", "-", flt], - stdin=offline, - stdout=subprocess.PIPE - ).stdout - ) - ] - else: - sniff_sockets = [PcapReader(offline)] + sniff_sockets = [PcapReader( + offline if flt is None else + tcpdump(offline, args=["-w", "-", flt], getfd=True) + )] lst = [] if timeout is not None: stoptime = time.time()+timeout diff --git a/scapy/utils.py b/scapy/utils.py index ad5230e03..b26faf05e 100644 --- a/scapy/utils.py +++ b/scapy/utils.py @@ -1164,10 +1164,7 @@ u'64' stdout=subprocess.PIPE if dump or getfd else None, stderr=open(os.devnull), ) - def newdel(self, *args, **kargs): - os.unlink(tmpfile.name) - proc.__del__(self, *args, **kargs) - proc.__del__ = newdel + conf.temp_files.append(tmpfile.name) else: proc = subprocess.Popen( [conf.prog.tcpdump if prog is None else prog, "-r", "-"]