From 203100028566d74be60002231e97c6575b4e91a5 Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Wed, 21 Dec 2016 18:39:01 +0100 Subject: [PATCH 1/2] Support nanosec precision in Pcap + fix TS in PcapNg --- scapy/utils.py | 50 +++++++++++++++++++++++++++++---------------- test/regression.uts | 24 ++++++++++++++++------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/scapy/utils.py b/scapy/utils.py index 75afe0fa9..e52334576 100644 --- a/scapy/utils.py +++ b/scapy/utils.py @@ -653,10 +653,18 @@ class RawPcapReader: def __init__(self, filename, fdesc, magic): self.filename = filename self.f = fdesc - if magic == "\xa1\xb2\xc3\xd4": #big endian + if magic == "\xa1\xb2\xc3\xd4": # big endian self.endian = ">" - elif magic == "\xd4\xc3\xb2\xa1": #little endian + self.nano = False + elif magic == "\xd4\xc3\xb2\xa1": # little endian self.endian = "<" + self.nano = False + elif magic == "\xa1\xb2\x3c\x4d": # big endian, nanosecond-precision + self.endian = ">" + self.nano = True + elif magic == "\x4d\x3c\xb2\xa1": # little endian, nanosecond-precision + self.endian = "<" + self.nano = True else: raise Scapy_Exception( "Not a pcap capture file (bad magic: %r)" % magic @@ -755,7 +763,7 @@ class PcapReader(RawPcapReader): if conf.debug_dissector: raise p = conf.raw_layer(s) - p.time = sec+0.000001*usec + p.time = sec + (0.000000001 if self.nano else 0.000001) * usec return p def read_all(self,count=-1): res = RawPcapReader.read_all(self, count) @@ -824,14 +832,15 @@ class RawPcapNgReader(RawPcapReader): def read_block_idb(self, block, _): """Interface Description Block""" + # We should read options to set if_tsresol self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8])) def read_block_epb(self, block, size): """Enhanced Packet Block""" - intid, sec, usec, caplen, wirelen = struct.unpack(self.endian + "5I", + intid, tshigh, tslow, caplen, wirelen = struct.unpack(self.endian + "5I", block[:20]) return (block[20:20 + caplen][:size], - (self.interfaces[intid][0], sec, usec, wirelen)) + (self.interfaces[intid][0], tshigh, tslow, wirelen)) class PcapNgReader(RawPcapNgReader): @@ -845,7 +854,7 @@ class PcapNgReader(RawPcapNgReader): rp = RawPcapNgReader.read_packet(self, size=size) if rp is None: return None - s, (linktype, sec, usec, wirelen) = rp + s, (linktype, tshigh, tslow, wirelen) = rp try: p = conf.l2types[linktype](s) except KeyboardInterrupt: @@ -854,7 +863,9 @@ class PcapNgReader(RawPcapNgReader): if conf.debug_dissector: raise p = conf.raw_layer(s) - p.time = sec+0.000001*usec + # We should use if_tsresol when available (see + # RawPcapNgReader.read_block_idb) + p.time = float((tshigh << 32) + tslow) / 1000000 return p def read_all(self,count=-1): res = RawPcapNgReader.read_all(self, count) @@ -866,16 +877,18 @@ class PcapNgReader(RawPcapNgReader): class RawPcapWriter: """A stream PCAP writer with more control than wrpcap()""" - def __init__(self, filename, linktype=None, gz=False, endianness="", append=False, sync=False): + def __init__(self, filename, linktype=None, gz=False, endianness="", + append=False, sync=False, nano=False): """ -filename: the name of the file to write packets to, or an open, - writable file-like object. -linktype: force linktype to a given value. If None, linktype is taken - from the first writer packet -gz: compress the capture on the fly +filename: the name of the file to write packets to, or an open, + writable file-like object. +linktype: force linktype to a given value. If None, linktype is taken + from the first writer packet +gz: compress the capture on the fly endianness: force an endianness (little:"<", big:">"). Default is native -append: append packets to the capture file instead of truncating it -sync: do not bufferize writes to the capture file +append: append packets to the capture file instead of truncating it +sync: do not bufferize writes to the capture file +nano: use nanosecond-precision (requires libpcap >= 1.5.0) """ @@ -885,6 +898,7 @@ sync: do not bufferize writes to the capture file self.gz = gz self.endian = endianness self.sync = sync + self.nano = nano bufsz=4096 if sync: bufsz = 0 @@ -913,7 +927,7 @@ sync: do not bufferize writes to the capture file if g.read(16): return - self.f.write(struct.pack(self.endian+"IHHIIII", 0xa1b2c3d4L, + self.f.write(struct.pack(self.endian+"IHHIIII", 0xa1b23c4dL if self.nano else 0xa1b2c3d4L, 2, 4, 0, 0, MTU, self.linktype)) self.f.flush() @@ -958,7 +972,7 @@ sync: do not bufferize writes to the capture file if sec is None: sec = it if usec is None: - usec = int(round((t-it)*1000000)) + usec = int(round((t - it) * (1000000000 if self.nano else 1000000))) self.f.write(struct.pack(self.endian+"IIII", sec, usec, caplen, wirelen)) self.f.write(packet) if self.sync: @@ -996,7 +1010,7 @@ class PcapWriter(RawPcapWriter): self._write_packet(pkt) return sec = int(packet.time) - usec = int(round((packet.time-sec)*1000000)) + usec = int(round((packet.time - sec) * (1000000000 if self.nano else 1000000))) s = str(packet) caplen = len(s) RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen) diff --git a/test/regression.uts b/test/regression.uts index a5a2f710b..5a649a2a7 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -4609,6 +4609,7 @@ nf5.version == 5 and nf5[NetflowHeaderV5].count == 2 and isinstance(nf5[NetflowR import cStringIO pcapfile = cStringIO.StringIO('\xd4\xc3\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacVo*\n\x00(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV_-\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\xf90\n\x00\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00') pcapngfile = cStringIO.StringIO('\n\r\r\n\\\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00,\x00File created by merging: \nFile1: test.pcap \n\x04\x00\x08\x00mergecap\x00\x00\x00\x00\\\x00\x00\x00\x01\x00\x00\x00\\\x00\x00\x00e\x00\x00\x00\xff\xff\x00\x00\x02\x006\x00Unknown/not available in original file format(libpcap)\x00\x00\t\x00\x01\x00\x06\x00\x00\x00\x00\x00\x00\x00\\\x00\x00\x00\x06\x00\x00\x00H\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00/\xfc[\xcd(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00H\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\x1f\xff[\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r<\x00\x00\x00\x06\x00\x00\x00<\x00\x00\x00\x00\x00\x00\x00\x8d*\x05\x00\xb9\x02\\\xcd\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00<\x00\x00\x00') +pcapnanofile = cStringIO.StringIO("M<\xb2\xa1\x02\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\x00\x00e\x00\x00\x00\xcf\xc5\xacV\xc9\xc1\xb5'(\x00\x00\x00(\x00\x00\x00E\x00\x00(\x00\x01\x00\x00@\x06|\xcd\x7f\x00\x00\x01\x7f\x00\x00\x01\x00\x14\x00P\x00\x00\x00\x00\x00\x00\x00\x00P\x02 \x00\x91|\x00\x00\xcf\xc5\xacV-;\xc1'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x11|\xce\x7f\x00\x00\x01\x7f\x00\x00\x01\x005\x005\x00\x08\x01r\xcf\xc5\xacV\x9aL\xcf'\x1c\x00\x00\x00\x1c\x00\x00\x00E\x00\x00\x1c\x00\x01\x00\x00@\x01|\xde\x7f\x00\x00\x01\x7f\x00\x00\x01\x08\x00\xf7\xff\x00\x00\x00\x00") = Read a pcap file pktpcap = rdpcap(pcapfile) @@ -4616,17 +4617,17 @@ pktpcap = rdpcap(pcapfile) = Read a pcapng file pktpcapng = rdpcap(pcapngfile) -= Check both packet lists are the same -assert list(pktpcap) == list(pktpcapng) += Read a pcap file with nanosecond precision +pktpcapnano = rdpcap(pcapnanofile) + += Check all packet lists are the same +assert list(pktpcap) == list(pktpcapng) == list(pktpcapnano) +assert [p.time for p in pktpcap] == [p.time for p in pktpcapng] == [p.time for p in pktpcapnano] = Check packets from pcap file assert all(IP in pkt for pkt in pktpcap) assert all(any(proto in pkt for pkt in pktpcap) for proto in [ICMP, UDP, TCP]) -= Check packets from pcap file -assert all(IP in pkt for pkt in pktpcapng) -assert all(any(proto in pkt for pkt in pktpcapng) for proto in [ICMP, UDP, TCP]) - = Check wrpcap() import os, tempfile fdesc, filename = tempfile.mkstemp() @@ -4656,6 +4657,17 @@ fdesc.close() assert list(pktpcap[TCP]) == list(pktpcap_tcp) os.unlink(filename) += Check wrpcap(nano=True) +fdesc, filename = tempfile.mkstemp() +fdesc = os.fdopen(fdesc, "w") +pktpcapnano[0].time += 0.000000001 +wrpcap(fdesc, pktpcapnano, nano=True) +fdesc.close() +pktpcapnanoread = rdpcap(filename) +assert pktpcapnanoread[0].time == pktpcapnano[0].time +assert pktpcapnanoread[0].time == pktpcap[0].time + 0.000000001 +os.unlink(filename) + ############ ############ From 15bf5f11d9768bd086311bc6435fe1e3ea5c9d78 Mon Sep 17 00:00:00 2001 From: Pierre LALET Date: Thu, 22 Dec 2016 03:44:27 +0100 Subject: [PATCH 2/2] PcapNg: support nanosec precision, simple packet and obsolete packet blocks --- scapy/utils.py | 71 ++++++++++++++++++++++++++++++++++++--------- test/regression.uts | 38 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/scapy/utils.py b/scapy/utils.py index e52334576..af8eda4b4 100644 --- a/scapy/utils.py +++ b/scapy/utils.py @@ -784,10 +784,12 @@ class RawPcapNgReader(RawPcapReader): def __init__(self, filename, fdesc, magic): self.filename = filename self.f = fdesc - # A list of (linktype, snaplen); will be populated by IDBs. + # A list of (linktype, snaplen, tsresol); will be populated by IDBs. self.interfaces = [] self.blocktypes = { 1: self.read_block_idb, + 2: self.read_block_pkt, + 3: self.read_block_spb, 6: self.read_block_epb, } if magic != "\x0a\x0d\x0d\x0a": # PcapNg: @@ -817,12 +819,14 @@ class RawPcapNgReader(RawPcapReader): except struct.error: return None block = self.f.read(blocklen - 12) + if blocklen % 4: + pad = self.f.read(4 - (blocklen % 4)) + warning("PcapNg: bad blocklen %d (MUST be a multiple of 4. " + "Ignored padding %r" % (blocklen, pad)) try: if (blocklen,) != struct.unpack(self.endian + 'I', self.f.read(4)): - raise Scapy_Exception( - "Invalid pcapng block (bad blocklen)" - ) + warning("PcapNg: Invalid pcapng block (bad blocklen)") except struct.error: return None res = self.blocktypes.get(blocktype, @@ -832,15 +836,57 @@ class RawPcapNgReader(RawPcapReader): def read_block_idb(self, block, _): """Interface Description Block""" - # We should read options to set if_tsresol - self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8])) + options = block[16:] + tsresol = 1000000 + while len(options) >= 4: + code, length = struct.unpack(self.endian + "HH", options[:4]) + # PCAP Next Generation (pcapng) Capture File Format + # 4.2. - Interface Description Block + # http://xml2rfc.tools.ietf.org/cgi-bin/xml2rfc.cgi?url=https://raw.githubusercontent.com/pcapng/pcapng/master/draft-tuexen-opsawg-pcapng.xml&modeAsFormat=html/ascii&type=ascii#rfc.section.4.2 + if code == 9 and length == 1 and len(options) >= 5: + tsresol = ord(options[4]) + tsresol = (2 if tsresol & 128 else 10) ** (tsresol & 127) + if code == 0: + if length != 0: + warning("PcapNg: invalid option length %d for end-of-option" % length) + break + if length % 4: + length += (4 - (length % 4)) + options = options[4 + length:] + self.interfaces.append(struct.unpack(self.endian + "HxxI", block[:8]) + + (tsresol,)) def read_block_epb(self, block, size): """Enhanced Packet Block""" - intid, tshigh, tslow, caplen, wirelen = struct.unpack(self.endian + "5I", - block[:20]) + intid, tshigh, tslow, caplen, wirelen = struct.unpack( + self.endian + "5I", + block[:20], + ) return (block[20:20 + caplen][:size], - (self.interfaces[intid][0], tshigh, tslow, wirelen)) + (self.interfaces[intid][0], self.interfaces[intid][2], + tshigh, tslow, wirelen)) + + def read_block_spb(self, block, size): + """Simple Packet Block""" + # "it MUST be assumed that all the Simple Packet Blocks have + # been captured on the interface previously specified in the + # first Interface Description Block." + intid = 0 + wirelen, = struct.unpack(self.endian + "I", block[:4]) + caplen = min(wirelen, self.interfaces[intid][1]) + return (block[4:4 + caplen][:size], + (self.interfaces[intid][0], self.interfaces[intid][2], + None, None, wirelen)) + + def read_block_pkt(self, block, size): + """(Obsolete) Packet Block""" + intid, drops, tshigh, tslow, caplen, wirelen = struct.unpack( + self.endian + "HH4I", + block[:20], + ) + return (block[20:20 + caplen][:size], + (self.interfaces[intid][0], self.interfaces[intid][2], + tshigh, tslow, wirelen)) class PcapNgReader(RawPcapNgReader): @@ -854,7 +900,7 @@ class PcapNgReader(RawPcapNgReader): rp = RawPcapNgReader.read_packet(self, size=size) if rp is None: return None - s, (linktype, tshigh, tslow, wirelen) = rp + s, (linktype, tsresol, tshigh, tslow, wirelen) = rp try: p = conf.l2types[linktype](s) except KeyboardInterrupt: @@ -863,9 +909,8 @@ class PcapNgReader(RawPcapNgReader): if conf.debug_dissector: raise p = conf.raw_layer(s) - # We should use if_tsresol when available (see - # RawPcapNgReader.read_block_idb) - p.time = float((tshigh << 32) + tslow) / 1000000 + if tshigh is not None: + p.time = float((tshigh << 32) + tslow) / tsresol return p def read_all(self,count=-1): res = RawPcapNgReader.read_all(self, count) diff --git a/test/regression.uts b/test/regression.uts index 5a649a2a7..83cb7a457 100644 --- a/test/regression.uts +++ b/test/regression.uts @@ -4668,6 +4668,44 @@ assert pktpcapnanoread[0].time == pktpcapnano[0].time assert pktpcapnanoread[0].time == pktpcap[0].time + 0.000000001 os.unlink(filename) += Check PcapNg with nanosecond precision using obsolete packet block +* first packet from capture file icmp2.ntar -- https://wiki.wireshark.org/Development/PcapNg?action=AttachFile&do=view&target=icmp2.ntar +pcapngfile = cStringIO.StringIO('\n\r\r\n\x1c\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xa8\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x01\x00\x00\x00(\x00\x00\x00\x01\x00\x00\x00\xff\xff\x00\x00\r\x00\x01\x00\x04\x04K\x00\t\x00\x01\x00\tK=N\x00\x00\x00\x00(\x00\x00\x00\x02\x00\x00\x00n\x00\x00\x00\x00\x00\x00\x00e\x14\x00\x00)4\'ON\x00\x00\x00N\x00\x00\x00\x00\x12\xf0\x11h\xd6\x00\x13r\t{\xea\x08\x00E\x00\x00<\x90\xa1\x00\x00\x80\x01\x8e\xad\xc0\xa8M\x07\xc0\xa8M\x1a\x08\x00r[\x03\x00\xd8\x00abcdefghijklmnopqrstuvwabcdefghi\xeay$\xf6\x00\x00n\x00\x00\x00') +pktpcapng = rdpcap(pcapngfile) +assert len(pktpcapng) == 1 +pkt = pktpcapng[0] +# weird, but wireshark agrees +assert pkt.time == 22425.352221737 +assert isinstance(pkt, Ether) +pkt = pkt.payload +assert isinstance(pkt, IP) +pkt = pkt.payload +assert isinstance(pkt, ICMP) +pkt = pkt.payload +assert isinstance(pkt, Raw) and pkt.load == 'abcdefghijklmnopqrstuvwabcdefghi' +pkt = pkt.payload +assert isinstance(pkt, Padding) and pkt.load == '\xeay$\xf6' +pkt = pkt.payload +assert isinstance(pkt, NoPayload) + += Check PcapNg using Simple Packet Block +* previous file with the (obsolete) packet block replaced by a Simple Packet Block +pcapngfile = cStringIO.StringIO('\n\r\r\n\x1c\x00\x00\x00M<+\x1a\x01\x00\x00\x00\xa8\x03\x00\x00\x00\x00\x00\x00\x1c\x00\x00\x00\x01\x00\x00\x00(\x00\x00\x00\x01\x00\x00\x00\xff\xff\x00\x00\r\x00\x01\x00\x04\x04K\x00\t\x00\x01\x00\tK=N\x00\x00\x00\x00(\x00\x00\x00\x03\x00\x00\x00`\x00\x00\x00N\x00\x00\x00\x00\x12\xf0\x11h\xd6\x00\x13r\t{\xea\x08\x00E\x00\x00<\x90\xa1\x00\x00\x80\x01\x8e\xad\xc0\xa8M\x07\xc0\xa8M\x1a\x08\x00r[\x03\x00\xd8\x00abcdefghijklmnopqrstuvwabcdefghi\xeay$\xf6\x00\x00`\x00\x00\x00') +pktpcapng = rdpcap(pcapngfile) +assert len(pktpcapng) == 1 +pkt = pktpcapng[0] +assert isinstance(pkt, Ether) +pkt = pkt.payload +assert isinstance(pkt, IP) +pkt = pkt.payload +assert isinstance(pkt, ICMP) +pkt = pkt.payload +assert isinstance(pkt, Raw) and pkt.load == 'abcdefghijklmnopqrstuvwabcdefghi' +pkt = pkt.payload +assert isinstance(pkt, Padding) and pkt.load == '\xeay$\xf6' +pkt = pkt.payload +assert isinstance(pkt, NoPayload) + ############ ############