From c2ce8dc5d35eb47aed9f5b6fcb515bef151d976e Mon Sep 17 00:00:00 2001 From: gpotter2 <10530980+gpotter2@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:35:18 +0100 Subject: [PATCH] Small HTTP session fixes (#4601) --- scapy/layers/http.py | 21 ++++++++------------- scapy/layers/l2.py | 6 +++++- scapy/sessions.py | 27 ++++++++++++++++----------- test/scapy/layers/http.uts | 12 ++++++------ 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/scapy/layers/http.py b/scapy/layers/http.py index 394d220e4..0f3cd1181 100644 --- a/scapy/layers/http.py +++ b/scapy/layers/http.py @@ -652,16 +652,7 @@ class HTTP(Packet): is_response = isinstance(http_packet.payload, cls.clsresp) # Packets may have a Content-Length we must honnor length = http_packet.Content_Length - # Heuristic to try and detect instant HEAD responses, as those include a - # Content-Length that must not be honored. This is a bit crappy, and assumes - # that a 'HEAD' will never include an Encoding... - if ( - is_response and - data.endswith(b"\r\n\r\n") and - not http_packet[HTTPResponse]._get_encodings() - ): - detect_end = lambda _: True - elif length is not None: + if length is not None: # The packet provides a Content-Length attribute: let's # use it. When the total size of the frags is high enough, # we have the packet @@ -672,8 +663,12 @@ class HTTP(Packet): detect_end = lambda dat: len(dat) - http_length >= length else: # The HTTP layer isn't fully received. - detect_end = lambda dat: False - metadata["detect_unknown"] = True + if metadata.get("tcp_end", False): + # This was likely a HEAD response. Ugh + detect_end = lambda dat: True + else: + detect_end = lambda dat: False + metadata["detect_unknown"] = True else: # It's not Content-Length based. It could be chunked encodings = http_packet[cls].payload._get_encodings() @@ -833,7 +828,7 @@ class HTTP_Client(object): Perform a HTTP(s) request. """ # Parse request url - m = re.match(r"(https?)://([^/:]+)(?:\:(\d+))?(?:/(.*))?", url) + m = re.match(r"(https?)://([^/:]+)(?:\:(\d+))?(/.*)?", url) if not m: raise ValueError("Bad URL !") transport, host, port, path = m.groups() diff --git a/scapy/layers/l2.py b/scapy/layers/l2.py index 1c46b246c..532cd9a5f 100644 --- a/scapy/layers/l2.py +++ b/scapy/layers/l2.py @@ -158,7 +158,11 @@ def getmacbyip(ip, chainCC=0): # Check the routing table iff, _, gw = conf.route.route(ip) - # Broadcast case + # Limited broadcast + if ip == "255.255.255.255": + return "ff:ff:ff:ff:ff:ff" + + # Directed broadcast if (iff == conf.loopback_name) or (ip in conf.route.get_if_bcast(iff)): return "ff:ff:ff:ff:ff:ff" diff --git a/scapy/sessions.py b/scapy/sessions.py index be95b7693..3c58dc2c6 100644 --- a/scapy/sessions.py +++ b/scapy/sessions.py @@ -12,7 +12,7 @@ import struct from scapy.compat import orb from scapy.config import conf -from scapy.packet import NoPayload, Packet +from scapy.packet import Packet from scapy.pton_ntop import inet_pton # Typing imports @@ -310,8 +310,6 @@ class TCPSession(IPSession): if TCP not in pkt: return pkt pay = pkt[TCP].payload - if isinstance(pay, (NoPayload, conf.padding_layer)): - return pkt new_data = pay.original # Match packets by a unique TCP identifier ident = self._get_ident(pkt) @@ -333,16 +331,22 @@ class TCPSession(IPSession): metadata["tcp_reassemble"] = tcp_reassemble = streamcls(pay_class) else: tcp_reassemble = metadata["tcp_reassemble"] - # Get a relative sequence number for a storage purpose - relative_seq = metadata.get("relative_seq", None) - if relative_seq is None: - relative_seq = metadata["relative_seq"] = seq - 1 - seq = seq - relative_seq - # Add the data to the buffer - data.append(new_data, seq) + + if pay: + # Get a relative sequence number for a storage purpose + relative_seq = metadata.get("relative_seq", None) + if relative_seq is None: + relative_seq = metadata["relative_seq"] = seq - 1 + seq = seq - relative_seq + # Add the data to the buffer + data.append(new_data, seq) + # Check TCP FIN or TCP RESET if pkt[TCP].flags.F or pkt[TCP].flags.R: metadata["tcp_end"] = True + elif not pay: + # If there's no payload and the stream isn't ending, ignore. + return pkt # In case any app layer protocol requires it, # allow the parser to inspect TCP PSH flag @@ -393,7 +397,8 @@ class TCPSession(IPSession): if isinstance(packet, conf.padding_layer): return None # Rebuild resulting packet - pay.underlayer.remove_payload() + if pay: + pay.underlayer.remove_payload() if IP in pkt: pkt[IP].len = None pkt[IP].chksum = None diff --git a/test/scapy/layers/http.uts b/test/scapy/layers/http.uts index 5a80a0f4a..f81a53311 100644 --- a/test/scapy/layers/http.uts +++ b/test/scapy/layers/http.uts @@ -79,11 +79,11 @@ assert HTTPRequest in a[3] assert a[3].Method == b"HEAD" assert a[3].User_Agent == b'curl/7.88.1' -assert HTTPResponse in a[5] -assert a[5].Content_Type == b'text/html; charset=UTF-8' -assert a[5].Expires == b'Mon, 01 Apr 2024 22:25:38 GMT' -assert a[5].Reason_Phrase == b'Moved Permanently' -assert a[5].X_Frame_Options == b"SAMEORIGIN" +assert HTTPResponse in a[6] +assert a[6].Content_Type == b'text/html; charset=UTF-8' +assert a[6].Expires == b'Mon, 01 Apr 2024 22:25:38 GMT' +assert a[6].Reason_Phrase == b'Moved Permanently' +assert a[6].X_Frame_Options == b"SAMEORIGIN" = HTTP build with 'chunked' content type @@ -214,7 +214,7 @@ filename = scapy_path("/test/pcaps/http_tcp_psh.pcap.gz") pkts = sniff(offline=filename, session=TCPSession) -assert len(pkts) == 15 +assert len(pkts) == 14 # Verify a split header exists in the packet assert pkts[5].User_Agent == b'example_user_agent'