From f6ac5006982fd18dfe9f9a67c3534300c7ba8192 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 12 Sep 2022 19:18:41 +0200 Subject: [PATCH] add HTTP/3 content view --- CHANGELOG.md | 2 + mitmproxy/contentviews/__init__.py | 38 +++++- mitmproxy/contentviews/http3.py | 137 ++++++++++++++++++++++ setup.cfg | 1 + test/mitmproxy/contentviews/test_http3.py | 46 ++++++++ test/mitmproxy/net/test_tls.py | 1 - 6 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 mitmproxy/contentviews/http3.py create mode 100644 test/mitmproxy/contentviews/test_http3.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8280422a1..dda38859c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ ([#5548](https://github.com/mitmproxy/mitmproxy/pull/5548), @sanlengjingvv) * Fix a mitmweb crash when scrolling down the flow list. ([#5507](https://github.com/mitmproxy/mitmproxy/pull/5507), @LIU-shuyi) +* Add HTTP/3 binary frame content view. + ([#5582](https://github.com/mitmproxy/mitmproxy/pull/5582), @mhils) ## 28 June 2022: mitmproxy 8.1.1 diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 1b4b696d6..f1f85363f 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -15,7 +15,7 @@ import traceback from typing import Union from typing import Optional -from mitmproxy import flow +from mitmproxy import flow, tcp, udp from mitmproxy import http from mitmproxy.utils import signals, strutils from . import ( @@ -36,6 +36,12 @@ from . import ( graphql, grpc, ) + +try: + from . import http3 +except ImportError: + # FIXME: Remove once QUIC is merged. + http3 = None # type: ignore from .base import View, KEY_MAX, format_text, format_dict, TViewResult from ..http import HTTPFlow from ..tcp import TCPMessage, TCPFlow @@ -128,12 +134,22 @@ def get_message_content_view( if ct := http.parse_content_type(ctype): content_type = f"{ct[0]}/{ct[1]}" + tcp_message = None + if isinstance(message, TCPMessage): + tcp_message = message + + udp_message = None + if isinstance(message, UDPMessage): + udp_message = message + description, lines, error = get_content_view( viewmode, content, content_type=content_type, flow=flow, http_message=http_message, + tcp_message=tcp_message, + udp_message=udp_message, ) if enc: @@ -166,6 +182,8 @@ def get_content_view( content_type: Optional[str] = None, flow: Optional[flow.Flow] = None, http_message: Optional[http.Message] = None, + tcp_message: Optional[tcp.TCPMessage] = None, + udp_message: Optional[udp.UDPMessage] = None, ): """ Args: @@ -180,7 +198,12 @@ def get_content_view( """ try: ret = viewmode( - data, content_type=content_type, flow=flow, http_message=http_message + data, + content_type=content_type, + flow=flow, + http_message=http_message, + tcp_message=tcp_message, + udp_message=udp_message, ) if ret is None: ret = ( @@ -190,6 +213,8 @@ def get_content_view( content_type=content_type, flow=flow, http_message=http_message, + tcp_message=tcp_message, + udp_message=udp_message, )[1], ) desc, content = ret @@ -200,7 +225,12 @@ def get_content_view( raw = get("Raw") assert raw content = raw( - data, content_type=content_type, flow=flow, http_message=http_message + data, + content_type=content_type, + flow=flow, + http_message=http_message, + tcp_message=tcp_message, + udp_message=udp_message, )[1] error = f"{getattr(viewmode, 'name')} content viewer failed: \n{traceback.format_exc()}" @@ -224,6 +254,8 @@ add(query.ViewQuery()) add(protobuf.ViewProtobuf()) add(msgpack.ViewMsgPack()) add(grpc.ViewGrpcProtobuf()) +if http3 is not None: + add(http3.ViewHttp3()) __all__ = [ "View", diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py new file mode 100644 index 000000000..bead71f8a --- /dev/null +++ b/mitmproxy/contentviews/http3.py @@ -0,0 +1,137 @@ +from collections import defaultdict +from collections.abc import Iterator +from dataclasses import dataclass, field + +from aioquic.h3.connection import Setting, parse_settings + +from mitmproxy import flow, tcp +from . import base +from .hex import ViewHex +from ..proxy.layers.http import is_h3_alpn # type: ignore + +from aioquic.buffer import Buffer, BufferReadError +import pylsqpack + + +@dataclass(frozen=True) +class Frame: + """Representation of an HTTP/3 frame.""" + type: int + data: bytes + + def pretty(self): + frame_name = f"0x{self.type:x} Frame" + if self.type == 0: + frame_name = "DATA Frame" + elif self.type == 1: + try: + hdrs = pylsqpack.Decoder(4096, 16).feed_header(0, self.data)[1] + return [ + [("header", "HEADERS Frame")], + *base.format_pairs(hdrs) + ] + except Exception as e: + frame_name = f"HEADERS Frame (error: {e})" + elif self.type == 4: + settings = [] + try: + s = parse_settings(self.data) + except Exception as e: + frame_name = f"SETTINGS Frame (error: {e})" + else: + for k, v in s.items(): + try: + key = Setting(k).name + except ValueError: + key = f"0x{k:x}" + settings.append((key, f"0x{v:x}")) + return [ + [("header", "SETTINGS Frame")], + *base.format_pairs(settings) + ] + return [ + [("header", frame_name)], + *ViewHex._format(self.data), + ] + + +@dataclass +class ConnectionState: + message_count: int = 0 + frames: dict[int, list[Frame]] = field(default_factory=dict) + client_buf: bytearray = field(default_factory=bytearray) + server_buf: bytearray = field(default_factory=bytearray) + + +class ViewHttp3(base.View): + name = "HTTP/3 Frames" + + def __init__(self): + self.connections: defaultdict[tcp.TCPFlow, ConnectionState] = defaultdict(ConnectionState) + + def __call__( + self, + data, + flow: flow.Flow | None = None, + tcp_message: tcp.TCPMessage | None = None, + **metadata + ): + assert isinstance(flow, tcp.TCPFlow) + assert tcp_message + + state = self.connections[flow] + + for message in flow.messages[state.message_count:]: + if message.from_client: + buf = state.client_buf + else: + buf = state.server_buf + buf += message.content + + # TODO: It would be much better to know if the stream is unidirectional. + if buf.startswith(b"\x00\x04"): + del buf[:1] + + while True: + h3_buf = Buffer(data=bytes(buf[:16])) + try: + frame_type = h3_buf.pull_uint_var() + frame_size = h3_buf.pull_uint_var() + except BufferReadError: + break + + consumed = h3_buf.tell() + + if len(buf) < consumed + frame_size: + break + + frame_data = bytes(buf[consumed:consumed + frame_size]) + + frame = Frame(frame_type, frame_data) + + state.frames.setdefault(state.message_count, []).append(frame) + + del buf[:consumed + frame_size] + + state.message_count += 1 + + frames = state.frames.get(flow.messages.index(tcp_message), []) + if not frames: + return "HTTP/3", [] # base.format_text(f"(no complete frames here, {state=})") + else: + return "HTTP/3", fmt_frames(frames) + + def render_priority( + self, + data: bytes, + flow: flow.Flow | None = None, + **metadata + ) -> float: + return 2 * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) * float(isinstance(flow, tcp.TCPFlow)) + + +def fmt_frames(frames: list[Frame]) -> Iterator[base.TViewLine]: + for i, frame in enumerate(frames): + if i > 0: + yield [("text", "")] + yield from frame.pretty() diff --git a/setup.cfg b/setup.cfg index d720b392f..0a3cb9560 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,6 +56,7 @@ exclude = mitmproxy/connections.py mitmproxy/contentviews/base.py mitmproxy/contentviews/grpc.py + mitmproxy/contentviews/http3.py mitmproxy/ctx.py mitmproxy/exceptions.py mitmproxy/flow.py diff --git a/test/mitmproxy/contentviews/test_http3.py b/test/mitmproxy/contentviews/test_http3.py new file mode 100644 index 000000000..496e8f926 --- /dev/null +++ b/test/mitmproxy/contentviews/test_http3.py @@ -0,0 +1,46 @@ +import pytest + +from mitmproxy.tcp import TCPMessage +from mitmproxy.test import tflow +from mitmproxy.contentviews import http3 + +from . import full_eval + +if http3 is None: + pytest.skip("HTTP/3 not available.", allow_module_level=True) + + +@pytest.mark.parametrize("data", [ + # HEADERS + b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xbc\xda\xe0\xdd", + # broken HEADERS + b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xff\xff\xff\xff", + # SETTINGS + b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", + # unknown setting + b"\x00\x04\r\x3f\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", + # out of bounds + b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x42\x00", + # incomplete + b"\x00\x04\r\x06\xff\xff\xff", + # headers + data + ( + b'\x01@I\x00\x00\xdb_\'\x93I|\xa5\x89\xd3M\x1fj\x12q\xd8\x82\xa6\x0bP\xb0\xd0C\x1b_M\x90\xd0bXt\x1eT\xad\x8f~\xfdp' + b'\xeb\xc8\xc0\x97\x07V\x96\xd0z\xbe\x94\x08\x94\xdcZ\xd4\x10\x04%\x02\xe5\xc6\xde\xb8\x17\x14\xc5\xa3\x7fT\x03315' + b'\x00A;\r\n<' + b'TITLE>Not Found\r\n\r\n

Not Found

\r\n

HTTP Error 404. The requested resource is not found.

\r\n\r\n' + ), + b"", +]) +def test_view_http3(data): + v = full_eval(http3.ViewHttp3()) + t = tflow.ttcpflow(messages=[ + TCPMessage(from_client=len(data) > 16, content=data) + ]) + assert (v(b"", flow=t, tcp_message=t.messages[0])) + + +def test_render_priority(): + v = http3.ViewHttp3() + assert not v.render_priority(b"random stuff") diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index de8f7af52..900160329 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -59,7 +59,6 @@ def test_sslkeylogfile(tdata, monkeypatch): read, write = client, server while True: try: - print(read) read.do_handshake() except SSL.WantReadError: write.bio_write(read.bio_read(2 ** 16))