add HTTP/3 content view
This commit is contained in:
Maximilian Hils
parent
b5d06863b3
commit
f6ac500698
|
|
@ -29,6 +29,8 @@
|
|||
([#5548](https://github.com/mitmproxy/mitmproxy/pull/5548), @sanlengjingvv)
|
||||
* Fix a mitmweb crash when scrolling down the flow list.
|
||||
([#5507](https://github.com/mitmproxy/mitmproxy/pull/5507), @LIU-shuyi)
|
||||
* Add HTTP/3 binary frame content view.
|
||||
([#5582](https://github.com/mitmproxy/mitmproxy/pull/5582), @mhils)
|
||||
|
||||
## 28 June 2022: mitmproxy 8.1.1
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import traceback
|
|||
from typing import Union
|
||||
from typing import Optional
|
||||
|
||||
from mitmproxy import flow
|
||||
from mitmproxy import flow, tcp, udp
|
||||
from mitmproxy import http
|
||||
from mitmproxy.utils import signals, strutils
|
||||
from . import (
|
||||
|
|
@ -36,6 +36,12 @@ from . import (
|
|||
graphql,
|
||||
grpc,
|
||||
)
|
||||
|
||||
try:
|
||||
from . import http3
|
||||
except ImportError:
|
||||
# FIXME: Remove once QUIC is merged.
|
||||
http3 = None # type: ignore
|
||||
from .base import View, KEY_MAX, format_text, format_dict, TViewResult
|
||||
from ..http import HTTPFlow
|
||||
from ..tcp import TCPMessage, TCPFlow
|
||||
|
|
@ -128,12 +134,22 @@ def get_message_content_view(
|
|||
if ct := http.parse_content_type(ctype):
|
||||
content_type = f"{ct[0]}/{ct[1]}"
|
||||
|
||||
tcp_message = None
|
||||
if isinstance(message, TCPMessage):
|
||||
tcp_message = message
|
||||
|
||||
udp_message = None
|
||||
if isinstance(message, UDPMessage):
|
||||
udp_message = message
|
||||
|
||||
description, lines, error = get_content_view(
|
||||
viewmode,
|
||||
content,
|
||||
content_type=content_type,
|
||||
flow=flow,
|
||||
http_message=http_message,
|
||||
tcp_message=tcp_message,
|
||||
udp_message=udp_message,
|
||||
)
|
||||
|
||||
if enc:
|
||||
|
|
@ -166,6 +182,8 @@ def get_content_view(
|
|||
content_type: Optional[str] = None,
|
||||
flow: Optional[flow.Flow] = None,
|
||||
http_message: Optional[http.Message] = None,
|
||||
tcp_message: Optional[tcp.TCPMessage] = None,
|
||||
udp_message: Optional[udp.UDPMessage] = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
|
|
@ -180,7 +198,12 @@ def get_content_view(
|
|||
"""
|
||||
try:
|
||||
ret = viewmode(
|
||||
data, content_type=content_type, flow=flow, http_message=http_message
|
||||
data,
|
||||
content_type=content_type,
|
||||
flow=flow,
|
||||
http_message=http_message,
|
||||
tcp_message=tcp_message,
|
||||
udp_message=udp_message,
|
||||
)
|
||||
if ret is None:
|
||||
ret = (
|
||||
|
|
@ -190,6 +213,8 @@ def get_content_view(
|
|||
content_type=content_type,
|
||||
flow=flow,
|
||||
http_message=http_message,
|
||||
tcp_message=tcp_message,
|
||||
udp_message=udp_message,
|
||||
)[1],
|
||||
)
|
||||
desc, content = ret
|
||||
|
|
@ -200,7 +225,12 @@ def get_content_view(
|
|||
raw = get("Raw")
|
||||
assert raw
|
||||
content = raw(
|
||||
data, content_type=content_type, flow=flow, http_message=http_message
|
||||
data,
|
||||
content_type=content_type,
|
||||
flow=flow,
|
||||
http_message=http_message,
|
||||
tcp_message=tcp_message,
|
||||
udp_message=udp_message,
|
||||
)[1]
|
||||
error = f"{getattr(viewmode, 'name')} content viewer failed: \n{traceback.format_exc()}"
|
||||
|
||||
|
|
@ -224,6 +254,8 @@ add(query.ViewQuery())
|
|||
add(protobuf.ViewProtobuf())
|
||||
add(msgpack.ViewMsgPack())
|
||||
add(grpc.ViewGrpcProtobuf())
|
||||
if http3 is not None:
|
||||
add(http3.ViewHttp3())
|
||||
|
||||
__all__ = [
|
||||
"View",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,137 @@
|
|||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from aioquic.h3.connection import Setting, parse_settings
|
||||
|
||||
from mitmproxy import flow, tcp
|
||||
from . import base
|
||||
from .hex import ViewHex
|
||||
from ..proxy.layers.http import is_h3_alpn # type: ignore
|
||||
|
||||
from aioquic.buffer import Buffer, BufferReadError
|
||||
import pylsqpack
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Frame:
|
||||
"""Representation of an HTTP/3 frame."""
|
||||
type: int
|
||||
data: bytes
|
||||
|
||||
def pretty(self):
|
||||
frame_name = f"0x{self.type:x} Frame"
|
||||
if self.type == 0:
|
||||
frame_name = "DATA Frame"
|
||||
elif self.type == 1:
|
||||
try:
|
||||
hdrs = pylsqpack.Decoder(4096, 16).feed_header(0, self.data)[1]
|
||||
return [
|
||||
[("header", "HEADERS Frame")],
|
||||
*base.format_pairs(hdrs)
|
||||
]
|
||||
except Exception as e:
|
||||
frame_name = f"HEADERS Frame (error: {e})"
|
||||
elif self.type == 4:
|
||||
settings = []
|
||||
try:
|
||||
s = parse_settings(self.data)
|
||||
except Exception as e:
|
||||
frame_name = f"SETTINGS Frame (error: {e})"
|
||||
else:
|
||||
for k, v in s.items():
|
||||
try:
|
||||
key = Setting(k).name
|
||||
except ValueError:
|
||||
key = f"0x{k:x}"
|
||||
settings.append((key, f"0x{v:x}"))
|
||||
return [
|
||||
[("header", "SETTINGS Frame")],
|
||||
*base.format_pairs(settings)
|
||||
]
|
||||
return [
|
||||
[("header", frame_name)],
|
||||
*ViewHex._format(self.data),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConnectionState:
|
||||
message_count: int = 0
|
||||
frames: dict[int, list[Frame]] = field(default_factory=dict)
|
||||
client_buf: bytearray = field(default_factory=bytearray)
|
||||
server_buf: bytearray = field(default_factory=bytearray)
|
||||
|
||||
|
||||
class ViewHttp3(base.View):
|
||||
name = "HTTP/3 Frames"
|
||||
|
||||
def __init__(self):
|
||||
self.connections: defaultdict[tcp.TCPFlow, ConnectionState] = defaultdict(ConnectionState)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
data,
|
||||
flow: flow.Flow | None = None,
|
||||
tcp_message: tcp.TCPMessage | None = None,
|
||||
**metadata
|
||||
):
|
||||
assert isinstance(flow, tcp.TCPFlow)
|
||||
assert tcp_message
|
||||
|
||||
state = self.connections[flow]
|
||||
|
||||
for message in flow.messages[state.message_count:]:
|
||||
if message.from_client:
|
||||
buf = state.client_buf
|
||||
else:
|
||||
buf = state.server_buf
|
||||
buf += message.content
|
||||
|
||||
# TODO: It would be much better to know if the stream is unidirectional.
|
||||
if buf.startswith(b"\x00\x04"):
|
||||
del buf[:1]
|
||||
|
||||
while True:
|
||||
h3_buf = Buffer(data=bytes(buf[:16]))
|
||||
try:
|
||||
frame_type = h3_buf.pull_uint_var()
|
||||
frame_size = h3_buf.pull_uint_var()
|
||||
except BufferReadError:
|
||||
break
|
||||
|
||||
consumed = h3_buf.tell()
|
||||
|
||||
if len(buf) < consumed + frame_size:
|
||||
break
|
||||
|
||||
frame_data = bytes(buf[consumed:consumed + frame_size])
|
||||
|
||||
frame = Frame(frame_type, frame_data)
|
||||
|
||||
state.frames.setdefault(state.message_count, []).append(frame)
|
||||
|
||||
del buf[:consumed + frame_size]
|
||||
|
||||
state.message_count += 1
|
||||
|
||||
frames = state.frames.get(flow.messages.index(tcp_message), [])
|
||||
if not frames:
|
||||
return "HTTP/3", [] # base.format_text(f"(no complete frames here, {state=})")
|
||||
else:
|
||||
return "HTTP/3", fmt_frames(frames)
|
||||
|
||||
def render_priority(
|
||||
self,
|
||||
data: bytes,
|
||||
flow: flow.Flow | None = None,
|
||||
**metadata
|
||||
) -> float:
|
||||
return 2 * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) * float(isinstance(flow, tcp.TCPFlow))
|
||||
|
||||
|
||||
def fmt_frames(frames: list[Frame]) -> Iterator[base.TViewLine]:
|
||||
for i, frame in enumerate(frames):
|
||||
if i > 0:
|
||||
yield [("text", "")]
|
||||
yield from frame.pretty()
|
||||
|
|
@ -56,6 +56,7 @@ exclude =
|
|||
mitmproxy/connections.py
|
||||
mitmproxy/contentviews/base.py
|
||||
mitmproxy/contentviews/grpc.py
|
||||
mitmproxy/contentviews/http3.py
|
||||
mitmproxy/ctx.py
|
||||
mitmproxy/exceptions.py
|
||||
mitmproxy/flow.py
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
import pytest
|
||||
|
||||
from mitmproxy.tcp import TCPMessage
|
||||
from mitmproxy.test import tflow
|
||||
from mitmproxy.contentviews import http3
|
||||
|
||||
from . import full_eval
|
||||
|
||||
if http3 is None:
|
||||
pytest.skip("HTTP/3 not available.", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data", [
|
||||
# HEADERS
|
||||
b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xbc\xda\xe0\xdd",
|
||||
# broken HEADERS
|
||||
b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xff\xff\xff\xff",
|
||||
# SETTINGS
|
||||
b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00",
|
||||
# unknown setting
|
||||
b"\x00\x04\r\x3f\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00",
|
||||
# out of bounds
|
||||
b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x42\x00",
|
||||
# incomplete
|
||||
b"\x00\x04\r\x06\xff\xff\xff",
|
||||
# headers + data
|
||||
(
|
||||
b'\x01@I\x00\x00\xdb_\'\x93I|\xa5\x89\xd3M\x1fj\x12q\xd8\x82\xa6\x0bP\xb0\xd0C\x1b_M\x90\xd0bXt\x1eT\xad\x8f~\xfdp'
|
||||
b'\xeb\xc8\xc0\x97\x07V\x96\xd0z\xbe\x94\x08\x94\xdcZ\xd4\x10\x04%\x02\xe5\xc6\xde\xb8\x17\x14\xc5\xa3\x7fT\x03315'
|
||||
b'\x00A;<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN""http://www.w3.org/TR/html4/strict.dtd">\r\n<HTML><HEAD><'
|
||||
b'TITLE>Not Found</TITLE>\r\n<META HTTP-EQUIV="Content-Type" Content="text/html; charset=us-ascii"></HEAD>\r\n<BOD'
|
||||
b'Y><h2>Not Found</h2>\r\n<hr><p>HTTP Error 404. The requested resource is not found.</p>\r\n</BODY></HTML>\r\n'
|
||||
),
|
||||
b"",
|
||||
])
|
||||
def test_view_http3(data):
|
||||
v = full_eval(http3.ViewHttp3())
|
||||
t = tflow.ttcpflow(messages=[
|
||||
TCPMessage(from_client=len(data) > 16, content=data)
|
||||
])
|
||||
assert (v(b"", flow=t, tcp_message=t.messages[0]))
|
||||
|
||||
|
||||
def test_render_priority():
|
||||
v = http3.ViewHttp3()
|
||||
assert not v.render_priority(b"random stuff")
|
||||
|
|
@ -59,7 +59,6 @@ def test_sslkeylogfile(tdata, monkeypatch):
|
|||
read, write = client, server
|
||||
while True:
|
||||
try:
|
||||
print(read)
|
||||
read.do_handshake()
|
||||
except SSL.WantReadError:
|
||||
write.bio_write(read.bio_read(2 ** 16))
|
||||
|
|
|
|||
Loading…
Reference in New Issue