diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c1ffd9db..822555a85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ ## Unreleased: mitmproxy next +- mitmproxy now supports transparent HTTP/3 proxying. + ([#7202](https://github.com/mitmproxy/mitmproxy/pull/7202), @errorxyz, @meitinger, @mhils) - Fix endless tnetstring parsing in case of very large tnetstring ([#7121](https://github.com/mitmproxy/mitmproxy/pull/7121), @mik1904) - Tighten HTTP detection heuristic to better support custom TCP-based protocols. diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 97f93eb48..eeb26d8fb 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -26,6 +26,7 @@ from typing import Any from typing import cast from mitmproxy import ctx +from mitmproxy.connection import Address from mitmproxy.net.tls import starts_like_dtls_record from mitmproxy.net.tls import starts_like_tls_record from mitmproxy.proxy import layer @@ -152,7 +153,7 @@ class NextLayer: server_tls.child_layer = ClientTLSLayer(context) return server_tls # 3b) QUIC - if udp_based and _starts_like_quic(data_client): + if udp_based and _starts_like_quic(data_client, context.server.address): server_quic = ServerQuicLayer(context) server_quic.child_layer = ClientQuicLayer(context) return server_quic @@ -164,19 +165,16 @@ class NextLayer: return layers.UDPLayer(context) # 5) Handle application protocol - # 5a) Is it DNS? + # 5a) Do we have a known ALPN negotiation? + if context.client.alpn: + if context.client.alpn in HTTP_ALPNS: + return layers.HttpLayer(context, HTTPMode.transparent) + elif context.client.tls_version == "QUICv1": + # TODO: Once we support more QUIC-based protocols, relax force_raw here. + return layers.RawQuicLayer(context, force_raw=True) + # 5b) Is it DNS? if context.server.address and context.server.address[1] in (53, 5353): return layers.DNSLayer(context) - - # 5b) Do we have a known ALPN negotiation? - if context.client.alpn in HTTP_ALPNS: - explicit_quic_proxy = ( - isinstance(context.client.proxy_mode, modes.ReverseMode) - and context.client.proxy_mode.scheme == "quic" - ) - if not explicit_quic_proxy: - return layers.HttpLayer(context, HTTPMode.transparent) - # 5c) We have no other specialized layers for UDP, so we fall back to raw forwarding. if udp_based: return layers.UDPLayer(context) @@ -398,7 +396,7 @@ class NextLayer: case "quic": stack /= ServerQuicLayer(context) stack /= ClientQuicLayer(context) - stack /= RawQuicLayer(context) + stack /= RawQuicLayer(context, force_raw=True) case _: # pragma: no cover assert_never(spec.scheme) @@ -430,11 +428,47 @@ class NextLayer: ) -def _starts_like_quic(data_client: bytes) -> bool: - # FIXME: perf - try: - quic_parse_client_hello_from_datagrams([data_client]) - except ValueError: +# https://www.iana.org/assignments/quic/quic.xhtml +KNOWN_QUIC_VERSIONS = { + 0x00000001, # QUIC v1 + 0x51303433, # Google QUIC Q043 + 0x51303436, # Google QUIC Q046 + 0x51303530, # Google QUIC Q050 + 0x6B3343CF, # QUIC v2 + 0x709A50C4, # QUIC v2 draft codepoint +} + +TYPICAL_QUIC_PORTS = {80, 443, 8443} + + +def _starts_like_quic(data_client: bytes, server_address: Address | None) -> bool: + """ + Make an educated guess on whether this could be QUIC. + This turns out to be quite hard in practice as 1-RTT packets are hardly distinguishable from noise. + + Returns: + True, if the passed bytes could be the start of a QUIC packet. + False, otherwise. + """ + # Minimum size: 1 flag byte + 1+ packet number bytes + 16+ bytes encrypted payload + if len(data_client) < 18: return False + if starts_like_dtls_record(data_client): + return False + # TODO: Add more checks here to detect true negatives. + + # Long Header Packets + if data_client[0] & 0x80: + version = int.from_bytes(data_client[1:5], "big") + if version in KNOWN_QUIC_VERSIONS: + return True + # https://www.rfc-editor.org/rfc/rfc9000.html#name-versions + # Versions that follow the pattern 0x?a?a?a?a are reserved for use in forcing version negotiation + if version & 0x0F0F0F0F == 0x0A0A0A0A: + return True else: - return True + # ¯\_(ツ)_/¯ + # We can't even rely on the QUIC bit, see https://datatracker.ietf.org/doc/rfc9287/. + pass + + return bool(server_address and server_address[1] in TYPICAL_QUIC_PORTS) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index cd5cad9a0..e2624a426 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -149,12 +149,6 @@ class Options(optmanager.OptManager): True, "Enable/disable support for QUIC and HTTP/3. Enabled by default.", ) - self.add_option( - "experimental_transparent_http3", - bool, - False, - "Experimental support for QUIC in transparent mode. This option is for development only and will be removed soon.", - ) self.add_option( "http_connect_send_host_header", bool, diff --git a/mitmproxy/proxy/layers/quic/_raw_layers.py b/mitmproxy/proxy/layers/quic/_raw_layers.py index f9984dea7..18d1a367a 100644 --- a/mitmproxy/proxy/layers/quic/_raw_layers.py +++ b/mitmproxy/proxy/layers/quic/_raw_layers.py @@ -66,7 +66,9 @@ class QuicStreamLayer(layer.Layer): child_layer: layer.Layer """The stream's child layer.""" - def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None: + def __init__( + self, context: context.Context, force_raw: bool, stream_id: int + ) -> None: # we mustn't reuse the client from the QUIC connection, as the state and protocol differs self.client = context.client = context.client.copy() self.client.transport_protocol = "tcp" @@ -88,12 +90,9 @@ class QuicStreamLayer(layer.Layer): ) self._server_stream_id: int | None = None - # ignored connections will be assigned a TCPLayer immediately super().__init__(context) self.child_layer = ( - TCPLayer(context, ignore=True) - if ignore - else QuicStreamNextLayer(context, self) + TCPLayer(context) if force_raw else QuicStreamNextLayer(context, self) ) self.refresh_metadata() @@ -150,8 +149,8 @@ class RawQuicLayer(layer.Layer): This layer is responsible for de-multiplexing QUIC streams into an individual layer stack per stream. """ - ignore: bool - """Indicates whether traffic should be routed as-is.""" + force_raw: bool + """Indicates whether traffic should be treated as raw TCP/UDP without further protocol detection.""" datagram_layer: layer.Layer """ The layer that is handling datagrams over QUIC. It's like a child_layer, but with a forked context. @@ -170,12 +169,12 @@ class RawQuicLayer(layer.Layer): next_stream_id: list[int] """List containing the next stream ID for all four is_unidirectional/is_client combinations.""" - def __init__(self, context: context.Context, ignore: bool = False) -> None: + def __init__(self, context: context.Context, force_raw: bool = False) -> None: super().__init__(context) - self.ignore = ignore + self.force_raw = force_raw self.datagram_layer = ( - UDPLayer(self.context.fork(), ignore=True) - if ignore + UDPLayer(self.context.fork()) + if force_raw else layer.NextLayer(self.context.fork()) ) self.client_stream_ids = {} @@ -247,7 +246,9 @@ class RawQuicLayer(layer.Layer): # create, register and start the layer stream_layer = QuicStreamLayer( - self.context.fork(), self.ignore, client_stream_id + self.context.fork(), + force_raw=self.force_raw, + stream_id=client_stream_id, ) self.client_stream_ids[client_stream_id] = stream_layer if server_stream_id is not None: diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index cc0c879bc..81c33c93a 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -9,11 +9,13 @@ from unittest.mock import MagicMock import pytest +from mitmproxy.addons.next_layer import _starts_like_quic from mitmproxy.addons.next_layer import NeedsMoreData from mitmproxy.addons.next_layer import NextLayer from mitmproxy.addons.next_layer import stack_match from mitmproxy.connection import Address from mitmproxy.connection import Client +from mitmproxy.connection import TlsVersion from mitmproxy.connection import TransportProtocol from mitmproxy.proxy.context import Context from mitmproxy.proxy.layer import Layer @@ -22,7 +24,6 @@ from mitmproxy.proxy.layers import ClientTLSLayer from mitmproxy.proxy.layers import DNSLayer from mitmproxy.proxy.layers import HttpLayer from mitmproxy.proxy.layers import modes -from mitmproxy.proxy.layers import QuicStreamLayer from mitmproxy.proxy.layers import RawQuicLayer from mitmproxy.proxy.layers import ServerQuicLayer from mitmproxy.proxy.layers import ServerTLSLayer @@ -31,7 +32,6 @@ from mitmproxy.proxy.layers import UDPLayer from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.http import HttpStream from mitmproxy.proxy.layers.tls import HTTP1_ALPNS -from mitmproxy.proxy.layers.tls import HTTP3_ALPN from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.test import taddons @@ -92,6 +92,13 @@ quic_client_hello = bytes.fromhex( "297c0013924e88248684fe8f2098326ce51aa6e5" ) +quic_short_header_packet = bytes.fromhex( + "52e23539dde270bb19f7a8b63b7bcf3cdacf7d3dc68a7e00318bfa2dac3bad12cb7d78112efb5bcb1ee8e0b347" + "641cccd2736577d0178b4c4c4e97a8e9e2af1d28502e58c4882223e70c4d5124c4b016855340e982c5c453d61d" + "7d0720be075fce3126de3f0d54dc059150e0f80f1a8db5e542eb03240b0a1db44a322fb4fd3c6f2e054b369e14" + "5a5ff925db617d187ec65a7f00d77651968e74c1a9ddc3c7fab57e8df821b07e103264244a3a03d17984e29933" +) + dns_query = bytes.fromhex("002a01000001000000000000076578616d706c6503636f6d0000010001") # Custom protocol with just base64-encoded messages @@ -413,6 +420,7 @@ class TConf: after: list[type[Layer]] proxy_mode: str = "regular" transport_protocol: TransportProtocol = "tcp" + tls_version: TlsVersion = None data_client: bytes = b"" data_server: bytes = b"" ignore_hosts: Sequence[str] = () @@ -632,28 +640,6 @@ reverse_proxy_configs.extend( ), id="reverse proxy: quic", ), - pytest.param( - TConf( - before=[ - modes.ReverseProxy, - ServerQuicLayer, - ClientQuicLayer, - RawQuicLayer, - lambda ctx: QuicStreamLayer(ctx, False, 0), - ], - after=[ - modes.ReverseProxy, - ServerQuicLayer, - ClientQuicLayer, - RawQuicLayer, - QuicStreamLayer, - TCPLayer, - ], - proxy_mode="reverse:quic://example.com", - alpn=HTTP3_ALPN, - ), - id="reverse proxy: quic", - ), pytest.param( TConf( before=[modes.ReverseProxy], @@ -688,14 +674,22 @@ transparent_proxy_configs = [ id=f"transparent proxy: dtls", ), pytest.param( - TConf( + quic := TConf( before=[modes.TransparentProxy], after=[modes.TransparentProxy, ServerQuicLayer, ClientQuicLayer], data_client=quic_client_hello, transport_protocol="udp", + server_address=("192.0.2.1", 443), ), id="transparent proxy: quic", ), + pytest.param( + dataclasses.replace( + quic, + data_client=quic_short_header_packet, + ), + id="transparent proxy: existing quic session", + ), pytest.param( TConf( before=[modes.TransparentProxy], @@ -802,6 +796,21 @@ transparent_proxy_configs = [ ), id="wireguard proxy: dns should not be ignored", ), + pytest.param( + TConf( + before=[modes.TransparentProxy, ServerQuicLayer, ClientQuicLayer], + after=[ + modes.TransparentProxy, + ServerQuicLayer, + ClientQuicLayer, + RawQuicLayer, + ], + data_client=b"", + alpn=b"doq", + tls_version="QUICv1", + ), + id=f"transparent proxy: non-http quic", + ), ] @@ -835,6 +844,7 @@ def test_next_layer( ) ctx.server.address = test_conf.server_address ctx.client.transport_protocol = test_conf.transport_protocol + ctx.client.tls_version = test_conf.tls_version ctx.client.proxy_mode = ProxyMode.parse(test_conf.proxy_mode) ctx.layers = [x(ctx) for x in test_conf.before] nl._next_layer( @@ -847,3 +857,19 @@ def test_next_layer( last_layer = ctx.layers[-1] if isinstance(last_layer, (UDPLayer, TCPLayer)): assert bool(last_layer.flow) ^ test_conf.ignore_conn + + +def test_starts_like_quic(): + assert not _starts_like_quic(b"", ("192.0.2.1", 443)) + assert not _starts_like_quic(dtls_client_hello_with_extensions, ("192.0.2.1", 443)) + + # Long Header - we can get definite answers from version numbers. + assert _starts_like_quic(quic_client_hello, None) + quic_version_negotation_grease = bytes.fromhex( + "ca0a0a0a0a08c0618c84b54541320823fcce946c38d8210044e6a93bbb283593f75ffb6f2696b16cfdcb5b1255" + ) + assert _starts_like_quic(quic_version_negotation_grease, None) + + # Short Header - port-based is the best we can do. + assert _starts_like_quic(quic_short_header_packet, ("192.0.2.1", 443)) + assert not _starts_like_quic(quic_short_header_packet, ("192.0.2.1", 444)) diff --git a/test/mitmproxy/proxy/layers/quic/test__raw_layers.py b/test/mitmproxy/proxy/layers/quic/test__raw_layers.py index 6531279c4..4036ce138 100644 --- a/test/mitmproxy/proxy/layers/quic/test__raw_layers.py +++ b/test/mitmproxy/proxy/layers/quic/test__raw_layers.py @@ -27,10 +27,9 @@ from test.mitmproxy.proxy.layers.quic.test__stream_layers import TlsEchoLayer class TestQuicStreamLayer: - def test_ignored(self, tctx: context.Context): + def test_force_raw(self, tctx: context.Context): quic_layer = QuicStreamLayer(tctx, True, 1) assert isinstance(quic_layer.child_layer, layers.TCPLayer) - assert not quic_layer.child_layer.flow quic_layer.child_layer.flow = TCPFlow(tctx.client, tctx.server) quic_layer.refresh_metadata() assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False @@ -57,9 +56,9 @@ class TestQuicStreamLayer: class TestRawQuicLayer: - @pytest.mark.parametrize("ignore", [True, False]) - def test_error(self, tctx: context.Context, ignore: bool): - quic_layer = RawQuicLayer(tctx, ignore=ignore) + @pytest.mark.parametrize("force_raw", [True, False]) + def test_error(self, tctx: context.Context, force_raw: bool): + quic_layer = RawQuicLayer(tctx, force_raw=force_raw) assert ( tutils.Playbook(quic_layer) << commands.OpenConnection(tctx.server) @@ -68,10 +67,10 @@ class TestRawQuicLayer: ) assert quic_layer._handle_event == quic_layer.done - def test_ignored(self, tctx: context.Context): - quic_layer = RawQuicLayer(tctx, ignore=True) + def test_force_raw(self, tctx: context.Context): + quic_layer = RawQuicLayer(tctx, force_raw=True) assert ( - tutils.Playbook(quic_layer) + tutils.Playbook(quic_layer, hooks=False) << commands.OpenConnection(tctx.server) >> tutils.reply(None) >> events.DataReceived(tctx.client, b"msg1") diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index edeca9c34..5d82c26ae 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -23,7 +23,6 @@ export interface OptionsState { content_view_lines_cutoff: number; dns_name_servers: string[]; dns_use_hosts_file: boolean; - experimental_transparent_http3: boolean; export_preserve_original_ip: boolean; hardump: string; http2: boolean; @@ -126,7 +125,6 @@ export const defaultState: OptionsState = { content_view_lines_cutoff: 512, dns_name_servers: [], dns_use_hosts_file: true, - experimental_transparent_http3: false, export_preserve_original_ip: false, hardump: "", http2: true,