Enable HTTP/3 in transparent mode by default (#7202)

* fixup raw quic handling

* enable HTTP/3 in transparent mode by default

* fix nits

CHANGELOG.md

@@ -7,6 +7,8 @@
 
 ## Unreleased: mitmproxy next
 
+- mitmproxy now supports transparent HTTP/3 proxying.
+  ([#7202](https://github.com/mitmproxy/mitmproxy/pull/7202), @errorxyz, @meitinger, @mhils)
 - Fix endless tnetstring parsing in case of very large tnetstring
   ([#7121](https://github.com/mitmproxy/mitmproxy/pull/7121), @mik1904)
 - Tighten HTTP detection heuristic to better support custom TCP-based protocols.

mitmproxy/addons/next_layer.py

@@ -26,6 +26,7 @@ from typing import Any
 from typing import cast
 
 from mitmproxy import ctx
+from mitmproxy.connection import Address
 from mitmproxy.net.tls import starts_like_dtls_record
 from mitmproxy.net.tls import starts_like_tls_record
 from mitmproxy.proxy import layer
@@ -152,7 +153,7 @@ class NextLayer:
             server_tls.child_layer = ClientTLSLayer(context)
             return server_tls
         # 3b) QUIC
-        if udp_based and _starts_like_quic(data_client):
+        if udp_based and _starts_like_quic(data_client, context.server.address):
             server_quic = ServerQuicLayer(context)
             server_quic.child_layer = ClientQuicLayer(context)
             return server_quic
@@ -164,19 +165,16 @@ class NextLayer:
             return layers.UDPLayer(context)
 
         # 5)  Handle application protocol
-        # 5a) Is it DNS?
+        # 5a) Do we have a known ALPN negotiation?
+        if context.client.alpn:
+            if context.client.alpn in HTTP_ALPNS:
+                return layers.HttpLayer(context, HTTPMode.transparent)
+            elif context.client.tls_version == "QUICv1":
+                # TODO: Once we support more QUIC-based protocols, relax force_raw here.
+                return layers.RawQuicLayer(context, force_raw=True)
+        # 5b) Is it DNS?
         if context.server.address and context.server.address[1] in (53, 5353):
             return layers.DNSLayer(context)
-
-        # 5b) Do we have a known ALPN negotiation?
-        if context.client.alpn in HTTP_ALPNS:
-            explicit_quic_proxy = (
-                isinstance(context.client.proxy_mode, modes.ReverseMode)
-                and context.client.proxy_mode.scheme == "quic"
-            )
-            if not explicit_quic_proxy:
-                return layers.HttpLayer(context, HTTPMode.transparent)
-
         # 5c) We have no other specialized layers for UDP, so we fall back to raw forwarding.
         if udp_based:
             return layers.UDPLayer(context)
@@ -398,7 +396,7 @@ class NextLayer:
             case "quic":
                 stack /= ServerQuicLayer(context)
                 stack /= ClientQuicLayer(context)
-                stack /= RawQuicLayer(context)
+                stack /= RawQuicLayer(context, force_raw=True)
 
             case _:  # pragma: no cover
                 assert_never(spec.scheme)
@@ -430,11 +428,47 @@ class NextLayer:
         )
 
 
-def _starts_like_quic(data_client: bytes) -> bool:
+# https://www.iana.org/assignments/quic/quic.xhtml
-    # FIXME: perf
+KNOWN_QUIC_VERSIONS = {
-    try:
+    0x00000001,  # QUIC v1
-        quic_parse_client_hello_from_datagrams([data_client])
+    0x51303433,  # Google QUIC Q043
-    except ValueError:
+    0x51303436,  # Google QUIC Q046
+    0x51303530,  # Google QUIC Q050
+    0x6B3343CF,  # QUIC v2
+    0x709A50C4,  # QUIC v2 draft codepoint
+}
+
+TYPICAL_QUIC_PORTS = {80, 443, 8443}
+
+
+def _starts_like_quic(data_client: bytes, server_address: Address | None) -> bool:
+    """
+    Make an educated guess on whether this could be QUIC.
+    This turns out to be quite hard in practice as 1-RTT packets are hardly distinguishable from noise.
+
+    Returns:
+        True, if the passed bytes could be the start of a QUIC packet.
+        False, otherwise.
+    """
+    # Minimum size: 1 flag byte + 1+ packet number bytes + 16+ bytes encrypted payload
+    if len(data_client) < 18:
         return False
+    if starts_like_dtls_record(data_client):
+        return False
+    # TODO: Add more checks here to detect true negatives.
+
+    # Long Header Packets
+    if data_client[0] & 0x80:
+        version = int.from_bytes(data_client[1:5], "big")
+        if version in KNOWN_QUIC_VERSIONS:
+            return True
+        # https://www.rfc-editor.org/rfc/rfc9000.html#name-versions
+        # Versions that follow the pattern 0x?a?a?a?a are reserved for use in forcing version negotiation
+        if version & 0x0F0F0F0F == 0x0A0A0A0A:
+            return True
     else:
-        return True
+        # ¯\_(ツ)_/¯
+        # We can't even rely on the QUIC bit, see https://datatracker.ietf.org/doc/rfc9287/.
+        pass
+
+    return bool(server_address and server_address[1] in TYPICAL_QUIC_PORTS)

mitmproxy/options.py

@@ -149,12 +149,6 @@ class Options(optmanager.OptManager):
             True,
             "Enable/disable support for QUIC and HTTP/3. Enabled by default.",
         )
-        self.add_option(
-            "experimental_transparent_http3",
-            bool,
-            False,
-            "Experimental support for QUIC in transparent mode. This option is for development only and will be removed soon.",
-        )
         self.add_option(
             "http_connect_send_host_header",
             bool,

mitmproxy/proxy/layers/quic/_raw_layers.py

@@ -66,7 +66,9 @@ class QuicStreamLayer(layer.Layer):
     child_layer: layer.Layer
     """The stream's child layer."""
 
-    def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None:
+    def __init__(
+        self, context: context.Context, force_raw: bool, stream_id: int
+    ) -> None:
         # we mustn't reuse the client from the QUIC connection, as the state and protocol differs
         self.client = context.client = context.client.copy()
         self.client.transport_protocol = "tcp"
@@ -88,12 +90,9 @@ class QuicStreamLayer(layer.Layer):
         )
         self._server_stream_id: int | None = None
 
-        # ignored connections will be assigned a TCPLayer immediately
         super().__init__(context)
         self.child_layer = (
-            TCPLayer(context, ignore=True)
+            TCPLayer(context) if force_raw else QuicStreamNextLayer(context, self)
-            if ignore
-            else QuicStreamNextLayer(context, self)
         )
         self.refresh_metadata()
 
@@ -150,8 +149,8 @@ class RawQuicLayer(layer.Layer):
     This layer is responsible for de-multiplexing QUIC streams into an individual layer stack per stream.
     """
 
-    ignore: bool
+    force_raw: bool
-    """Indicates whether traffic should be routed as-is."""
+    """Indicates whether traffic should be treated as raw TCP/UDP without further protocol detection."""
     datagram_layer: layer.Layer
     """
   The layer that is handling datagrams over QUIC. It's like a child_layer, but with a forked context.
@@ -170,12 +169,12 @@ class RawQuicLayer(layer.Layer):
     next_stream_id: list[int]
     """List containing the next stream ID for all four is_unidirectional/is_client combinations."""
 
-    def __init__(self, context: context.Context, ignore: bool = False) -> None:
+    def __init__(self, context: context.Context, force_raw: bool = False) -> None:
         super().__init__(context)
-        self.ignore = ignore
+        self.force_raw = force_raw
         self.datagram_layer = (
-            UDPLayer(self.context.fork(), ignore=True)
+            UDPLayer(self.context.fork())
-            if ignore
+            if force_raw
             else layer.NextLayer(self.context.fork())
         )
         self.client_stream_ids = {}
@@ -247,7 +246,9 @@ class RawQuicLayer(layer.Layer):
 
                 # create, register and start the layer
                 stream_layer = QuicStreamLayer(
-                    self.context.fork(), self.ignore, client_stream_id
+                    self.context.fork(),
+                    force_raw=self.force_raw,
+                    stream_id=client_stream_id,
                 )
                 self.client_stream_ids[client_stream_id] = stream_layer
                 if server_stream_id is not None:

test/mitmproxy/addons/test_next_layer.py

@@ -9,11 +9,13 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from mitmproxy.addons.next_layer import _starts_like_quic
 from mitmproxy.addons.next_layer import NeedsMoreData
 from mitmproxy.addons.next_layer import NextLayer
 from mitmproxy.addons.next_layer import stack_match
 from mitmproxy.connection import Address
 from mitmproxy.connection import Client
+from mitmproxy.connection import TlsVersion
 from mitmproxy.connection import TransportProtocol
 from mitmproxy.proxy.context import Context
 from mitmproxy.proxy.layer import Layer
@@ -22,7 +24,6 @@ from mitmproxy.proxy.layers import ClientTLSLayer
 from mitmproxy.proxy.layers import DNSLayer
 from mitmproxy.proxy.layers import HttpLayer
 from mitmproxy.proxy.layers import modes
-from mitmproxy.proxy.layers import QuicStreamLayer
 from mitmproxy.proxy.layers import RawQuicLayer
 from mitmproxy.proxy.layers import ServerQuicLayer
 from mitmproxy.proxy.layers import ServerTLSLayer
@@ -31,7 +32,6 @@ from mitmproxy.proxy.layers import UDPLayer
 from mitmproxy.proxy.layers.http import HTTPMode
 from mitmproxy.proxy.layers.http import HttpStream
 from mitmproxy.proxy.layers.tls import HTTP1_ALPNS
-from mitmproxy.proxy.layers.tls import HTTP3_ALPN
 from mitmproxy.proxy.mode_specs import ProxyMode
 from mitmproxy.test import taddons
 
@@ -92,6 +92,13 @@ quic_client_hello = bytes.fromhex(
     "297c0013924e88248684fe8f2098326ce51aa6e5"
 )
 
+quic_short_header_packet = bytes.fromhex(
+    "52e23539dde270bb19f7a8b63b7bcf3cdacf7d3dc68a7e00318bfa2dac3bad12cb7d78112efb5bcb1ee8e0b347"
+    "641cccd2736577d0178b4c4c4e97a8e9e2af1d28502e58c4882223e70c4d5124c4b016855340e982c5c453d61d"
+    "7d0720be075fce3126de3f0d54dc059150e0f80f1a8db5e542eb03240b0a1db44a322fb4fd3c6f2e054b369e14"
+    "5a5ff925db617d187ec65a7f00d77651968e74c1a9ddc3c7fab57e8df821b07e103264244a3a03d17984e29933"
+)
+
 dns_query = bytes.fromhex("002a01000001000000000000076578616d706c6503636f6d0000010001")
 
 # Custom protocol with just base64-encoded messages
@@ -413,6 +420,7 @@ class TConf:
     after: list[type[Layer]]
     proxy_mode: str = "regular"
     transport_protocol: TransportProtocol = "tcp"
+    tls_version: TlsVersion = None
     data_client: bytes = b""
     data_server: bytes = b""
     ignore_hosts: Sequence[str] = ()
@@ -632,28 +640,6 @@ reverse_proxy_configs.extend(
             ),
             id="reverse proxy: quic",
         ),
-        pytest.param(
-            TConf(
-                before=[
-                    modes.ReverseProxy,
-                    ServerQuicLayer,
-                    ClientQuicLayer,
-                    RawQuicLayer,
-                    lambda ctx: QuicStreamLayer(ctx, False, 0),
-                ],
-                after=[
-                    modes.ReverseProxy,
-                    ServerQuicLayer,
-                    ClientQuicLayer,
-                    RawQuicLayer,
-                    QuicStreamLayer,
-                    TCPLayer,
-                ],
-                proxy_mode="reverse:quic://example.com",
-                alpn=HTTP3_ALPN,
-            ),
-            id="reverse proxy: quic",
-        ),
         pytest.param(
             TConf(
                 before=[modes.ReverseProxy],
@@ -688,14 +674,22 @@ transparent_proxy_configs = [
         id=f"transparent proxy: dtls",
     ),
     pytest.param(
-        TConf(
+        quic := TConf(
             before=[modes.TransparentProxy],
             after=[modes.TransparentProxy, ServerQuicLayer, ClientQuicLayer],
             data_client=quic_client_hello,
             transport_protocol="udp",
+            server_address=("192.0.2.1", 443),
         ),
         id="transparent proxy: quic",
     ),
+    pytest.param(
+        dataclasses.replace(
+            quic,
+            data_client=quic_short_header_packet,
+        ),
+        id="transparent proxy: existing quic session",
+    ),
     pytest.param(
         TConf(
             before=[modes.TransparentProxy],
@@ -802,6 +796,21 @@ transparent_proxy_configs = [
         ),
         id="wireguard proxy: dns should not be ignored",
     ),
+    pytest.param(
+        TConf(
+            before=[modes.TransparentProxy, ServerQuicLayer, ClientQuicLayer],
+            after=[
+                modes.TransparentProxy,
+                ServerQuicLayer,
+                ClientQuicLayer,
+                RawQuicLayer,
+            ],
+            data_client=b"<insert valid quic here>",
+            alpn=b"doq",
+            tls_version="QUICv1",
+        ),
+        id=f"transparent proxy: non-http quic",
+    ),
 ]
 
 
@@ -835,6 +844,7 @@ def test_next_layer(
         )
         ctx.server.address = test_conf.server_address
         ctx.client.transport_protocol = test_conf.transport_protocol
+        ctx.client.tls_version = test_conf.tls_version
         ctx.client.proxy_mode = ProxyMode.parse(test_conf.proxy_mode)
         ctx.layers = [x(ctx) for x in test_conf.before]
         nl._next_layer(
@@ -847,3 +857,19 @@ def test_next_layer(
         last_layer = ctx.layers[-1]
         if isinstance(last_layer, (UDPLayer, TCPLayer)):
             assert bool(last_layer.flow) ^ test_conf.ignore_conn
+
+
+def test_starts_like_quic():
+    assert not _starts_like_quic(b"", ("192.0.2.1", 443))
+    assert not _starts_like_quic(dtls_client_hello_with_extensions, ("192.0.2.1", 443))
+
+    # Long Header - we can get definite answers from version numbers.
+    assert _starts_like_quic(quic_client_hello, None)
+    quic_version_negotation_grease = bytes.fromhex(
+        "ca0a0a0a0a08c0618c84b54541320823fcce946c38d8210044e6a93bbb283593f75ffb6f2696b16cfdcb5b1255"
+    )
+    assert _starts_like_quic(quic_version_negotation_grease, None)
+
+    # Short Header - port-based is the best we can do.
+    assert _starts_like_quic(quic_short_header_packet, ("192.0.2.1", 443))
+    assert not _starts_like_quic(quic_short_header_packet, ("192.0.2.1", 444))

test/mitmproxy/proxy/layers/quic/test__raw_layers.py

@@ -27,10 +27,9 @@ from test.mitmproxy.proxy.layers.quic.test__stream_layers import TlsEchoLayer
 
 
 class TestQuicStreamLayer:
-    def test_ignored(self, tctx: context.Context):
+    def test_force_raw(self, tctx: context.Context):
         quic_layer = QuicStreamLayer(tctx, True, 1)
         assert isinstance(quic_layer.child_layer, layers.TCPLayer)
-        assert not quic_layer.child_layer.flow
         quic_layer.child_layer.flow = TCPFlow(tctx.client, tctx.server)
         quic_layer.refresh_metadata()
         assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False
@@ -57,9 +56,9 @@ class TestQuicStreamLayer:
 
 
 class TestRawQuicLayer:
-    @pytest.mark.parametrize("ignore", [True, False])
+    @pytest.mark.parametrize("force_raw", [True, False])
-    def test_error(self, tctx: context.Context, ignore: bool):
+    def test_error(self, tctx: context.Context, force_raw: bool):
-        quic_layer = RawQuicLayer(tctx, ignore=ignore)
+        quic_layer = RawQuicLayer(tctx, force_raw=force_raw)
         assert (
             tutils.Playbook(quic_layer)
             << commands.OpenConnection(tctx.server)
@@ -68,10 +67,10 @@ class TestRawQuicLayer:
         )
         assert quic_layer._handle_event == quic_layer.done
 
-    def test_ignored(self, tctx: context.Context):
+    def test_force_raw(self, tctx: context.Context):
-        quic_layer = RawQuicLayer(tctx, ignore=True)
+        quic_layer = RawQuicLayer(tctx, force_raw=True)
         assert (
-            tutils.Playbook(quic_layer)
+            tutils.Playbook(quic_layer, hooks=False)
             << commands.OpenConnection(tctx.server)
             >> tutils.reply(None)
             >> events.DataReceived(tctx.client, b"msg1")

web/src/js/ducks/_options_gen.ts

@@ -23,7 +23,6 @@ export interface OptionsState {
     content_view_lines_cutoff: number;
     dns_name_servers: string[];
     dns_use_hosts_file: boolean;
-    experimental_transparent_http3: boolean;
     export_preserve_original_ip: boolean;
     hardump: string;
     http2: boolean;
@@ -126,7 +125,6 @@ export const defaultState: OptionsState = {
     content_view_lines_cutoff: 512,
     dns_name_servers: [],
     dns_use_hosts_file: true,
-    experimental_transparent_http3: false,
     export_preserve_original_ip: false,
     hardump: "",
     http2: true,