Extract port number from authority before IDNA decode (#4410)

* Extract port from authority before IDNA decode

A UnicodeError exception may be raised if the port is present

* Update Changelog

* Test for badly formed byte input

CHANGELOG.md

@@ -51,6 +51,7 @@ If you depend on these features, please raise your voice in
 * Remove the following options: `http2_priority`, `relax_http_form_validation`, `upstream_bind_address`,
   `spoof_source_address`, and `stream_websockets`. If you depended on one of them please let us know. 
   mitmproxy never phones home, which means we don't know how prominently these options were used. (@mhils)
+* Fix IDNA host 'Bad HTTP request line' error (@grahamrobbins)
 * --- TODO: add new PRs above this line ---
 * ... and various other fixes, documentation improvements, dependency version bumps, etc.
 

mitmproxy/net/http/url.py

@@ -160,14 +160,16 @@ def parse_authority(authority: AnyStr, check: bool) -> Tuple[str, Optional[int]]
     """
     try:
         if isinstance(authority, bytes):
-            authority_str = authority.decode("idna")
+            m = _authority_re.match(authority.decode("utf-8"))
+            if not m:
+                raise ValueError
+            host = m["host"].encode("utf-8").decode("idna")
         else:
-            authority_str = authority
-        m = _authority_re.match(authority_str)
-        if not m:
-            raise ValueError
+            m = _authority_re.match(authority)
+            if not m:
+                raise ValueError
+            host = m.group("host")
 
-        host = m.group("host")
         if host.startswith("[") and host.endswith("]"):
             host = host[1:-1]
         if not is_valid_host(host):

test/mitmproxy/net/http/test_url.py

@@ -162,9 +162,11 @@ def test_default_port():
         ["127.0.0.1:443", True, ("127.0.0.1", 443)],
         ["[2001:db8:42::]:443", True, ("2001:db8:42::", 443)],
         [b"xn--aaa-pla.example:80", True, ("äaaa.example", 80)],
+        [b"xn--r8jz45g.xn--zckzah:80", True, ('例え.テスト', 80)],
         ["foo", True, ("foo", None)],
         ["foo..bar", False, ("foo..bar", None)],
         ["foo:bar", False, ("foo:bar", None)],
+        [b"foo:bar", False, ("foo:bar", None)],
         ["foo:999999999", False, ("foo:999999999", None)],
         [b"\xff", False, ('\udcff', None)]
     ]