From 59ec291b6cff1dfa83b316401418b6308df93aac Mon Sep 17 00:00:00 2001 From: iroiro123 Date: Thu, 18 Jun 2015 23:53:27 +0900 Subject: [PATCH 1/5] HTTP Transparent Proxy --- libmproxy/cmdline.py | 5 +++++ libmproxy/protocol/http.py | 15 ++++++++++++++- libmproxy/proxy/config.py | 7 ++++++- libmproxy/proxy/primitives.py | 8 ++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index eb24bed7a..d0e54dfe2 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -362,6 +362,11 @@ def common_options(parser): action="store_true", dest="transparent_proxy", default=False, help="Set transparent proxy mode." ) + group.add_argument( + "-H", "--http-transparent", + action="store_true", dest="http_transparent_proxy", default=False, + help="Use the Host header to connect to server." + ) group.add_argument( "-U", "--upstream", action="store", diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 9c1433865..c7479b762 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1328,7 +1328,20 @@ class HTTPHandler(ProtocolHandler): # value at flow.server_conn self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn - + + elif request.form_in == "relative": + if self.c.config.mode == "httptransparent": + h = request.headers.get_first("host") + if h is None: + raise http.HttpError( + 400, + "Invalid request: No Host header" + ) + p = http.parse_url("http://" + h) + request.host, request.port = p[1], p[2] + self.c.set_server_address((request.host, request.port)) + flow.server_conn = self.c.server_conn + return None raise http.HttpError( 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 3f5796694..2074d0bf7 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -4,7 +4,7 @@ import re from OpenSSL import SSL from netlib import http_auth, certutils, tcp from .. import utils, platform, version -from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .primitives import RegularProxyMode, HTTPTransparentProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -70,6 +70,8 @@ class ProxyConfig: self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": self.mode = UpstreamProxyMode(upstream_server) + elif mode == "httptransparent": + self.mode = HTTPTransparentProxyMode() else: self.mode = RegularProxyMode() @@ -144,6 +146,9 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy + if options.http_transparent_proxy: + c += 1 + mode = "httptransparent" if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 9e7dae9af..a9718051f 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -51,6 +51,14 @@ class RegularProxyMode(ProxyMode): return None +class HTTPTransparentProxyMode(ProxyMode): + http_form_in = "relative" + http_form_out = "relative" + + def get_upstream_server(self, client_conn): + return None + + class TransparentProxyMode(ProxyMode): http_form_in = "relative" http_form_out = "relative" From 378aa783243cf23d84a39d02dde5420beadc188b Mon Sep 17 00:00:00 2001 From: iroiro123 Date: Sat, 20 Jun 2015 21:43:50 +0900 Subject: [PATCH 2/5] Spoof mode --- libmproxy/cmdline.py | 10 +++++----- libmproxy/protocol/http.py | 9 ++++++--- libmproxy/proxy/config.py | 10 +++++----- libmproxy/proxy/primitives.py | 6 +++++- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index d0e54dfe2..5111fdd86 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -362,11 +362,6 @@ def common_options(parser): action="store_true", dest="transparent_proxy", default=False, help="Set transparent proxy mode." ) - group.add_argument( - "-H", "--http-transparent", - action="store_true", dest="http_transparent_proxy", default=False, - help="Use the Host header to connect to server." - ) group.add_argument( "-U", "--upstream", action="store", @@ -375,6 +370,11 @@ def common_options(parser): default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) + group.add_argument( + "--spoof", + action="store_true", dest="spoof_mode", default=False, + help="Use Host header to connect to HTTP server." + ) group = parser.add_argument_group( "Advanced Proxy Options", diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index c7479b762..617826981 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1330,19 +1330,22 @@ class HTTPHandler(ProtocolHandler): flow.server_conn = self.c.server_conn elif request.form_in == "relative": - if self.c.config.mode == "httptransparent": - h = request.headers.get_first("host") + if self.c.config.mode == "spoof": + # Host header + h = request.pretty_host(hostheader=True) if h is None: raise http.HttpError( 400, "Invalid request: No Host header" ) p = http.parse_url("http://" + h) - request.host, request.port = p[1], p[2] + request.host = p[1] + request.port = p[2] self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn return None + raise http.HttpError( 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( self.expected_form_in, request.form_in diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 2074d0bf7..7305d72aa 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -4,7 +4,7 @@ import re from OpenSSL import SSL from netlib import http_auth, certutils, tcp from .. import utils, platform, version -from .primitives import RegularProxyMode, HTTPTransparentProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .primitives import RegularProxyMode, SpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -70,8 +70,8 @@ class ProxyConfig: self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": self.mode = UpstreamProxyMode(upstream_server) - elif mode == "httptransparent": - self.mode = HTTPTransparentProxyMode() + elif mode == "spoof": + self.mode = SpoofMode() else: self.mode = RegularProxyMode() @@ -146,9 +146,9 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.http_transparent_proxy: + if options.spoof_mode: c += 1 - mode = "httptransparent" + mode = "spoof" if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index a9718051f..f514d37f8 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -51,13 +51,17 @@ class RegularProxyMode(ProxyMode): return None -class HTTPTransparentProxyMode(ProxyMode): +class SpoofMode(ProxyMode): http_form_in = "relative" http_form_out = "relative" def get_upstream_server(self, client_conn): return None + @property + def name(self): + return "spoof" + class TransparentProxyMode(ProxyMode): http_form_in = "relative" From fd903673299c050b7b4137aabf6b9265df3d6233 Mon Sep 17 00:00:00 2001 From: iroiro123 Date: Sun, 21 Jun 2015 00:51:56 +0900 Subject: [PATCH 3/5] SSL Spoof mode --- libmproxy/cmdline.py | 12 +++++++++++- libmproxy/protocol/http.py | 11 ++++++++--- libmproxy/proxy/config.py | 16 ++++++++++++---- libmproxy/proxy/primitives.py | 15 +++++++++++++++ libmproxy/proxy/server.py | 19 +++++++++++++++++++ 5 files changed, 65 insertions(+), 8 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 5111fdd86..08639f6db 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -373,7 +373,17 @@ def common_options(parser): group.add_argument( "--spoof", action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP server." + help="Use Host header to connect to HTTP servers." + ) + group.add_argument( + "--ssl-spoof", + action="store_true", dest="ssl_spoof_mode", default=False, + help="Use TLS SNI to connect to HTTPS servers." + ) + group.add_argument( + "--spoofed-port", + action="store", dest="spoofed_ssl_port", type=int, default=443, + help="Port number of upstream HTTPS servers in SSL spoof mode." ) group = parser.add_argument_group( diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 617826981..11436b300 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1336,13 +1336,18 @@ class HTTPHandler(ProtocolHandler): if h is None: raise http.HttpError( 400, - "Invalid request: No Host header" + "Invalid request: No host information" ) p = http.parse_url("http://" + h) - request.host = p[1] - request.port = p[2] + request.scheme = p[0] + request.host = p[1] + request.port = p[2] self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn + + if self.c.config.mode == "sslspoof": + # SNI is processed in server.py + return None return None diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 7305d72aa..07dc5c892 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -4,7 +4,7 @@ import re from OpenSSL import SSL from netlib import http_auth, certutils, tcp from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -51,7 +51,8 @@ class ProxyConfig: certforward=False, ssl_version_client="secure", ssl_version_server="secure", - ssl_ports=TRANSPARENT_SSL_PORTS + ssl_ports=TRANSPARENT_SSL_PORTS, + spoofed_ssl_port=None ): self.host = host self.port = port @@ -72,6 +73,8 @@ class ProxyConfig: self.mode = UpstreamProxyMode(upstream_server) elif mode == "spoof": self.mode = SpoofMode() + elif mode == "sslspoof": + self.mode = SSLSpoofMode(spoofed_ssl_port) else: self.mode = RegularProxyMode() @@ -128,7 +131,7 @@ def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server = None, None + mode, upstream_server, spoofed_ssl_port = None, None, None if options.transparent_proxy: c += 1 if not platform.resolver: @@ -149,6 +152,10 @@ def process_proxy_options(parser, options): if options.spoof_mode: c += 1 mode = "spoof" + if options.ssl_spoof_mode: + c += 1 + mode = "sslspoof" + spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " @@ -218,7 +225,8 @@ def process_proxy_options(parser, options): certforward=options.certforward, ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports + ssl_ports=ssl_ports, + spoofed_ssl_port=spoofed_ssl_port ) diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index f514d37f8..b01ddde32 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -63,6 +63,21 @@ class SpoofMode(ProxyMode): return "spoof" +class SSLSpoofMode(ProxyMode): + http_form_in = "relative" + http_form_out = "relative" + + def __init__(self, sslport): + self.sslport = sslport + + def get_upstream_server(self, client_conn): + return None + + @property + def name(self): + return "sslspoof" + + class TransparentProxyMode(ProxyMode): http_form_in = "relative" http_form_out = "relative" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index e1587df17..df890f7ca 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -117,6 +117,20 @@ class ConnectionHandler: self.server_conn.address(), "info") self.conntype = "tcp" + + elif not self.server_conn and self.config.mode == "sslspoof": + port = self.config.mode.sslport + self.set_server_address(("-", port)) + self.establish_ssl(client=True) + host = self.client_conn.connection.get_servername() + if host is None: + raise ProxyError( + 400, + "Invalid request: No host information" + ) + self.set_server_address((host, port)) + self.establish_server_connection() + self.establish_ssl(server=True, sni=host) # Delegate handling to the protocol handler protocol_handler( @@ -308,6 +322,9 @@ class ConnectionHandler: host = upstream_cert.cn.decode("utf8").encode("idna") if self.server_conn.sni: sans.append(self.server_conn.sni) + # for ssl spoof mode + if hasattr(self.client_conn, "sni"): + sans.append(self.client_conn.sni) ret = self.config.certstore.get_cert(host, sans) if not ret: @@ -325,6 +342,8 @@ class ConnectionHandler: if not sn: return sni = sn.decode("utf8").encode("idna") + # for ssl spoof mode + self.client_conn.sni = sni if sni != self.server_conn.sni: self.log("SNI received: %s" % sni, "debug") From 5c7fa7a594b2179fe5a223c2d8a449f0ca4c629b Mon Sep 17 00:00:00 2001 From: iroiro123 Date: Tue, 23 Jun 2015 00:57:33 +0900 Subject: [PATCH 4/5] add unit tests for spoof mode --- test/test_proxy.py | 6 +++++ test/test_server.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ test/tservers.py | 43 ++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/test/test_proxy.py b/test/test_proxy.py index a618ae6cc..d1e72f75c 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -90,6 +90,12 @@ class TestProcessProxyOptions: self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") + self.assert_noerr("--spoof") + self.assert_noerr("--ssl-spoof") + + self.assert_noerr("--spoofed-port", "443") + self.assert_err("expected one argument", "--spoofed-port") + self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") def test_client_certs(self): diff --git a/test/test_server.py b/test/test_server.py index 2ab48422b..58a4b5b5b 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -368,6 +368,60 @@ class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin): reverse = True +class TestSpoof(tservers.SpoofModeTest): + def test_http(self): + alist = ( + ("localhost", self.server.port), + ("127.0.0.1", self.server.port) + ) + for a in alist: + self.server.clear_log() + p = self.pathoc() + f = p.request("get:/p/304:h'Host'='%s:%s'" % a) + assert self.server.last_log() + assert f.status_code == 304 + l = self.master.state.view[-1] + assert l.server_conn.address + assert l.server_conn.address.host == a[0] + assert l.server_conn.address.port == a[1] + + def test_http_without_host(self): + p = self.pathoc() + f = p.request("get:/p/304:r") + assert f.status_code == 400 + + +class TestSSLSpoof(tservers.SSLSpoofModeTest): + def test_https(self): + alist = ( + ("localhost", self.server.port), + ("127.0.0.1", self.server.port) + ) + for a in alist: + self.server.clear_log() + self.config.mode.sslport = a[1] + p = self.pathoc(sni=a[0]) + f = p.request("get:/p/304") + assert self.server.last_log() + assert f.status_code == 304 + l = self.master.state.view[-1] + assert l.server_conn.address + assert l.server_conn.address.host == a[0] + assert l.server_conn.address.port == a[1] + + def test_https_without_sni(self): + a = ("localhost", self.server.port) + self.config.mode.sslport = a[1] + p = self.pathoc(sni=None) + #assert p.ssl_established == False + try: + f = p.request("get:/p/304") + #assert f.status_code == 400 + assert False + except tcp.NetLibSSLError as v: + assert True + + class TestHttps2Http(tservers.ReverseProxTest): @classmethod def get_proxy_config(cls): diff --git a/test/tservers.py b/test/tservers.py index dc14fb37c..c70ad68a0 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -270,6 +270,49 @@ class ReverseProxTest(ProxTestBase): return p.request(q) +class SpoofModeTest(ProxTestBase): + ssl = None + + @classmethod + def get_proxy_config(cls): + d = ProxTestBase.get_proxy_config() + d["upstream_server"] = None + d["mode"] = "spoof" + return d + + def pathoc(self, sni=None): + """ + Returns a connected Pathoc instance. + """ + p = libpathod.pathoc.Pathoc( + ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None + ) + p.connect() + return p + + +class SSLSpoofModeTest(ProxTestBase): + ssl = True + + @classmethod + def get_proxy_config(cls): + d = ProxTestBase.get_proxy_config() + d["upstream_server"] = None + d["mode"] = "sslspoof" + d["spoofed_ssl_port"] = 443 + return d + + def pathoc(self, sni=None): + """ + Returns a connected Pathoc instance. + """ + p = libpathod.pathoc.Pathoc( + ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None + ) + p.connect() + return p + + class ChainProxTest(ProxTestBase): """ Chain three instances of mitmproxy in a row to test upstream mode. From fbb23b5c9fae6e402d84ddae3c3b8c218def366c Mon Sep 17 00:00:00 2001 From: iroiro123 Date: Tue, 23 Jun 2015 01:49:22 +0900 Subject: [PATCH 5/5] changed error handling (ssl spoof mode) --- libmproxy/protocol/http.py | 7 ++++++- libmproxy/proxy/server.py | 12 ++++-------- test/test_server.py | 9 ++------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 11436b300..9bce72063 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1347,7 +1347,12 @@ class HTTPHandler(ProtocolHandler): if self.c.config.mode == "sslspoof": # SNI is processed in server.py - return None + if not (flow.server_conn and flow.server_conn.ssl_established): + print ":::::::::::::::" + raise http.HttpError( + 400, + "Invalid request: No host information" + ) return None diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index df890f7ca..71704413a 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -123,14 +123,10 @@ class ConnectionHandler: self.set_server_address(("-", port)) self.establish_ssl(client=True) host = self.client_conn.connection.get_servername() - if host is None: - raise ProxyError( - 400, - "Invalid request: No host information" - ) - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) + if host: + self.set_server_address((host, port)) + self.establish_server_connection() + self.establish_ssl(server=True, sni=host) # Delegate handling to the protocol handler protocol_handler( diff --git a/test/test_server.py b/test/test_server.py index 58a4b5b5b..07b8a5f23 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -413,13 +413,8 @@ class TestSSLSpoof(tservers.SSLSpoofModeTest): a = ("localhost", self.server.port) self.config.mode.sslport = a[1] p = self.pathoc(sni=None) - #assert p.ssl_established == False - try: - f = p.request("get:/p/304") - #assert f.status_code == 400 - assert False - except tcp.NetLibSSLError as v: - assert True + f = p.request("get:/p/304") + assert f.status_code == 400 class TestHttps2Http(tservers.ReverseProxTest):