diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index 2d53df80c..c9e188fcb 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -25,7 +25,7 @@ The new header will be added to all responses passing through the proxy. ## Events -### start(ScriptContext) +### start(ScriptContext, argv) Called once on startup, before any other events. diff --git a/doc-src/transparent/osx.html b/doc-src/transparent/osx.html index 77eea63b4..205e4c762 100644 --- a/doc-src/transparent/osx.html +++ b/doc-src/transparent/osx.html @@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080 + +Note that the **rdr** rules in the pf.conf given above only apply to inbound +traffic. This means that they will NOT redirect traffic coming from the box +running pf itself. We can't distinguish between an outbound connection from a +non-mitmproxy app, and an outbound connection from mitmproxy itself - if you +want to intercept your OSX traffic, you should use an external host to run +mitmproxy. None the less, pf is flexible to cater for a range of creative +possibilities, like intercepting traffic emanating from VMs. See the +**pf.conf** man page for more. + + + diff --git a/examples/stub.py b/examples/stub.py index 314111459..42b2935a9 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -1,8 +1,7 @@ """ This is a script stub, with definitions for all events. """ - -def start(ctx): +def start(ctx, argv): """ Called once on script startup, before any other events. """ diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index a394d7f3a..fc054b5e9 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,6 +1,8 @@ import proxy import re, filt import argparse +import shlex +import os APP_DOMAIN = "mitm" APP_IP = "1.1.1.1" @@ -143,7 +145,7 @@ def get_common_options(options): replacements = reps, setheaders = setheaders, server_replay = options.server_replay, - script = options.script, + scripts = options.scripts, stickycookie = stickycookie, stickyauth = stickyauth, showhost = options.showhost, @@ -189,6 +191,11 @@ def common_options(parser): action="store", dest="reverse_proxy", default=None, help="Reverse proxy to upstream server: http[s]://host[:port]" ) + parser.add_argument( + "-F", + action="store", dest="forward_proxy", default=None, + help="Proxy to unconditionally forward to: http[s]://host[:port]" + ) parser.add_argument( "-q", action="store_true", dest="quiet", @@ -201,8 +208,9 @@ def common_options(parser): ) parser.add_argument( "-s", - action="store", dest="script", default=None, - help="Run a script." + action="append", type=lambda x: shlex.split(x,posix=(os.name != "nt")), dest="scripts", default=[], + metavar='"script.py --bar"', + help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times." ) parser.add_argument( "-t", diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 37f5d6462..04e240d3d 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -459,7 +459,7 @@ class ConsoleMaster(flow.FlowMaster): self._run_script_method("response", s, f) if f.error: self._run_script_method("error", s, f) - s.run("done") + s.unload() self.refresh_flow(f) self.state.last_script = path diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 006303a76..951d2c2a9 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -190,7 +190,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): delta = f.response.timestamp_end - f.response.timestamp_start size = len(f.response.content) + f.response.get_header_size() - rate = utils.pretty_size(size / delta) + rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) ) d.update(dict( resp_code = f.response.code, diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 7a49cee01..124889371 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -24,7 +24,7 @@ class Options(object): "rheaders", "setheaders", "server_replay", - "script", + "scripts", "showhost", "stickycookie", "stickyauth", @@ -109,8 +109,8 @@ class DumpMaster(flow.FlowMaster): not options.keepserving ) - if options.script: - err = self.load_script(options.script) + for script_argv in options.scripts: + err = self.load_script(script_argv) if err: raise DumpError(err) @@ -221,8 +221,8 @@ class DumpMaster(flow.FlowMaster): def run(self): # pragma: no cover if self.o.rfile and not self.o.keepserving: - if self.script: - self.load_script(None) + for script in self.scripts: + self.unload_script(script) return try: return flow.FlowMaster.run(self) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 240428126..d79f28623 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -1349,7 +1349,7 @@ class FlowMaster(controller.Master): self.server_playback = None self.client_playback = None self.kill_nonreplay = False - self.script = None + self.scripts = [] self.pause_scripts = False self.stickycookie_state = False @@ -1385,37 +1385,43 @@ class FlowMaster(controller.Master): """ pass - def get_script(self, path): + def get_script(self, script_argv): """ Returns an (error, script) tuple. """ - s = script.Script(path, ScriptContext(self)) + s = script.Script(script_argv, ScriptContext(self)) try: s.load() except script.ScriptError, v: return (v.args[0], None) - ret = s.run("start") - if not ret[0] and ret[1]: - return ("Error in script start:\n\n" + ret[1][1], None) return (None, s) - def load_script(self, path): + def unload_script(self,script): + script.unload() + self.scripts.remove(script) + + def load_script(self, script_argv): """ Loads a script. Returns an error description if something went - wrong. If path is None, the current script is terminated. + wrong. """ - if path is None: - self.run_script_hook("done") - self.script = None + r = self.get_script(script_argv) + if r[0]: + return r[0] else: - r = self.get_script(path) - if r[0]: - return r[0] - else: - if self.script: - self.run_script_hook("done") - self.script = r[1] + self.scripts.append(r[1]) + def run_single_script_hook(self, script, name, *args, **kwargs): + if script and not self.pause_scripts: + ret = script.run(name, *args, **kwargs) + if not ret[0] and ret[1]: + e = "Script error:\n" + ret[1][1] + self.add_event(e, "error") + + def run_script_hook(self, name, *args, **kwargs): + for script in self.scripts: + self.run_single_script_hook(script, name, *args, **kwargs) + def set_stickycookie(self, txt): if txt: flt = filt.parse(txt) @@ -1565,13 +1571,6 @@ class FlowMaster(controller.Master): if block: rt.join() - def run_script_hook(self, name, *args, **kwargs): - if self.script and not self.pause_scripts: - ret = self.script.run(name, *args, **kwargs) - if not ret[0] and ret[1]: - e = "Script error:\n" + ret[1][1] - self.add_event(e, "error") - def handle_clientconnect(self, cc): self.run_script_hook("clientconnect", cc) cc.reply() @@ -1580,6 +1579,13 @@ class FlowMaster(controller.Master): self.run_script_hook("clientdisconnect", r) r.reply() + def handle_serverconnection(self, sc): + # To unify the mitmproxy script API, we call the script hook "serverconnect" rather than "serverconnection". + # As things are handled differently in libmproxy (ClientConnect + ClientDisconnect vs ServerConnection class), + # there is no "serverdisonnect" event at the moment. + self.run_script_hook("serverconnect", sc) + sc.reply() + def handle_error(self, r): f = self.state.add_error(r) if f: @@ -1613,8 +1619,8 @@ class FlowMaster(controller.Master): return f def shutdown(self): - if self.script: - self.load_script(None) + for script in self.scripts: + self.unload_script(script) controller.Master.shutdown(self) if self.stream: for i in self.state._flow_list: diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 75a541920..1fc289ed8 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -23,13 +23,14 @@ class Log: class ProxyConfig: - def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None): + def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None): self.certfile = certfile self.cacert = cacert self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit self.reverse_proxy = reverse_proxy + self.forward_proxy = forward_proxy self.transparent_proxy = transparent_proxy self.authenticator = authenticator self.certstore = certutils.CertStore() @@ -158,6 +159,7 @@ class ProxyHandler(tcp.BaseHandler): if not self.server_conn: try: self.server_conn = ServerConnection(self.config, scheme, host, port, sni) + self.channel.ask(self.server_conn) self.server_conn.connect() except tcp.NetLibError, v: raise ProxyError(502, v) @@ -219,7 +221,12 @@ class ProxyHandler(tcp.BaseHandler): # the case, we want to reconnect without sending an error # to the client. while 1: - sc = self.get_server_connection(cc, scheme, host, port, self.sni) + if self.config.forward_proxy: + forward_scheme, forward_host, forward_port = self.config.forward_proxy + sc = self.get_server_connection(cc, forward_scheme, forward_host, forward_port, self.sni) + else: + sc = self.get_server_connection(cc, scheme, host, port, self.sni) + sc.send(request) if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected) request.tcp_setup_timestamp = sc.tcp_setup_timestamp @@ -258,13 +265,13 @@ class ProxyHandler(tcp.BaseHandler): else: response = response_reply self.send_response(response) - if request and http.request_connection_close(request.httpversion, request.headers): + if request and http.connection_close(request.httpversion, request.headers): return # We could keep the client connection when the server # connection needs to go away. However, we want to mimic # behaviour as closely as possible to the client, so we # disconnect. - if http.response_connection_close(response.httpversion, response.headers): + if http.connection_close(response.httpversion, response.headers): return except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e: if hasattr(e, "code"): @@ -310,6 +317,17 @@ class ProxyHandler(tcp.BaseHandler): raise ProxyError(502, "Unable to generate dummy cert.") return ret + def establish_ssl(self, client_conn, host, port): + dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) + def get_line(self, fp): """ Get a line, possibly preceded by a blank. @@ -329,15 +347,7 @@ class ProxyHandler(tcp.BaseHandler): if port in self.config.transparent_proxy["sslports"]: scheme = "https" if not self.ssl_established: - dummycert = self.find_cert(client_conn, host, port, host) - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) - try: - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + self.establish_ssl(client_conn, host, port) else: scheme = "http" line = self.get_line(self.rfile) @@ -372,15 +382,7 @@ class ProxyHandler(tcp.BaseHandler): '\r\n' ) self.wfile.flush() - dummycert = self.find_cert(client_conn, host, port, host) - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) - try: - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + self.establish_ssl(client_conn, host, port) self.proxy_connect_state = (host, port, httpversion) line = self.rfile.readline(line) @@ -414,10 +416,12 @@ class ProxyHandler(tcp.BaseHandler): ) def read_request_reverse(self, client_conn): + scheme, host, port = self.config.reverse_proxy + if scheme.lower() == "https" and not self.ssl_established: + self.establish_ssl(client_conn, host, port) line = self.get_line(self.rfile) if line == "": return None - scheme, host, port = self.config.reverse_proxy r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) @@ -427,7 +431,7 @@ class ProxyHandler(tcp.BaseHandler): self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request( - client_conn, httpversion, host, port, "http", method, path, headers, content, + client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp() ) @@ -594,6 +598,13 @@ def process_proxy_options(parser, options): else: rp = None + if options.forward_proxy: + fp = utils.parse_proxy_spec(options.forward_proxy) + if not fp: + return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy) + else: + fp = None + if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): @@ -623,6 +634,7 @@ def process_proxy_options(parser, options): body_size_limit = body_size_limit, no_upstream_cert = options.no_upstream_cert, reverse_proxy = rp, + forward_proxy = fp, transparent_proxy = trans, authenticator = authenticator ) diff --git a/libmproxy/script.py b/libmproxy/script.py index 95fd5be22..c32628b0c 100644 --- a/libmproxy/script.py +++ b/libmproxy/script.py @@ -8,12 +8,12 @@ class Script: """ The instantiator should do something along this vein: - s = Script(path, master) + s = Script(argv, master) s.load() - s.run("start") """ - def __init__(self, path, ctx): - self.path, self.ctx = path, ctx + def __init__(self, argv, ctx): + self.argv = argv + self.ctx = ctx self.ns = None def load(self): @@ -23,17 +23,21 @@ class Script: Raises ScriptError on failure, with argument equal to an error message that may be a formatted traceback. """ - path = os.path.expanduser(self.path) + path = os.path.expanduser(self.argv[0]) if not os.path.exists(path): - raise ScriptError("No such file: %s"%self.path) + raise ScriptError("No such file: %s" % path) if not os.path.isfile(path): - raise ScriptError("Not a file: %s"%self.path) + raise ScriptError("Not a file: %s" % path) ns = {} try: execfile(path, ns, ns) + self.ns = ns + self.run("start", self.argv) except Exception, v: raise ScriptError(traceback.format_exc(v)) - self.ns = ns + + def unload(self): + return self.run("done") def run(self, name, *args, **kwargs): """ diff --git a/test/scripts/a.py b/test/scripts/a.py index 0a21b6193..1d5717b0e 100644 --- a/test/scripts/a.py +++ b/test/scripts/a.py @@ -1,5 +1,13 @@ +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--var', type=int) var = 0 +def start(ctx, argv): + global var + var = parser.parse_args(argv[1:]).var + def here(ctx): global var var += 1 diff --git a/test/scripts/starterr.py b/test/scripts/starterr.py index 456fecc0e..b217bdfee 100644 --- a/test/scripts/starterr.py +++ b/test/scripts/starterr.py @@ -1,3 +1,3 @@ -def start(ctx): +def start(ctx, argv): raise ValueError diff --git a/test/test_cmdline.py b/test/test_cmdline.py index e1c5c88d7..92e2adbd1 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -1,6 +1,7 @@ import argparse from libmproxy import cmdline import tutils +import os.path def test_parse_replace_hook(): @@ -39,6 +40,18 @@ def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") +def test_shlex(): + """ + shlex.split assumes posix=True by default, we do manual detection for windows. + Test whether script paths are parsed correctly + """ + absfilepath = os.path.normcase(os.path.abspath(__file__)) + + parser = argparse.ArgumentParser() + cmdline.common_options(parser) + opts = parser.parse_args(args=["-s",absfilepath]) + + assert os.path.isfile(opts.scripts[0][0]) def test_common(): parser = argparse.ArgumentParser() diff --git a/test/test_flow.py b/test/test_flow.py index 32cfb0ca4..68aa14cd3 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -544,9 +544,11 @@ class TestFlowMaster: fm = flow.FlowMaster(None, s) assert not fm.load_script(tutils.test_data.path("scripts/a.py")) assert not fm.load_script(tutils.test_data.path("scripts/a.py")) - assert not fm.load_script(None) + assert not fm.unload_script(fm.scripts[0]) + assert not fm.unload_script(fm.scripts[0]) assert fm.load_script("nonexistent") assert "ValueError" in fm.load_script(tutils.test_data.path("scripts/starterr.py")) + assert len(fm.scripts) == 0 def test_replay(self): s = flow.State() @@ -572,20 +574,27 @@ class TestFlowMaster: assert not fm.load_script(tutils.test_data.path("scripts/all.py")) req = tutils.treq() fm.handle_clientconnect(req.client_conn) - assert fm.script.ns["log"][-1] == "clientconnect" + assert fm.scripts[0].ns["log"][-1] == "clientconnect" f = fm.handle_request(req) - assert fm.script.ns["log"][-1] == "request" + assert fm.scripts[0].ns["log"][-1] == "request" resp = tutils.tresp(req) fm.handle_response(resp) - assert fm.script.ns["log"][-1] == "response" + assert fm.scripts[0].ns["log"][-1] == "response" + #load second script + assert not fm.load_script(tutils.test_data.path("scripts/all.py")) + assert len(fm.scripts) == 2 dc = flow.ClientDisconnect(req.client_conn) dc.reply = controller.DummyReply() fm.handle_clientdisconnect(dc) - assert fm.script.ns["log"][-1] == "clientdisconnect" + assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" + assert fm.scripts[1].ns["log"][-1] == "clientdisconnect" + #unload first script + fm.unload_script(fm.scripts[0]) + assert len(fm.scripts) == 1 err = flow.Error(f.request, "msg") err.reply = controller.DummyReply() fm.handle_error(err) - assert fm.script.ns["log"][-1] == "error" + assert fm.scripts[0].ns["log"][-1] == "error" def test_duplicate_flow(self): s = flow.State() diff --git a/test/test_script.py b/test/test_script.py index 407c9b4b2..9033c4fca 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -1,5 +1,7 @@ from libmproxy import script, flow import tutils +import shlex +import os class TestScript: def test_simple(self): @@ -7,11 +9,12 @@ class TestScript: fm = flow.FlowMaster(None, s) ctx = flow.ScriptContext(fm) - p = script.Script(tutils.test_data.path("scripts/a.py"), ctx) + p = script.Script(shlex.split(tutils.test_data.path("scripts/a.py")+" --var 40", posix=(os.name != "nt")), ctx) p.load() + assert "here" in p.ns - assert p.run("here") == (True, 1) - assert p.run("here") == (True, 2) + assert p.run("here") == (True, 41) + assert p.run("here") == (True, 42) ret = p.run("errargs") assert not ret[0] @@ -19,12 +22,12 @@ class TestScript: # Check reload p.load() - assert p.run("here") == (True, 1) + assert p.run("here") == (True, 41) def test_duplicate_flow(self): s = flow.State() fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py")) + fm.load_script([tutils.test_data.path("scripts/duplicate_flow.py")]) r = tutils.treq() fm.handle_request(r) assert fm.state.flow_count() == 2 @@ -37,25 +40,25 @@ class TestScript: ctx = flow.ScriptContext(fm) - s = script.Script("nonexistent", ctx) + s = script.Script(["nonexistent"], ctx) tutils.raises( "no such file", s.load ) - s = script.Script(tutils.test_data.path("scripts"), ctx) + s = script.Script([tutils.test_data.path("scripts")], ctx) tutils.raises( "not a file", s.load ) - s = script.Script(tutils.test_data.path("scripts/syntaxerr.py"), ctx) + s = script.Script([tutils.test_data.path("scripts/syntaxerr.py")], ctx) tutils.raises( script.ScriptError, s.load ) - s = script.Script(tutils.test_data.path("scripts/loaderr.py"), ctx) + s = script.Script([tutils.test_data.path("scripts/loaderr.py")], ctx) tutils.raises( script.ScriptError, s.load diff --git a/test/test_server.py b/test/test_server.py index 079ed8ce1..e9a6b7275 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -191,16 +191,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin): assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400 -class TestHTTPSNoUpstream(tservers.HTTPProxTest, CommonMixin): - ssl = True - no_upstream_cert = True - def test_cert_gen_error(self): - f = self.pathoc_raw() - f.connect((u"foo..bar".encode("utf8"), 0)) - f.request("get:/") - assert "dummy cert" in "".join(self.proxy.log) - - class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin): ssl = True certfile = True