diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html
index 2d53df80c..c9e188fcb 100644
--- a/doc-src/scripting/inlinescripts.html
+++ b/doc-src/scripting/inlinescripts.html
@@ -25,7 +25,7 @@ The new header will be added to all responses passing through the proxy.
## Events
-### start(ScriptContext)
+### start(ScriptContext, argv)
Called once on startup, before any other events.
diff --git a/doc-src/transparent/osx.html b/doc-src/transparent/osx.html
index 77eea63b4..205e4c762 100644
--- a/doc-src/transparent/osx.html
+++ b/doc-src/transparent/osx.html
@@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080
+
+Note that the **rdr** rules in the pf.conf given above only apply to inbound
+traffic. This means that they will NOT redirect traffic coming from the box
+running pf itself. We can't distinguish between an outbound connection from a
+non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
+want to intercept your OSX traffic, you should use an external host to run
+mitmproxy. None the less, pf is flexible to cater for a range of creative
+possibilities, like intercepting traffic emanating from VMs. See the
+**pf.conf** man page for more.
+
+
+
diff --git a/examples/stub.py b/examples/stub.py
index 314111459..42b2935a9 100644
--- a/examples/stub.py
+++ b/examples/stub.py
@@ -1,8 +1,7 @@
"""
This is a script stub, with definitions for all events.
"""
-
-def start(ctx):
+def start(ctx, argv):
"""
Called once on script startup, before any other events.
"""
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index a394d7f3a..fc054b5e9 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -1,6 +1,8 @@
import proxy
import re, filt
import argparse
+import shlex
+import os
APP_DOMAIN = "mitm"
APP_IP = "1.1.1.1"
@@ -143,7 +145,7 @@ def get_common_options(options):
replacements = reps,
setheaders = setheaders,
server_replay = options.server_replay,
- script = options.script,
+ scripts = options.scripts,
stickycookie = stickycookie,
stickyauth = stickyauth,
showhost = options.showhost,
@@ -189,6 +191,11 @@ def common_options(parser):
action="store", dest="reverse_proxy", default=None,
help="Reverse proxy to upstream server: http[s]://host[:port]"
)
+ parser.add_argument(
+ "-F",
+ action="store", dest="forward_proxy", default=None,
+ help="Proxy to unconditionally forward to: http[s]://host[:port]"
+ )
parser.add_argument(
"-q",
action="store_true", dest="quiet",
@@ -201,8 +208,9 @@ def common_options(parser):
)
parser.add_argument(
"-s",
- action="store", dest="script", default=None,
- help="Run a script."
+ action="append", type=lambda x: shlex.split(x,posix=(os.name != "nt")), dest="scripts", default=[],
+ metavar='"script.py --bar"',
+ help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
)
parser.add_argument(
"-t",
diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py
index 37f5d6462..04e240d3d 100644
--- a/libmproxy/console/__init__.py
+++ b/libmproxy/console/__init__.py
@@ -459,7 +459,7 @@ class ConsoleMaster(flow.FlowMaster):
self._run_script_method("response", s, f)
if f.error:
self._run_script_method("error", s, f)
- s.run("done")
+ s.unload()
self.refresh_flow(f)
self.state.last_script = path
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py
index 006303a76..951d2c2a9 100644
--- a/libmproxy/console/common.py
+++ b/libmproxy/console/common.py
@@ -190,7 +190,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
delta = f.response.timestamp_end - f.response.timestamp_start
size = len(f.response.content) + f.response.get_header_size()
- rate = utils.pretty_size(size / delta)
+ rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
d.update(dict(
resp_code = f.response.code,
diff --git a/libmproxy/dump.py b/libmproxy/dump.py
index 7a49cee01..124889371 100644
--- a/libmproxy/dump.py
+++ b/libmproxy/dump.py
@@ -24,7 +24,7 @@ class Options(object):
"rheaders",
"setheaders",
"server_replay",
- "script",
+ "scripts",
"showhost",
"stickycookie",
"stickyauth",
@@ -109,8 +109,8 @@ class DumpMaster(flow.FlowMaster):
not options.keepserving
)
- if options.script:
- err = self.load_script(options.script)
+ for script_argv in options.scripts:
+ err = self.load_script(script_argv)
if err:
raise DumpError(err)
@@ -221,8 +221,8 @@ class DumpMaster(flow.FlowMaster):
def run(self): # pragma: no cover
if self.o.rfile and not self.o.keepserving:
- if self.script:
- self.load_script(None)
+ for script in self.scripts:
+ self.unload_script(script)
return
try:
return flow.FlowMaster.run(self)
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index 240428126..d79f28623 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -1349,7 +1349,7 @@ class FlowMaster(controller.Master):
self.server_playback = None
self.client_playback = None
self.kill_nonreplay = False
- self.script = None
+ self.scripts = []
self.pause_scripts = False
self.stickycookie_state = False
@@ -1385,37 +1385,43 @@ class FlowMaster(controller.Master):
"""
pass
- def get_script(self, path):
+ def get_script(self, script_argv):
"""
Returns an (error, script) tuple.
"""
- s = script.Script(path, ScriptContext(self))
+ s = script.Script(script_argv, ScriptContext(self))
try:
s.load()
except script.ScriptError, v:
return (v.args[0], None)
- ret = s.run("start")
- if not ret[0] and ret[1]:
- return ("Error in script start:\n\n" + ret[1][1], None)
return (None, s)
- def load_script(self, path):
+ def unload_script(self,script):
+ script.unload()
+ self.scripts.remove(script)
+
+ def load_script(self, script_argv):
"""
Loads a script. Returns an error description if something went
- wrong. If path is None, the current script is terminated.
+ wrong.
"""
- if path is None:
- self.run_script_hook("done")
- self.script = None
+ r = self.get_script(script_argv)
+ if r[0]:
+ return r[0]
else:
- r = self.get_script(path)
- if r[0]:
- return r[0]
- else:
- if self.script:
- self.run_script_hook("done")
- self.script = r[1]
+ self.scripts.append(r[1])
+ def run_single_script_hook(self, script, name, *args, **kwargs):
+ if script and not self.pause_scripts:
+ ret = script.run(name, *args, **kwargs)
+ if not ret[0] and ret[1]:
+ e = "Script error:\n" + ret[1][1]
+ self.add_event(e, "error")
+
+ def run_script_hook(self, name, *args, **kwargs):
+ for script in self.scripts:
+ self.run_single_script_hook(script, name, *args, **kwargs)
+
def set_stickycookie(self, txt):
if txt:
flt = filt.parse(txt)
@@ -1565,13 +1571,6 @@ class FlowMaster(controller.Master):
if block:
rt.join()
- def run_script_hook(self, name, *args, **kwargs):
- if self.script and not self.pause_scripts:
- ret = self.script.run(name, *args, **kwargs)
- if not ret[0] and ret[1]:
- e = "Script error:\n" + ret[1][1]
- self.add_event(e, "error")
-
def handle_clientconnect(self, cc):
self.run_script_hook("clientconnect", cc)
cc.reply()
@@ -1580,6 +1579,13 @@ class FlowMaster(controller.Master):
self.run_script_hook("clientdisconnect", r)
r.reply()
+ def handle_serverconnection(self, sc):
+ # To unify the mitmproxy script API, we call the script hook "serverconnect" rather than "serverconnection".
+ # As things are handled differently in libmproxy (ClientConnect + ClientDisconnect vs ServerConnection class),
+ # there is no "serverdisonnect" event at the moment.
+ self.run_script_hook("serverconnect", sc)
+ sc.reply()
+
def handle_error(self, r):
f = self.state.add_error(r)
if f:
@@ -1613,8 +1619,8 @@ class FlowMaster(controller.Master):
return f
def shutdown(self):
- if self.script:
- self.load_script(None)
+ for script in self.scripts:
+ self.unload_script(script)
controller.Master.shutdown(self)
if self.stream:
for i in self.state._flow_list:
diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py
index 75a541920..1fc289ed8 100644
--- a/libmproxy/proxy.py
+++ b/libmproxy/proxy.py
@@ -23,13 +23,14 @@ class Log:
class ProxyConfig:
- def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None):
+ def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
self.certfile = certfile
self.cacert = cacert
self.clientcerts = clientcerts
self.no_upstream_cert = no_upstream_cert
self.body_size_limit = body_size_limit
self.reverse_proxy = reverse_proxy
+ self.forward_proxy = forward_proxy
self.transparent_proxy = transparent_proxy
self.authenticator = authenticator
self.certstore = certutils.CertStore()
@@ -158,6 +159,7 @@ class ProxyHandler(tcp.BaseHandler):
if not self.server_conn:
try:
self.server_conn = ServerConnection(self.config, scheme, host, port, sni)
+ self.channel.ask(self.server_conn)
self.server_conn.connect()
except tcp.NetLibError, v:
raise ProxyError(502, v)
@@ -219,7 +221,12 @@ class ProxyHandler(tcp.BaseHandler):
# the case, we want to reconnect without sending an error
# to the client.
while 1:
- sc = self.get_server_connection(cc, scheme, host, port, self.sni)
+ if self.config.forward_proxy:
+ forward_scheme, forward_host, forward_port = self.config.forward_proxy
+ sc = self.get_server_connection(cc, forward_scheme, forward_host, forward_port, self.sni)
+ else:
+ sc = self.get_server_connection(cc, scheme, host, port, self.sni)
+
sc.send(request)
if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
request.tcp_setup_timestamp = sc.tcp_setup_timestamp
@@ -258,13 +265,13 @@ class ProxyHandler(tcp.BaseHandler):
else:
response = response_reply
self.send_response(response)
- if request and http.request_connection_close(request.httpversion, request.headers):
+ if request and http.connection_close(request.httpversion, request.headers):
return
# We could keep the client connection when the server
# connection needs to go away. However, we want to mimic
# behaviour as closely as possible to the client, so we
# disconnect.
- if http.response_connection_close(response.httpversion, response.headers):
+ if http.connection_close(response.httpversion, response.headers):
return
except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
if hasattr(e, "code"):
@@ -310,6 +317,17 @@ class ProxyHandler(tcp.BaseHandler):
raise ProxyError(502, "Unable to generate dummy cert.")
return ret
+ def establish_ssl(self, client_conn, host, port):
+ dummycert = self.find_cert(client_conn, host, port, host)
+ sni = HandleSNI(
+ self, client_conn, host, port,
+ dummycert, self.config.certfile or self.config.cacert
+ )
+ try:
+ self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
+ except tcp.NetLibError, v:
+ raise ProxyError(400, str(v))
+
def get_line(self, fp):
"""
Get a line, possibly preceded by a blank.
@@ -329,15 +347,7 @@ class ProxyHandler(tcp.BaseHandler):
if port in self.config.transparent_proxy["sslports"]:
scheme = "https"
if not self.ssl_established:
- dummycert = self.find_cert(client_conn, host, port, host)
- sni = HandleSNI(
- self, client_conn, host, port,
- dummycert, self.config.certfile or self.config.cacert
- )
- try:
- self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
- except tcp.NetLibError, v:
- raise ProxyError(400, str(v))
+ self.establish_ssl(client_conn, host, port)
else:
scheme = "http"
line = self.get_line(self.rfile)
@@ -372,15 +382,7 @@ class ProxyHandler(tcp.BaseHandler):
'\r\n'
)
self.wfile.flush()
- dummycert = self.find_cert(client_conn, host, port, host)
- sni = HandleSNI(
- self, client_conn, host, port,
- dummycert, self.config.certfile or self.config.cacert
- )
- try:
- self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
- except tcp.NetLibError, v:
- raise ProxyError(400, str(v))
+ self.establish_ssl(client_conn, host, port)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
@@ -414,10 +416,12 @@ class ProxyHandler(tcp.BaseHandler):
)
def read_request_reverse(self, client_conn):
+ scheme, host, port = self.config.reverse_proxy
+ if scheme.lower() == "https" and not self.ssl_established:
+ self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
- scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
@@ -427,7 +431,7 @@ class ProxyHandler(tcp.BaseHandler):
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
- client_conn, httpversion, host, port, "http", method, path, headers, content,
+ client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
@@ -594,6 +598,13 @@ def process_proxy_options(parser, options):
else:
rp = None
+ if options.forward_proxy:
+ fp = utils.parse_proxy_spec(options.forward_proxy)
+ if not fp:
+ return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy)
+ else:
+ fp = None
+
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
@@ -623,6 +634,7 @@ def process_proxy_options(parser, options):
body_size_limit = body_size_limit,
no_upstream_cert = options.no_upstream_cert,
reverse_proxy = rp,
+ forward_proxy = fp,
transparent_proxy = trans,
authenticator = authenticator
)
diff --git a/libmproxy/script.py b/libmproxy/script.py
index 95fd5be22..c32628b0c 100644
--- a/libmproxy/script.py
+++ b/libmproxy/script.py
@@ -8,12 +8,12 @@ class Script:
"""
The instantiator should do something along this vein:
- s = Script(path, master)
+ s = Script(argv, master)
s.load()
- s.run("start")
"""
- def __init__(self, path, ctx):
- self.path, self.ctx = path, ctx
+ def __init__(self, argv, ctx):
+ self.argv = argv
+ self.ctx = ctx
self.ns = None
def load(self):
@@ -23,17 +23,21 @@ class Script:
Raises ScriptError on failure, with argument equal to an error
message that may be a formatted traceback.
"""
- path = os.path.expanduser(self.path)
+ path = os.path.expanduser(self.argv[0])
if not os.path.exists(path):
- raise ScriptError("No such file: %s"%self.path)
+ raise ScriptError("No such file: %s" % path)
if not os.path.isfile(path):
- raise ScriptError("Not a file: %s"%self.path)
+ raise ScriptError("Not a file: %s" % path)
ns = {}
try:
execfile(path, ns, ns)
+ self.ns = ns
+ self.run("start", self.argv)
except Exception, v:
raise ScriptError(traceback.format_exc(v))
- self.ns = ns
+
+ def unload(self):
+ return self.run("done")
def run(self, name, *args, **kwargs):
"""
diff --git a/test/scripts/a.py b/test/scripts/a.py
index 0a21b6193..1d5717b0e 100644
--- a/test/scripts/a.py
+++ b/test/scripts/a.py
@@ -1,5 +1,13 @@
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--var', type=int)
var = 0
+def start(ctx, argv):
+ global var
+ var = parser.parse_args(argv[1:]).var
+
def here(ctx):
global var
var += 1
diff --git a/test/scripts/starterr.py b/test/scripts/starterr.py
index 456fecc0e..b217bdfee 100644
--- a/test/scripts/starterr.py
+++ b/test/scripts/starterr.py
@@ -1,3 +1,3 @@
-def start(ctx):
+def start(ctx, argv):
raise ValueError
diff --git a/test/test_cmdline.py b/test/test_cmdline.py
index e1c5c88d7..92e2adbd1 100644
--- a/test/test_cmdline.py
+++ b/test/test_cmdline.py
@@ -1,6 +1,7 @@
import argparse
from libmproxy import cmdline
import tutils
+import os.path
def test_parse_replace_hook():
@@ -39,6 +40,18 @@ def test_parse_setheaders():
x = cmdline.parse_setheader("/foo/bar/voing")
assert x == ("foo", "bar", "voing")
+def test_shlex():
+ """
+ shlex.split assumes posix=True by default, we do manual detection for windows.
+ Test whether script paths are parsed correctly
+ """
+ absfilepath = os.path.normcase(os.path.abspath(__file__))
+
+ parser = argparse.ArgumentParser()
+ cmdline.common_options(parser)
+ opts = parser.parse_args(args=["-s",absfilepath])
+
+ assert os.path.isfile(opts.scripts[0][0])
def test_common():
parser = argparse.ArgumentParser()
diff --git a/test/test_flow.py b/test/test_flow.py
index 32cfb0ca4..68aa14cd3 100644
--- a/test/test_flow.py
+++ b/test/test_flow.py
@@ -544,9 +544,11 @@ class TestFlowMaster:
fm = flow.FlowMaster(None, s)
assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
- assert not fm.load_script(None)
+ assert not fm.unload_script(fm.scripts[0])
+ assert not fm.unload_script(fm.scripts[0])
assert fm.load_script("nonexistent")
assert "ValueError" in fm.load_script(tutils.test_data.path("scripts/starterr.py"))
+ assert len(fm.scripts) == 0
def test_replay(self):
s = flow.State()
@@ -572,20 +574,27 @@ class TestFlowMaster:
assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
req = tutils.treq()
fm.handle_clientconnect(req.client_conn)
- assert fm.script.ns["log"][-1] == "clientconnect"
+ assert fm.scripts[0].ns["log"][-1] == "clientconnect"
f = fm.handle_request(req)
- assert fm.script.ns["log"][-1] == "request"
+ assert fm.scripts[0].ns["log"][-1] == "request"
resp = tutils.tresp(req)
fm.handle_response(resp)
- assert fm.script.ns["log"][-1] == "response"
+ assert fm.scripts[0].ns["log"][-1] == "response"
+ #load second script
+ assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
+ assert len(fm.scripts) == 2
dc = flow.ClientDisconnect(req.client_conn)
dc.reply = controller.DummyReply()
fm.handle_clientdisconnect(dc)
- assert fm.script.ns["log"][-1] == "clientdisconnect"
+ assert fm.scripts[0].ns["log"][-1] == "clientdisconnect"
+ assert fm.scripts[1].ns["log"][-1] == "clientdisconnect"
+ #unload first script
+ fm.unload_script(fm.scripts[0])
+ assert len(fm.scripts) == 1
err = flow.Error(f.request, "msg")
err.reply = controller.DummyReply()
fm.handle_error(err)
- assert fm.script.ns["log"][-1] == "error"
+ assert fm.scripts[0].ns["log"][-1] == "error"
def test_duplicate_flow(self):
s = flow.State()
diff --git a/test/test_script.py b/test/test_script.py
index 407c9b4b2..9033c4fca 100644
--- a/test/test_script.py
+++ b/test/test_script.py
@@ -1,5 +1,7 @@
from libmproxy import script, flow
import tutils
+import shlex
+import os
class TestScript:
def test_simple(self):
@@ -7,11 +9,12 @@ class TestScript:
fm = flow.FlowMaster(None, s)
ctx = flow.ScriptContext(fm)
- p = script.Script(tutils.test_data.path("scripts/a.py"), ctx)
+ p = script.Script(shlex.split(tutils.test_data.path("scripts/a.py")+" --var 40", posix=(os.name != "nt")), ctx)
p.load()
+
assert "here" in p.ns
- assert p.run("here") == (True, 1)
- assert p.run("here") == (True, 2)
+ assert p.run("here") == (True, 41)
+ assert p.run("here") == (True, 42)
ret = p.run("errargs")
assert not ret[0]
@@ -19,12 +22,12 @@ class TestScript:
# Check reload
p.load()
- assert p.run("here") == (True, 1)
+ assert p.run("here") == (True, 41)
def test_duplicate_flow(self):
s = flow.State()
fm = flow.FlowMaster(None, s)
- fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py"))
+ fm.load_script([tutils.test_data.path("scripts/duplicate_flow.py")])
r = tutils.treq()
fm.handle_request(r)
assert fm.state.flow_count() == 2
@@ -37,25 +40,25 @@ class TestScript:
ctx = flow.ScriptContext(fm)
- s = script.Script("nonexistent", ctx)
+ s = script.Script(["nonexistent"], ctx)
tutils.raises(
"no such file",
s.load
)
- s = script.Script(tutils.test_data.path("scripts"), ctx)
+ s = script.Script([tutils.test_data.path("scripts")], ctx)
tutils.raises(
"not a file",
s.load
)
- s = script.Script(tutils.test_data.path("scripts/syntaxerr.py"), ctx)
+ s = script.Script([tutils.test_data.path("scripts/syntaxerr.py")], ctx)
tutils.raises(
script.ScriptError,
s.load
)
- s = script.Script(tutils.test_data.path("scripts/loaderr.py"), ctx)
+ s = script.Script([tutils.test_data.path("scripts/loaderr.py")], ctx)
tutils.raises(
script.ScriptError,
s.load
diff --git a/test/test_server.py b/test/test_server.py
index 079ed8ce1..e9a6b7275 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -191,16 +191,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
-class TestHTTPSNoUpstream(tservers.HTTPProxTest, CommonMixin):
- ssl = True
- no_upstream_cert = True
- def test_cert_gen_error(self):
- f = self.pathoc_raw()
- f.connect((u"foo..bar".encode("utf8"), 0))
- f.request("get:/")
- assert "dummy cert" in "".join(self.proxy.log)
-
-
class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin):
ssl = True
certfile = True