From fb7526fc4f133e9c8c0957ed4d6ab35972110e6d Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Fri, 11 Jan 2013 15:05:40 -0600 Subject: [PATCH 1/4] fix some syntax / formatting in the docs --- doc-src/scripting/inlinescripts.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index 860ad9b64..7abf743c6 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -1,4 +1,3 @@ - __mitmproxy__ has a powerful scripting API that allows you to modify flows on-the-fly or rewrite previously saved flows locally. @@ -22,13 +21,13 @@ response itself. Called once on startup, before any other events. -###clientconnect(ScriptContext, ClientConnect) +### clientconnect(ScriptContext, ClientConnect) Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests. -###request(ScriptContext, Flow) +### request(ScriptContext, Flow) Called when a client request has been received. The __Flow__ object is guaranteed to have a non-None __request__ attribute. From a658dba4106ddcd84f2a5275efa51867e6b5fa0a Mon Sep 17 00:00:00 2001 From: Jakub Nawalaniec Date: Sun, 13 Jan 2013 20:54:16 +0100 Subject: [PATCH 2/4] New example - standalone proxy injecting iframes into responses --- examples/iframe_injector | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 examples/iframe_injector diff --git a/examples/iframe_injector b/examples/iframe_injector new file mode 100755 index 000000000..6dd28674c --- /dev/null +++ b/examples/iframe_injector @@ -0,0 +1,50 @@ +#!/usr/bin/env python +""" + Zap encoding in requests and inject iframe after body tag in html responses. + Usage: + iframe_injector http://someurl/somefile.html +""" +from libmproxy import controller, proxy +import os +import sys + + +class InjectingMaster(controller.Master): + def __init__(self, server, iframe_url): + controller.Master.__init__(self, server) + self._iframe_url = iframe_url + + def run(self): + try: + return controller.Master.run(self) + except KeyboardInterrupt: + self.shutdown() + + def handle_request(self, msg): + if 'Accept-Encoding' in msg.headers: + msg.headers["Accept-Encoding"] = 'none' + msg._ack() + + def handle_response(self, msg): + if msg.content: + c = msg.replace('', '' % self._iframe_url) + if c > 0: + print 'Iframe injected!' + msg._ack() + + +def main(argv): + if len(argv) != 2: + print "Usage: %s IFRAME_URL" % argv[0] + sys.exit(1) + iframe_url = argv[1] + config = proxy.ProxyConfig( + cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") + ) + server = proxy.ProxyServer(config, 8080) + print 'Starting proxy...' + m = InjectingMaster(server, iframe_url) + m.run() + +if __name__ == '__main__': + main(sys.argv) From 20fa6a30839500207d7d509fe3b8697dbd22a33e Mon Sep 17 00:00:00 2001 From: Rouli Date: Thu, 17 Jan 2013 17:32:56 +0200 Subject: [PATCH 3/4] changing requests and responses to have two timestamps, one marking their initiation, and the other their complete --- libmproxy/flow.py | 40 ++++++++++++++++++++++++++-------------- libmproxy/proxy.py | 13 ++++++++----- test/test_server.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 19 deletions(-) mode change 100644 => 100755 libmproxy/flow.py mode change 100644 => 100755 libmproxy/proxy.py diff --git a/libmproxy/flow.py b/libmproxy/flow.py old mode 100644 new mode 100755 index 8e4c2117f..9a6b5527d --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -256,17 +256,20 @@ class Request(HTTPMsg): path: Path portion of the URL - timestamp: Seconds since the epoch + timestamp_start: Seconds since the epoch signifying request transmission started method: HTTP method + + timestamp_end: Seconds since the epoch signifying request transmission ended """ - def __init__(self, client_conn, httpversion, host, port, scheme, method, path, headers, content, timestamp=None): + def __init__(self, client_conn, httpversion, host, port, scheme, method, path, headers, content, timestamp_start=None, timestamp_end=None): assert isinstance(headers, ODictCaseless) self.client_conn = client_conn self.httpversion = httpversion self.host, self.port, self.scheme = host, port, scheme self.method, self.path, self.headers, self.content = method, path, headers, content - self.timestamp = timestamp or utils.timestamp() + self.timestamp_start = timestamp_start or utils.timestamp() + self.timestamp_end = max(timestamp_end or utils.timestamp(), timestamp_start) self.close = False controller.Msg.__init__(self) @@ -330,7 +333,8 @@ class Request(HTTPMsg): self.path = state["path"] self.headers = ODictCaseless._from_state(state["headers"]) self.content = state["content"] - self.timestamp = state["timestamp"] + self.timestamp_start = state["timestamp_start"] + self.timestamp_end = state["timestamp_end"] def _get_state(self): return dict( @@ -343,7 +347,8 @@ class Request(HTTPMsg): path = self.path, headers = self.headers._get_state(), content = self.content, - timestamp = self.timestamp, + timestamp_start = self.timestamp_start, + timestamp_end = self.timestamp_end ) @classmethod @@ -358,7 +363,8 @@ class Request(HTTPMsg): str(state["path"]), ODictCaseless._from_state(state["headers"]), state["content"], - state["timestamp"] + state["timestamp_start"], + state["timestamp_end"], ) def __hash__(self): @@ -545,15 +551,18 @@ class Response(HTTPMsg): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - timestamp: Seconds since the epoch + timestamp_start: Seconds since the epoch signifying response transmission started + + timestamp_end: Seconds since the epoch signifying response transmission ended """ - def __init__(self, request, httpversion, code, msg, headers, content, cert, timestamp=None): + def __init__(self, request, httpversion, code, msg, headers, content, cert, timestamp_start=None, timestamp_end=None): assert isinstance(headers, ODictCaseless) self.request = request self.httpversion, self.code, self.msg = httpversion, code, msg self.headers, self.content = headers, content self.cert = cert - self.timestamp = timestamp or utils.timestamp() + self.timestamp_start = timestamp_start or utils.timestamp() + self.timestamp_end = max(timestamp_end or utils.timestamp(), timestamp_start) controller.Msg.__init__(self) self.replay = False @@ -589,7 +598,7 @@ class Response(HTTPMsg): """ if not now: now = time.time() - delta = now - self.timestamp + delta = now - self.timestamp_start refresh_headers = [ "date", "expires", @@ -621,7 +630,8 @@ class Response(HTTPMsg): self.msg = state["msg"] self.headers = ODictCaseless._from_state(state["headers"]) self.content = state["content"] - self.timestamp = state["timestamp"] + self.timestamp_start = state["timestamp_start"] + self.timestamp_end = state["timestamp_end"] self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None def _get_state(self): @@ -630,9 +640,10 @@ class Response(HTTPMsg): code = self.code, msg = self.msg, headers = self.headers._get_state(), - timestamp = self.timestamp, + timestamp_start = self.timestamp_start, + timestamp_end = self.timestamp_end, cert = self.cert.to_pem() if self.cert else None, - content = self.content + content = self.content, ) @classmethod @@ -645,7 +656,8 @@ class Response(HTTPMsg): ODictCaseless._from_state(state["headers"]), state["content"], certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None, - state["timestamp"], + state["timestamp_start"], + state["timestamp_end"], ) def __eq__(self, other): diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py old mode 100644 new mode 100755 index b1ce310c2..4c57aeb0c --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -180,14 +180,16 @@ class ProxyHandler(tcp.BaseHandler): scheme, host, port = request.scheme, request.host, request.port self.server_connect(scheme, host, port) self.server_conn.send(request) + self.server_conn.rfile.reset_timestamps() httpversion, code, msg, headers, content = http.read_response( self.server_conn.rfile, request.method, self.config.body_size_limit ) response = flow.Response( - request, httpversion, code, msg, headers, content, self.server_conn.cert + request, httpversion, code, msg, headers, content, self.server_conn.cert, self.server_conn.rfile.first_byte_timestamp, utils.timestamp() ) + response = response._send(self.mqueue) if response is None: self.server_conn.terminate() @@ -266,6 +268,7 @@ class ProxyHandler(tcp.BaseHandler): self.sni = sn.decode("utf8").encode("idna") def read_request(self, client_conn): + self.rfile.reset_timestamps() if self.config.transparent_proxy: host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): @@ -289,7 +292,7 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content) + return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp()) elif self.config.reverse_proxy: line = self.get_line(self.rfile) if line == "": @@ -303,7 +306,7 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content) + return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp()) else: line = self.get_line(self.rfile) if line == "": @@ -340,7 +343,7 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content) + return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp()) else: r = http.parse_init_proxy(line) if not r: @@ -350,7 +353,7 @@ class ProxyHandler(tcp.BaseHandler): content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) - return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content) + return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp()) def read_headers(self, authenticate=False): headers = http.read_headers(self.rfile) diff --git a/test/test_server.py b/test/test_server.py index 38d33d902..76f247435 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -74,3 +74,44 @@ class TestProxy(tutils.HTTPProxTest): assert l.request.client_conn.address assert "host" in l.request.headers assert l.response.code == 304 + + def test_response_timestamps(self): + # test that we notice at least 2 sec delay between timestamps + # in response object + f = self.pathod("304:b@1k:p50,2") + assert f.status_code == 304 + + response = self.master.state.view[0].response + assert 2 <= response.timestamp_end - response.timestamp_start <= 2.2 + + def test_request_timestamps(self): + # test that we notice at least 2 sec delay between timestamps + # in request object + connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connection.connect(("127.0.0.1", self.proxy.port)) + + # call pathod server, wait a second to complete the request + connection.send("GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n"%self.server.port) + sleep(2.1) + connection.send("\r\n"); + connection.recv(50000) + connection.close() + + request, response = self.master.state.view[0].request, self.master.state.view[0].response + assert response.code == 304 # sanity test for our low level request + assert 2 <= request.timestamp_end - request.timestamp_start <= 2.2 + + def test_request_timestamps_not_affected_by_client_time(self): + # test that don't include user wait time in request's timestamps + + f = self.pathod("304:b@10k") + assert f.status_code == 304 + sleep(1) + f = self.pathod("304:b@10k") + assert f.status_code == 304 + + request = self.master.state.view[0].request + assert request.timestamp_end - request.timestamp_start <= 0.1 + + request = self.master.state.view[1].request + assert request.timestamp_end - request.timestamp_start <= 0.1 From 6212b69fb4f137db0c72bb8f3126f530c3f0daee Mon Sep 17 00:00:00 2001 From: Rouli Date: Thu, 17 Jan 2013 17:36:18 +0200 Subject: [PATCH 4/4] fixes due to merge with master --- test/test_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_server.py b/test/test_server.py index 76f247435..ee6873c7c 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,5 +1,6 @@ from netlib import tcp -import tutils +from time import sleep +import tutils, socket """ Note that the choice of response code in these tests matters more than you