From 77bb0b74ed9cf156490f52a162a4d1856694db4f Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 02:44:47 +0900 Subject: [PATCH 1/4] Maybe it should work; https://github.com/mitmproxy/mitmproxy/issues/319 --- doc-src/features/responsestreaming.html | 3 +++ examples/stream_modify.py | 11 +++++++++++ libmproxy/protocol/http.py | 7 ++++++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 examples/stream_modify.py diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 47fafef7f..59847c8f7 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,6 +40,9 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ +In addition, if the .stream attribute is set to callable(), .stream will work as a hook in chunk data processing. + +$!example("examples/stream_modify.py")!$

Implementation Details

diff --git a/examples/stream_modify.py b/examples/stream_modify.py new file mode 100644 index 000000000..517f730af --- /dev/null +++ b/examples/stream_modify.py @@ -0,0 +1,11 @@ +def modify(chunks): + """ + chunks is a generator that can be used to iterate over all chunks. + Each chunk is a (prefix, content, suffix) tuple. + For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n") + """ + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo","bar"), suffix + +def responseheaders(ctx, flow): + flow.response.stream = modify diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 046d0b42e..fff2b84ff 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,12 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, + for chunk in hasattr(flow.response.stream, "__call__") and \ + flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, flow.request.method, + flow.response.code, False, 4096)) or \ + http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): From 1d42c1b3c4f97a9e0d08bf55580b17ce144bd723 Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 03:17:40 +0900 Subject: [PATCH 2/4] Modify example to notify incompatibility with --stream SIZE command line option; https://github.com/mitmproxy/mitmproxy/issues/319 --- examples/stream_modify.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/stream_modify.py b/examples/stream_modify.py index 517f730af..a28d95c7a 100644 --- a/examples/stream_modify.py +++ b/examples/stream_modify.py @@ -1,3 +1,10 @@ +""" +This inline script won't work with --stream SIZE command line option. + +That's because flow.response.stream will be overwritten to True if the +command line option exists. +""" + def modify(chunks): """ chunks is a generator that can be used to iterate over all chunks. @@ -9,3 +16,4 @@ def modify(chunks): def responseheaders(ctx, flow): flow.response.stream = modify + flow.response.stream_large_bodies = 1024 # = 1KB From 10f81e596bc0db37c62c0326ae6f7d3891f7756c Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 10:15:07 +0900 Subject: [PATCH 3/4] Change from checking __call__ to using callable; https://github.com/mitmproxy/mitmproxy/issues/319 --- doc-src/features/responsestreaming.html | 2 +- libmproxy/protocol/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 59847c8f7..20785269d 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,7 +40,7 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ -In addition, if the .stream attribute is set to callable(), .stream will work as a hook in chunk data processing. +In addition, if the .stream attribute is callable, .stream will work as a hook in chunk data processing. $!example("examples/stream_modify.py")!$ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fff2b84ff..78c4ac80a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,7 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in hasattr(flow.response.stream, "__call__") and \ + for chunk in callabe(flow.response.stream) and \ flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, From 5916767e036c6c7a816aa964bcd2b2721c7316bb Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 10:22:27 +0900 Subject: [PATCH 4/4] Correct typo; https://github.com/mitmproxy/mitmproxy/issues/319 --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 78c4ac80a..51fd503fe 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,7 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in callabe(flow.response.stream) and \ + for chunk in callable(flow.response.stream) and \ flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method,