added option for read_response to only read the headers, beginnings of implementing streamed result in mitmproxy

netlib/http.py

@@ -292,7 +292,7 @@ def parse_response_line(line):
     return (proto, code, msg)
 
 
-def read_response(rfile, method, body_size_limit):
+def read_response(rfile, method, body_size_limit, include_body=True):
     """
         Return an (httpversion, code, msg, headers, content) tuple.
     """
@@ -315,8 +315,10 @@ def read_response(rfile, method, body_size_limit):
     # Parse response body according to http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-16#section-3.3
     if method in ["HEAD", "CONNECT"] or (code in [204, 304]) or 100 <= code <= 199:
         content = ""
-    else:
+    elif include_body:
         content = read_http_body(rfile, headers, body_size_limit, False)
+    else:
+        content = None # if include_body==False then a None content means the body should be read separately
     return httpversion, code, msg, headers, content
 
 

test/test_http.py

@@ -229,10 +229,10 @@ class TestReadResponseNoContentLength(test.ServerTestBase):
         assert content == "bar\r\n\r\n"
 
 def test_read_response():
-    def tst(data, method, limit):
+    def tst(data, method, limit, include_body=True):
         data = textwrap.dedent(data)
         r = cStringIO.StringIO(data)
-        return  http.read_response(r, method, limit)
+        return  http.read_response(r, method, limit, include_body=include_body)
 
     tutils.raises("server disconnect", tst, "", "GET", None)
     tutils.raises("invalid server response", tst, "foo", "GET", None)
@@ -277,6 +277,14 @@ def test_read_response():
     """
     tutils.raises("invalid headers", tst, data, "GET", None)
 
+    data = """
+        HTTP/1.1 200 OK
+        Content-Length: 3
+
+        foo
+    """
+    assert tst(data, "GET", None, include_body=False)[4] == None
+
 
 def test_parse_url():
     assert not http.parse_url("")