Merge branch 'master' of ssh.github.com:mitmproxy/mitmproxy

dev

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 VENV=../venv.mitmproxy
 
-virtualenv $VENV
+python2 -m virtualenv $VENV
 source $VENV/bin/activate
 pip install --src .. -r requirements.txt
 

dev.bat

@@ -2,8 +2,11 @@
 set VENV=..\venv.mitmproxy
 
 virtualenv %VENV%
+if %errorlevel% neq 0 exit /b %errorlevel%
 call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
 pip install --src .. -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.
 echo * Created virtualenv environment in %VENV%.

examples/README

@@ -4,6 +4,7 @@ change_upstream_proxy.py  Dynamically change the upstream proxy
 dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
 dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
 filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
 iframe_injector.py        Inject configurable iframe into pages.
 modify_form.py            Modify all form submissions to add a parameter.
 modify_querystring.py     Modify all query strings to add a parameters.

examples/flowwriter.py

@@ -0,0 +1,20 @@
+import random
+import sys
+
+from libmproxy.flow import FlowWriter
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "flowriter.py filename"')
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

libmproxy/protocol/http.py

@@ -328,9 +328,22 @@ class HTTPRequest(HTTPMessage):
         )
 
     @classmethod
-    def from_stream(cls, rfile, include_body=True, body_size_limit=None):
+    def from_stream(cls, rfile, include_body=True, body_size_limit=None, wfile=None):
         """
         Parse an HTTP request from a file stream
+
+        Args:
+            rfile (file): Input file to read from
+            include_body (bool): Read response body as well
+            body_size_limit (bool): Maximum body size
+            wfile (file): If specified, HTTP Expect headers are handled automatically.
+                          by writing a HTTP 100 CONTINUE response to the stream.
+
+        Returns:
+            HTTPRequest: The HTTP request
+
+        Raises:
+            HttpError: If the input is invalid.
         """
         httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end = (
             None, None, None, None, None, None, None, None, None, None)
@@ -385,6 +398,15 @@ class HTTPRequest(HTTPMessage):
         if headers is None:
             raise http.HttpError(400, "Invalid headers")
 
+        expect_header = headers.get_first("expect")
+        if expect_header and expect_header.lower() == "100-continue" and httpversion >= (1, 1):
+            wfile.write(
+                'HTTP/1.1 100 Continue\r\n'
+                '\r\n'
+            )
+            wfile.flush()
+            del headers['expect']
+
         if include_body:
             content = http.read_http_body(rfile, headers, body_size_limit,
                                           method, None, True)
@@ -609,8 +631,10 @@ class HTTPRequest(HTTPMessage):
             host = self.headers.get_first("host")
         if not host:
             host = self.host
-        host = host.encode("idna")
-        return host
+        if host:
+            return host.encode("idna")
+        else:
+            return None
 
     def pretty_url(self, hostheader):
         if self.form_out == "authority":  # upstream proxy mode
@@ -1062,7 +1086,8 @@ class HTTPHandler(ProtocolHandler):
             try:
                 req = HTTPRequest.from_stream(
                     self.c.client_conn.rfile,
-                    body_size_limit=self.c.config.body_size_limit
+                    body_size_limit=self.c.config.body_size_limit,
+                    wfile=self.c.client_conn.wfile
                 )
             except tcp.NetLibError:
                 # don't throw an error for disconnects that happen

test/test_examples.py

@@ -11,7 +11,7 @@ def test_load_scripts():
     tmaster = tservers.TestMaster(config.ProxyConfig())
 
     for f in scripts:
-        if "har_extractor" in f:
+        if "har_extractor" in f or "flowwriter" in f:
             f += " -"
         if "iframe_injector" in f:
             f += " foo"  # one argument required

test/test_protocol_http.py

@@ -59,6 +59,14 @@ class TestHTTPRequest:
         r.update_host_header()
         assert "Host" in r.headers
 
+    def test_expect_header(self):
+        s = StringIO("GET / HTTP/1.1\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\nfoobar")
+        w = StringIO()
+        r = HTTPRequest.from_stream(s, wfile=w)
+        assert w.getvalue() == "HTTP/1.1 100 Continue\r\n\r\n"
+        assert r.content == "foo"
+        assert s.read(3) == "bar"
+
     def test_authority_form_in(self):
         s = StringIO("CONNECT oops-no-port.com HTTP/1.1")
         tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
@@ -117,6 +125,20 @@ class TestHTTPRequest:
         r = tutils.treq()
         assert repr(r)
 
+    def test_pretty_host(self):
+        r = tutils.treq()
+        assert r.pretty_host(True) == "address"
+        assert r.pretty_host(False) == "address"
+        r.headers["host"] = ["other"]
+        assert r.pretty_host(True) == "other"
+        assert r.pretty_host(False) == "address"
+        r.host = None
+        assert r.pretty_host(True) == "other"
+        assert r.pretty_host(False) is None
+        del r.headers["host"]
+        assert r.pretty_host(True) is None
+        assert r.pretty_host(False) is None
+
     def test_get_form_for_urlencoded(self):
         r = tutils.treq()
         r.headers.add("content-type", "application/x-www-form-urlencoded")