Add --norefresh to stop refreshing server playback to mitmdump.

Also, make cookie parsing for refreshing more error-tolerant.

libmproxy/dump.py

@@ -10,6 +10,7 @@ class Options(object):
         "client_replay",
         "keepserving",
         "kill",
+        "refresh_server_playback",
         "request_script",
         "response_script",
         "rheaders",
@@ -85,6 +86,7 @@ class DumpMaster(flow.FlowMaster):
             )
 
         self.anticache = options.anticache
+        self.refresh_server_playback = options.refresh_server_playback
 
     def _readflow(self, path):
         path = os.path.expanduser(path)

libmproxy/flow.py

@@ -140,6 +140,8 @@ class StickyCookieState:
 
     def handle_response(self, f):
         for i in f.response.headers.get("set-cookie", []):
+            # FIXME: We now know that Cookie.py screws up some cookies with
+            # valid RFC 822/1123 datetime specifications for expiry. Sigh.
             c = Cookie.SimpleCookie(i)
             m = c.values()[0]
             k = self.ckey(m, f)
@@ -432,7 +434,9 @@ class FlowMaster(controller.Master):
         self.scripts = {}
         self.kill_nonreplay = False
         self.stickycookie_state = False
+
         self.anticache = False
+        self.refresh_server_playback = False
 
     def _runscript(self, f, script):
         #begin nocover
@@ -480,6 +484,8 @@ class FlowMaster(controller.Master):
             response = proxy.Response.from_state(flow.request, rflow.response.get_state())
             response.set_replay()
             flow.response = response
+            if self.refresh_server_playback:
+                response.refresh()
             flow.request.ack(response)
             return True
         return None

libmproxy/proxy.py

@@ -5,7 +5,7 @@
 
     Development started from Neil Schemenauer's munchy.py
 """
-import sys, os, string, socket, urlparse, re, select, copy, base64, time
+import sys, os, string, socket, urlparse, re, select, copy, base64, time, Cookie
 from email.utils import parsedate_tz, formatdate, mktime_tz
 import shutil, tempfile
 import optparse, SocketServer, ssl
@@ -281,6 +281,28 @@ class Response(controller.Msg):
         controller.Msg.__init__(self)
         self.replay = False
 
+    def _refresh_cookie(self, c, delta):
+        """
+            Takes a cookie string c and a time delta in seconds, and returns
+            a refreshed cookie string.
+        """
+        c = Cookie.SimpleCookie(str(c))
+        for i in c.values():
+            if "expires" in i:
+                d = parsedate_tz(i["expires"])
+                if d:
+                    d = mktime_tz(d) + delta
+                    i["expires"] = formatdate(d)
+                else:
+                    # This can happen when the expires tag is invalid.
+                    # reddit.com sends a an expires tag like this: "Thu, 31 Dec
+                    # 2037 23:59:59 GMT", which is valid RFC 1123, but not
+                    # strictly correct according tot he cookie spec. Browsers
+                    # appear to parse this tolerantly - maybe we should too.
+                    # For now, we just ignore this.
+                    del i["expires"]
+        return c.output(header="").strip()
+
     def refresh(self, now=None):
         """
             This fairly complex and heuristic function refreshes a server
@@ -302,8 +324,11 @@ class Response(controller.Msg):
                 d = parsedate_tz(self.headers[i][0])
                 new = mktime_tz(d) + delta
                 self.headers[i] = [formatdate(new)]
+        c = []
         for i in self.headers.get("set-cookie", []):
-            pass
+            c.append(self._refresh_cookie(i, delta))
+        if c:
+            self.headers["set-cookie"] = c
 
     def set_replay(self):
         self.replay = True

mitmdump

@@ -107,6 +107,12 @@ if __name__ == '__main__':
         help="Request headers to be considered during replay. "
            "Can be passed multiple times."
     )
+    group.add_option(
+        "--norefresh",
+        action="store_true", dest="norefresh", default=False,
+        help= "Disable response refresh, "
+        "which updates times in cookies and headers for replayed responses."
+    )
     parser.add_option_group(group)
 
     proxy.certificate_option_group(parser)
@@ -136,7 +142,8 @@ if __name__ == '__main__':
         client_replay = options.client_replay,
         keepserving = options.keepserving,
         stickycookie = stickycookie,
-        anticache = options.anticache
+        anticache = options.anticache,
+        refresh_server_playback = not options.norefresh,
     )
     if args:
         filt = " ".join(args)

test/test_flow.py

@@ -405,7 +405,6 @@ class uFlowMaster(libpry.AutoTree):
 
         fm.handle_error(proxy.Error(f.request, "error"))
 
-
     def test_server_playback(self):
         s = flow.State()
 
@@ -414,6 +413,7 @@ class uFlowMaster(libpry.AutoTree):
         pb = [f]
 
         fm = flow.FlowMaster(None, s)
+        fm.refresh_server_playback = True
         assert not fm.do_server_playback(tutils.tflow())
 
         fm.start_server_playback(pb, False, [], False)

test/test_proxy.py

@@ -132,6 +132,17 @@ class uResponse(libpry.AutoTree):
         r.headers["set-cookie"] = ["MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"]
         r.refresh()
 
+    def test_refresh_cookie(self):
+        r = tutils.tresp()
+
+        # Invalid expires format, sent to us by Reddit.
+        c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/"
+        assert r._refresh_cookie(c, 60)
+
+        c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
+        assert "00:21:38" in r._refresh_cookie(c, 60)
+
+
     def test_getset_state(self):
         h = utils.Headers()
         h["test"] = ["test"]