Introduce an anti-compression command-line argument.

This is on by default, which means we avoid compressed content unless the -z
flag is specified.

libmproxy/cmdline.py

@@ -16,6 +16,7 @@ def get_common_options(options):
 
     return dict(
         anticache = options.anticache,
+        anticomp = options.anticomp,
         client_replay = options.client_replay,
         kill = options.kill,
         no_server = options.no_server,
@@ -108,6 +109,12 @@ def common_options(parser):
         action="store", dest="wfile", default=None,
         help="Write flows to file."
     )
+    parser.add_option(
+        "-z",
+        action="store_false", dest="anticomp", default=True,
+        help="Try to convince servers to send us un-compressed data."
+    )
+
     group = optparse.OptionGroup(parser, "Client Replay")
     group.add_option(
         "-c",

libmproxy/console.py

@@ -767,6 +767,7 @@ class ConsoleState(flow.State):
 class Options(object):
     __slots__ = [
         "anticache",
+        "anticomp",
         "client_replay",
         "debug",
         "keepserving",
@@ -846,6 +847,7 @@ class ConsoleMaster(flow.FlowMaster):
 
         self.refresh_server_playback = options.refresh_server_playback
         self.anticache = options.anticache
+        self.anticomp = options.anticomp
         self.killextra = options.kill
         self.rheaders = options.rheaders
 

libmproxy/dump.py

@@ -7,6 +7,7 @@ class DumpError(Exception): pass
 class Options(object):
     __slots__ = [
         "anticache",
+        "anticomp",
         "client_replay",
         "keepserving",
         "kill",
@@ -54,6 +55,7 @@ class DumpMaster(flow.FlowMaster):
         self.outfile = outfile
         self.o = options
         self.anticache = options.anticache
+        self.anticomp = options.anticomp
         self.refresh_server_playback = options.refresh_server_playback
 
         if filtstr:

libmproxy/flow.py

@@ -453,6 +453,7 @@ class FlowMaster(controller.Master):
         self.stickyauth_txt = None
 
         self.anticache = False
+        self.anticomp = False
         self.refresh_server_playback = False
 
     def _runscript(self, f, script):
@@ -565,6 +566,8 @@ class FlowMaster(controller.Master):
             self._runscript(f, self.scripts["request"])
         if self.anticache:
             f.request.anticache()
+        if self.anticomp:
+            f.request.anticomp()
         if self.server_playback:
             pb = self.do_server_playback(f)
             if not pb:

libmproxy/proxy.py

@@ -154,6 +154,13 @@ class Request(controller.Msg):
         for i in delheaders:
             del self.headers[i]
 
+    def anticomp(self):
+        """
+            Modifies this request to remove headers that might produce a cached
+            response. That is, we remove ETags and If-Modified-Since headers.
+        """
+        self.headers["accept-encoding"] = ["identity"]
+
     def set_replay(self):
         self.client_conn = None
 
@@ -244,7 +251,6 @@ class Request(controller.Msg):
             modifications to make sure interception works properly.
         """
         headers = self.headers.copy()
-        utils.try_del(headers, 'accept-encoding')
         utils.try_del(headers, 'proxy-connection')
         utils.try_del(headers, 'keep-alive')
         utils.try_del(headers, 'connection')

test/test_flow.py

@@ -422,6 +422,7 @@ class uFlowMaster(libpry.AutoTree):
         s = flow.State()
         fm = flow.FlowMaster(None, s)
         fm.anticache = True
+        fm.anticomp = True
         req = tutils.treq()
 
         fm.handle_clientconnect(req.client_conn)