From 81f5636389e15d345a25a5374078b2abeab73abe Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Fri, 3 Oct 2014 07:29:44 -0300 Subject: [PATCH] --replay-ignore-content & --replay-ignore-param ported from branch 0.10 --- libmproxy/cmdline.py | 13 ++++++++++ libmproxy/dump.py | 8 +++++- libmproxy/flow.py | 35 ++++++++++++++++++++------ test/test_flow.py | 58 +++++++++++++++++++++++++++++++++++++------- 4 files changed, 97 insertions(+), 17 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index a3cf07ef6..f6cd1ab84 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -176,6 +176,8 @@ def get_common_options(options): wfile=options.wfile, verbosity=options.verbose, nopop=options.nopop, + replay_ignore_content = options.replay_ignore_content, + replay_ignore_params = options.replay_ignore_params ) @@ -369,6 +371,17 @@ def common_options(parser): help="Disable response pop from response flow. " "This makes it possible to replay same response multiple times." ) + group.add_argument( + "--replay-ignore-content", + action="store_true", dest="replay_ignore_content", default=False, + help="Ignore request's content while searching for a saved flow to replay" + ) + group.add_argument( + "--replay-ignore-param", + action="append", dest="replay_ignore_params", type=str, + help="Request's parameters to be ignored while searching for a saved flow to replay" + "Can be passed multiple times." + ) group = parser.add_argument_group( "Replacements", diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 0349a3b58..ccb2b5b54 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -34,6 +34,8 @@ class Options(object): "stream_large_bodies", "verbosity", "wfile", + "replay_ignore_content", + "replay_ignore_params", ] def __init__(self, **kwargs): for k, v in kwargs.items(): @@ -69,6 +71,8 @@ class DumpMaster(flow.FlowMaster): self.anticache = options.anticache self.anticomp = options.anticomp self.showhost = options.showhost + self.replay_ignore_params = options.replay_ignore_params + self.replay_ignore_content = options.replay_ignore_content self.refresh_server_playback = options.refresh_server_playback self.set_stream_large_bodies(options.stream_large_bodies) @@ -106,7 +110,9 @@ class DumpMaster(flow.FlowMaster): self._readflow(options.server_replay), options.kill, options.rheaders, not options.keepserving, - options.nopop + options.nopop, + options.replay_ignore_params, + options.replay_ignore_content ) if options.client_replay: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 4dc3a2729..440798bc2 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -12,6 +12,7 @@ from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle from .proxy.config import parse_host_pattern +import urlparse ODict = odict.ODict ODictCaseless = odict.ODictCaseless @@ -193,12 +194,12 @@ class ClientPlaybackState: class ServerPlaybackState: - def __init__(self, headers, flows, exit, nopop): + def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content): """ headers: Case-insensitive list of request headers that should be included in request-response matching. """ - self.headers, self.exit, self.nopop = headers, exit, nopop + self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content = headers, exit, nopop, ignore_params, ignore_content self.fmap = {} for i in flows: if i.response: @@ -213,14 +214,29 @@ class ServerPlaybackState: Calculates a loose hash of the flow request. """ r = flow.request + + _, _, path, _, query, _ = urlparse.urlparse(r.url) + queriesArray = urlparse.parse_qsl(query) + + filtered = [] + for p in queriesArray: + if p[0] not in self.ignore_params: + filtered.append(p) + key = [ str(r.host), str(r.port), str(r.scheme), str(r.method), - str(r.path), - str(r.content), - ] + str(path), + ] + if not self.ignore_content: + key.append(str(r.content)) + + for p in filtered: + key.append(p[0]) + key.append(p[1]) + if self.headers: hdrs = [] for i in self.headers: @@ -449,6 +465,9 @@ class FlowMaster(controller.Master): self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() + self.replay_ignore_params = False + self.replay_ignore_content = None + self.stream = None self.apps = AppRegistry() @@ -539,12 +558,14 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback(self, flows, kill, headers, exit, nopop): + def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content): """ flows: List of flows. kill: Boolean, should we kill requests not part of the replay? + ignore_params: list of parameters to ignore in server replay + ignore_content: true if request content should be ignored in server replay """ - self.server_playback = ServerPlaybackState(headers, flows, exit, nopop) + self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content) self.kill_nonreplay = kill def stop_server_playback(self): diff --git a/test/test_flow.py b/test/test_flow.py index c5254d117..b74119ddc 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -111,7 +111,7 @@ class TestClientPlaybackState: class TestServerPlaybackState: def test_hash(self): - s = flow.ServerPlaybackState(None, [], False, False) + s = flow.ServerPlaybackState(None, [], False, False, None, False) r = tutils.tflow() r2 = tutils.tflow() @@ -123,7 +123,7 @@ class TestServerPlaybackState: assert s._hash(r) != s._hash(r2) def test_headers(self): - s = flow.ServerPlaybackState(["foo"], [], False, False) + s = flow.ServerPlaybackState(["foo"], [], False, False, None, False) r = tutils.tflow(resp=True) r.request.headers["foo"] = ["bar"] r2 = tutils.tflow(resp=True) @@ -144,7 +144,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, False) + s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False) assert s.count() == 2 assert len(s.fmap.keys()) == 1 @@ -165,13 +165,53 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, True) + s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False) assert s.count() == 2 s.next_flow(r) assert s.count() == 2 + def test_ignore_params(self): + s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False) + r = tutils.tflow(resp=True) + r.request.path="/test?param1=1" + r2 = tutils.tflow(resp=True) + r2.request.path="/test" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param1=2" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param2=1" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param3=2" + assert not s._hash(r) == s._hash(r2) + + def test_ignore_content(self): + s = flow.ServerPlaybackState(None, [], False, False, None, False) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + + r.request.content = "foo" + r2.request.content = "foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = "bar" + assert not s._hash(r) == s._hash(r2) + + #now ignoring content + s = flow.ServerPlaybackState(None, [], False, False, None, True) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + r.request.content = "foo" + r2.request.content = "foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = "bar" + assert s._hash(r) == s._hash(r2) + r2.request.content = "" + assert s._hash(r) == s._hash(r2) + r2.request.content = None + assert s._hash(r) == s._hash(r2) + + class TestFlow: def test_copy(self): f = tutils.tflow(resp=True) @@ -640,7 +680,7 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] fm = flow.FlowMaster(None, s) - assert not fm.start_server_playback(pb, False, [], False, False) + assert not fm.start_server_playback(pb, False, [], False, False, None, False) assert not fm.start_client_playback(pb, False) q = Queue.Queue() @@ -662,16 +702,16 @@ class TestFlowMaster: fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], False, False) + fm.start_server_playback(pb, False, [], False, False, None, False) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False) + fm.start_server_playback(pb, False, [], True, False, None, False) r = tutils.tflow() r.request.content = "gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False) + fm.start_server_playback(pb, False, [], True, False, None, False) q = Queue.Queue() fm.tick(q, 0) assert fm.should_exit.is_set() @@ -686,7 +726,7 @@ class TestFlowMaster: pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True - fm.start_server_playback(pb, True, [], False, False) + fm.start_server_playback(pb, True, [], False, False, None, False) f = tutils.tflow() f.request.host = "nonexistent"