From 88671c3aac430dfac8a609c82123b138ded22fad Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 12 Jul 2016 11:56:57 +0530 Subject: [PATCH 01/16] Add test for error (~e) --- test/mitmproxy/test_filt.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 9fe36b2ab..f6c08c674 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -73,7 +73,7 @@ class TestParsing: self._dump(a) -class TestMatching: +class TestMatchingHTTPFlow: def req(self): return tutils.tflow() @@ -247,6 +247,21 @@ class TestMatching: assert not self.q("!~c 201 !~c 200", s) +class TestMatchingTCPFlow: + + def flow(self): + return tutils.ttcpflow() + + def err(self): + return tutils.ttcpflow(err=True) + + def q(self, q, o): + return filt.parse(q)(o) + + def test_ferr(self): + e = self.err() + assert self.q("~e", e) + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): """https://github.com/mitmproxy/mitmproxy/issues/1087""" From 124b87b8517b2fefe1a9486f7f2f12b91d6d1ea9 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 12 Jul 2016 15:50:13 +0530 Subject: [PATCH 02/16] Add test for body (~b) --- mitmproxy/filt.py | 22 ++++++++++++++++------ test/mitmproxy/test_filt.py | 7 +++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index b1b72aa7c..cb49283d2 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -193,12 +193,22 @@ class FBod(_Rex): help = "Body" def __call__(self, f): - if f.request and f.request.content: - if self.re.search(f.request.get_decoded_content()): - return True - if f.response and f.response.content: - if self.re.search(f.response.get_decoded_content()): - return True + + # HTTPFlow + if hasattr(f, 'request'): + if f.request and f.request.content: + if self.re.search(f.request.get_decoded_content()): + return True + if f.response and f.response.content: + if self.re.search(f.response.get_decoded_content()): + return True + + # TCPFlow + elif hasattr(f, 'messages'): + for msg in f.messages: + if self.re.search(msg.content): + return True + return False diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index f6c08c674..5f3622e9e 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -262,6 +262,13 @@ class TestMatchingTCPFlow: e = self.err() assert self.q("~e", e) + def test_body(self): + f = self.flow() + assert not self.q("~b nonexistent", f) + assert self.q("~b hello", f) + assert self.q("~b me", f) + + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): """https://github.com/mitmproxy/mitmproxy/issues/1087""" From 4e29993c1f3dc4798dc56bc45472758c1c4e61c3 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 12 Jul 2016 16:13:42 +0530 Subject: [PATCH 03/16] Add test for addresses (~src, ~dst) --- test/mitmproxy/test_filt.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 5f3622e9e..a910a946a 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -268,6 +268,22 @@ class TestMatchingTCPFlow: assert self.q("~b hello", f) assert self.q("~b me", f) + def test_src(self): + f = self.flow() + assert self.q("~src address", f) + assert not self.q("~src foobar", f) + assert self.q("~src :22", f) + assert not self.q("~src :99", f) + assert self.q("~src address:22", f) + + def test_dst(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~dst address", f) + assert not self.q("~dst foobar", f) + assert self.q("~dst :22", f) + assert not self.q("~dst :99", f) + assert self.q("~dst address:22", f) @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): From 317355a9af6e7f9f12fcbaab96fcc92fbda4c929 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Tue, 12 Jul 2016 17:23:13 +0530 Subject: [PATCH 04/16] Add test for operators (& | !) --- test/mitmproxy/test_filt.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index a910a946a..3f6704237 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -285,6 +285,31 @@ class TestMatchingTCPFlow: assert not self.q("~dst :99", f) assert self.q("~dst address:22", f) + def test_and(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~b hello & ~b me", f) + assert not self.q("~src wrongaddress & ~b hello", f) + assert self.q("(~src :22 & ~dst :22) & ~b hello", f) + assert not self.q("(~src address:22 & ~dst :22) & ~b nonexistent", f) + assert not self.q("(~src address:22 & ~dst :99) & ~b hello", f) + + def test_or(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~b hello | ~b me", f) + assert self.q("~src :22 | ~b me", f) + assert not self.q("~src :99 | ~dst :99", f) + assert self.q("(~src :22 | ~dst :22) | ~b me", f) + + def test_not(self): + f = self.flow() + assert not self.q("! ~src :22", f) + assert self.q("! ~src :99", f) + assert self.q("!~src :99 !~src :99", f) + assert not self.q("!~src :99 !~src :22", f) + + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): """https://github.com/mitmproxy/mitmproxy/issues/1087""" From ffcdd02e960e20760edfd7bf5a09132c411e26e4 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 13 Jul 2016 15:35:24 +0530 Subject: [PATCH 05/16] Add decorator for HTTPFlow filters --- mitmproxy/filt.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index cb49283d2..76f5df5fb 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -35,11 +35,21 @@ from __future__ import absolute_import, print_function, division import re import sys +import functools + +from mitmproxy.models.http import HTTPFlow from netlib import strutils import pyparsing as pp +def http(fn): + @functools.wraps(fn) + def filter_http_only(self, flow): + return isinstance(flow, HTTPFlow) and fn(self, flow) + return filter_http_only + + class _Token(object): def dump(self, indent=0, fp=sys.stdout): @@ -69,6 +79,7 @@ class FReq(_Action): code = "q" help = "Match request with no response" + @http def __call__(self, f): if not f.response: return True @@ -78,6 +89,7 @@ class FResp(_Action): code = "s" help = "Match response" + @http def __call__(self, f): return bool(f.response) @@ -117,6 +129,7 @@ class FAsset(_Action): ] ASSET_TYPES = [re.compile(x) for x in ASSET_TYPES] + @http def __call__(self, f): if f.response: for i in self.ASSET_TYPES: @@ -129,6 +142,7 @@ class FContentType(_Rex): code = "t" help = "Content-type header" + @http def __call__(self, f): if _check_content_type(self.re, f.request): return True @@ -141,6 +155,7 @@ class FRequestContentType(_Rex): code = "tq" help = "Request Content-Type header" + @http def __call__(self, f): return _check_content_type(self.re, f.request) @@ -149,6 +164,7 @@ class FResponseContentType(_Rex): code = "ts" help = "Response Content-Type header" + @http def __call__(self, f): if f.response: return _check_content_type(self.re, f.response) @@ -160,6 +176,7 @@ class FHead(_Rex): help = "Header" flags = re.MULTILINE + @http def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -173,6 +190,7 @@ class FHeadRequest(_Rex): help = "Request header" flags = re.MULTILINE + @http def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -183,6 +201,7 @@ class FHeadResponse(_Rex): help = "Response header" flags = re.MULTILINE + @http def __call__(self, f): if f.response and self.re.search(bytes(f.response.headers)): return True @@ -216,6 +235,7 @@ class FBodRequest(_Rex): code = "bq" help = "Request body" + @http def __call__(self, f): if f.request and f.request.content: if self.re.search(f.request.get_decoded_content()): @@ -226,6 +246,7 @@ class FBodResponse(_Rex): code = "bs" help = "Response body" + @http def __call__(self, f): if f.response and f.response.content: if self.re.search(f.response.get_decoded_content()): @@ -237,6 +258,7 @@ class FMethod(_Rex): help = "Method" flags = re.IGNORECASE + @http def __call__(self, f): return bool(self.re.search(f.request.data.method)) @@ -246,6 +268,7 @@ class FDomain(_Rex): help = "Domain" flags = re.IGNORECASE + @http def __call__(self, f): return bool(self.re.search(f.request.data.host)) @@ -262,6 +285,7 @@ class FUrl(_Rex): toks = toks[1:] return klass(*toks) + @http def __call__(self, f): return self.re.search(f.request.url) @@ -294,6 +318,7 @@ class FCode(_Int): code = "c" help = "HTTP response code" + @http def __call__(self, f): if f.response and f.response.status_code == self.num: return True From 767457fda9bbfb1bb478daac7250b2d89fee2e04 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 13 Jul 2016 15:37:13 +0530 Subject: [PATCH 06/16] Add tests for HTTPFlow only filters --- test/mitmproxy/test_filt.py | 50 +++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 3f6704237..2d2b87e6f 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -268,6 +268,12 @@ class TestMatchingTCPFlow: assert self.q("~b hello", f) assert self.q("~b me", f) + # Request Body + assert not self.q("~bq whatever", f) + + # Response Body + assert not self.q("~bs whatever", f) + def test_src(self): f = self.flow() assert self.q("~src address", f) @@ -309,6 +315,50 @@ class TestMatchingTCPFlow: assert self.q("!~src :99 !~src :99", f) assert not self.q("!~src :99 !~src :22", f) + def test_request(self): + f = self.flow() + assert not self.q("~q", f) + + def test_response(self): + f = self.flow() + assert not self.q("~s", f) + + def test_headers(self): + f = self.flow() + assert not self.q("~h whatever", f) + + # Request headers + assert not self.q("~hq whatever", f) + + # Response headers + assert not self.q("~hs whatever", f) + + def test_content_type(self): + f = self.flow() + assert not self.q("~t whatever", f) + + # Request content-type + assert not self.q("~tq whatever", f) + + # Response content-type + assert not self.q("~ts whatever", f) + + def test_code(self): + f = self.flow() + assert not self.q("~c 200", f) + + def test_domain(self): + f = self.flow() + assert not self.q("~d whatever", f) + + def test_method(self): + f = self.flow() + assert not self.q("~m whatever", f) + + def test_url(self): + f = self.flow() + assert not self.q("~u whatever", f) + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): From a7bb0f5d02e0cdba628887846523a2ca48b2d0a9 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 13 Jul 2016 17:24:33 +0530 Subject: [PATCH 07/16] Add TCPFlow.match --- mitmproxy/models/tcp.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mitmproxy/models/tcp.py b/mitmproxy/models/tcp.py index e33475c23..6650141d0 100644 --- a/mitmproxy/models/tcp.py +++ b/mitmproxy/models/tcp.py @@ -7,6 +7,8 @@ from typing import List import netlib.basetypes from mitmproxy.models.flow import Flow +import six + class TCPMessage(netlib.basetypes.Serializable): @@ -53,3 +55,22 @@ class TCPFlow(Flow): def __repr__(self): return "".format(len(self.messages)) + + def match(self, f): + """ + Match this flow against a compiled filter expression. Returns True + if matched, False if not. + + If f is a string, it will be compiled as a filter expression. If + the expression is invalid, ValueError is raised. + """ + if isinstance(f, six.string_types): + from .. import filt + + f = filt.parse(f) + if not f: + raise ValueError("Invalid filter expression.") + if f: + return f(self) + + return True From 5f7d61f864c93e9a2ee5d6f3a0a1563ea0ac8741 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 11:48:11 +0530 Subject: [PATCH 08/16] Replace http decorator with one that handles different flowtypes --- mitmproxy/filt.py | 49 ++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 76f5df5fb..009712189 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -38,16 +38,21 @@ import sys import functools from mitmproxy.models.http import HTTPFlow +from mitmproxy.models.tcp import TCPFlow from netlib import strutils import pyparsing as pp -def http(fn): - @functools.wraps(fn) - def filter_http_only(self, flow): - return isinstance(flow, HTTPFlow) and fn(self, flow) - return filter_http_only +def only(*types): + def decorator(fn): + @functools.wraps(fn) + def filter_types(self, flow): + if isinstance(flow, types): + return fn(self, flow) + return False + return filter_types + return decorator class _Token(object): @@ -71,6 +76,7 @@ class FErr(_Action): code = "e" help = "Match error" + @only(HTTPFlow, TCPFlow) def __call__(self, f): return True if f.error else False @@ -79,7 +85,7 @@ class FReq(_Action): code = "q" help = "Match request with no response" - @http + @only(HTTPFlow) def __call__(self, f): if not f.response: return True @@ -89,7 +95,7 @@ class FResp(_Action): code = "s" help = "Match response" - @http + @only(HTTPFlow) def __call__(self, f): return bool(f.response) @@ -129,7 +135,7 @@ class FAsset(_Action): ] ASSET_TYPES = [re.compile(x) for x in ASSET_TYPES] - @http + @only(HTTPFlow) def __call__(self, f): if f.response: for i in self.ASSET_TYPES: @@ -142,7 +148,7 @@ class FContentType(_Rex): code = "t" help = "Content-type header" - @http + @only(HTTPFlow) def __call__(self, f): if _check_content_type(self.re, f.request): return True @@ -155,7 +161,7 @@ class FRequestContentType(_Rex): code = "tq" help = "Request Content-Type header" - @http + @only(HTTPFlow) def __call__(self, f): return _check_content_type(self.re, f.request) @@ -164,7 +170,7 @@ class FResponseContentType(_Rex): code = "ts" help = "Response Content-Type header" - @http + @only(HTTPFlow) def __call__(self, f): if f.response: return _check_content_type(self.re, f.response) @@ -176,7 +182,7 @@ class FHead(_Rex): help = "Header" flags = re.MULTILINE - @http + @only(HTTPFlow) def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -190,7 +196,7 @@ class FHeadRequest(_Rex): help = "Request header" flags = re.MULTILINE - @http + @only(HTTPFlow) def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -201,7 +207,7 @@ class FHeadResponse(_Rex): help = "Response header" flags = re.MULTILINE - @http + @only(HTTPFlow) def __call__(self, f): if f.response and self.re.search(bytes(f.response.headers)): return True @@ -211,6 +217,7 @@ class FBod(_Rex): code = "b" help = "Body" + @only(HTTPFlow, TCPFlow) def __call__(self, f): # HTTPFlow @@ -235,7 +242,7 @@ class FBodRequest(_Rex): code = "bq" help = "Request body" - @http + @only(HTTPFlow) def __call__(self, f): if f.request and f.request.content: if self.re.search(f.request.get_decoded_content()): @@ -246,7 +253,7 @@ class FBodResponse(_Rex): code = "bs" help = "Response body" - @http + @only(HTTPFlow) def __call__(self, f): if f.response and f.response.content: if self.re.search(f.response.get_decoded_content()): @@ -258,7 +265,7 @@ class FMethod(_Rex): help = "Method" flags = re.IGNORECASE - @http + @only(HTTPFlow) def __call__(self, f): return bool(self.re.search(f.request.data.method)) @@ -268,7 +275,7 @@ class FDomain(_Rex): help = "Domain" flags = re.IGNORECASE - @http + @only(HTTPFlow) def __call__(self, f): return bool(self.re.search(f.request.data.host)) @@ -285,7 +292,7 @@ class FUrl(_Rex): toks = toks[1:] return klass(*toks) - @http + @only(HTTPFlow) def __call__(self, f): return self.re.search(f.request.url) @@ -295,6 +302,7 @@ class FSrc(_Rex): help = "Match source address" is_binary = False + @only(HTTPFlow, TCPFlow) def __call__(self, f): return f.client_conn.address and self.re.search(repr(f.client_conn.address)) @@ -304,6 +312,7 @@ class FDst(_Rex): help = "Match destination address" is_binary = False + @only(HTTPFlow, TCPFlow) def __call__(self, f): return f.server_conn.address and self.re.search(repr(f.server_conn.address)) @@ -318,7 +327,7 @@ class FCode(_Int): code = "c" help = "HTTP response code" - @http + @only(HTTPFlow) def __call__(self, f): if f.response and f.response.status_code == self.num: return True From 262a42055344b9ece7b94dc8ed762d8a9a8882de Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 14:35:18 +0530 Subject: [PATCH 09/16] Use ~bq / ~bs for messages sent from client / server --- mitmproxy/filt.py | 34 ++++++++++++++++++++-------------- test/mitmproxy/test_filt.py | 16 +++++++++++----- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 009712189..e0e5606b8 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -219,18 +219,14 @@ class FBod(_Rex): @only(HTTPFlow, TCPFlow) def __call__(self, f): - - # HTTPFlow - if hasattr(f, 'request'): + if isinstance(f, HTTPFlow): if f.request and f.request.content: if self.re.search(f.request.get_decoded_content()): return True if f.response and f.response.content: if self.re.search(f.response.get_decoded_content()): return True - - # TCPFlow - elif hasattr(f, 'messages'): + elif isinstance(f, TCPFlow): for msg in f.messages: if self.re.search(msg.content): return True @@ -242,22 +238,32 @@ class FBodRequest(_Rex): code = "bq" help = "Request body" - @only(HTTPFlow) + @only(HTTPFlow, TCPFlow) def __call__(self, f): - if f.request and f.request.content: - if self.re.search(f.request.get_decoded_content()): - return True + if isinstance(f, HTTPFlow): + if f.request and f.request.content: + if self.re.search(f.request.get_decoded_content()): + return True + elif isinstance(f, TCPFlow): + for msg in f.messages: + if msg.from_client and self.re.search(msg.content): + return True class FBodResponse(_Rex): code = "bs" help = "Response body" - @only(HTTPFlow) + @only(HTTPFlow, TCPFlow) def __call__(self, f): - if f.response and f.response.content: - if self.re.search(f.response.get_decoded_content()): - return True + if isinstance(f, HTTPFlow): + if f.response and f.response.content: + if self.re.search(f.response.get_decoded_content()): + return True + elif isinstance(f, TCPFlow): + for msg in f.messages: + if not msg.from_client and self.re.search(msg.content): + return True class FMethod(_Rex): diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 2d2b87e6f..8330b68c5 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -264,15 +264,21 @@ class TestMatchingTCPFlow: def test_body(self): f = self.flow() - assert not self.q("~b nonexistent", f) + + # Messages sent by client or server assert self.q("~b hello", f) assert self.q("~b me", f) + assert not self.q("~b nonexistent", f) - # Request Body - assert not self.q("~bq whatever", f) + # Messages sent by client + assert self.q("~bq hello", f) + assert not self.q("~bq me", f) + assert not self.q("~bq nonexistent", f) - # Response Body - assert not self.q("~bs whatever", f) + # Messages sent by server + assert self.q("~bs me", f) + assert not self.q("~bs hello", f) + assert not self.q("~bs nonexistent", f) def test_src(self): f = self.flow() From 6a2668d8651b52a688ce5771d9a834473d334a64 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 16:55:09 +0530 Subject: [PATCH 10/16] Add ~http and ~tcp filters --- mitmproxy/filt.py | 22 +++++++++++++++++++++- test/mitmproxy/test_filt.py | 10 ++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index e0e5606b8..11765dcd3 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -81,6 +81,24 @@ class FErr(_Action): return True if f.error else False +class FHTTP(_Action): + code = "http" + help = "Match HTTP flows" + + @only(HTTPFlow) + def __call__(self, f): + return True + + +class FTCP(_Action): + code = "tcp" + help = "Match TCP flows" + + @only(TCPFlow) + def __call__(self, f): + return True + + class FReq(_Action): code = "q" help = "Match request with no response" @@ -384,7 +402,9 @@ filt_unary = [ FReq, FResp, FAsset, - FErr + FErr, + FHTTP, + FTCP, ] filt_rex = [ FHeadRequest, diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 8330b68c5..46e86b579 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -87,6 +87,11 @@ class TestMatchingHTTPFlow: def q(self, q, o): return filt.parse(q)(o) + def test_http(self): + s = self.req() + assert self.q("~http", s) + assert not self.q("~tcp", s) + def test_asset(self): s = self.resp() assert not self.q("~a", s) @@ -258,6 +263,11 @@ class TestMatchingTCPFlow: def q(self, q, o): return filt.parse(q)(o) + def test_tcp(self): + f = self.flow() + assert self.q("~tcp", f) + assert not self.q("~http", f) + def test_ferr(self): e = self.err() assert self.q("~e", e) From 8c49f0e784a0470a45616d631483c5aeeb1f7937 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 17:02:08 +0530 Subject: [PATCH 11/16] Add tests for TCPFlow.match --- test/mitmproxy/test_flow.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index c58a97038..67aa13020 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -293,7 +293,7 @@ class TestServerPlaybackState: assert s._hash(r) == s._hash(r2) -class TestFlow(object): +class TestHTTPFlow(object): def test_copy(self): f = tutils.tflow(resp=True) @@ -443,6 +443,20 @@ class TestFlow(object): assert f.response.content == b"abarb" +class TestTCPFlow: + + def test_match(self): + f = tutils.ttcpflow() + assert not f.match("~b nonexistent") + assert f.match(None) + assert not f.match("~b nonexistent") + + f = tutils.ttcpflow(err=True) + assert f.match("~e") + + tutils.raises(ValueError, f.match, "~") + + class TestState: def test_backup(self): From 1d2ccb9170b2ada0fc017296c89cd53aa76572d6 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 17:31:06 +0530 Subject: [PATCH 12/16] Add DummyFlow and its Tests --- test/mitmproxy/test_filt.py | 60 ++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 46e86b579..92a48cd2e 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -1,6 +1,9 @@ from six.moves import cStringIO as StringIO -from mitmproxy import filt from mock import patch + +from mitmproxy import filt +from mitmproxy.models.flow import Flow + from . import tutils @@ -376,6 +379,61 @@ class TestMatchingTCPFlow: assert not self.q("~u whatever", f) +class DummyFlow(Flow): + """ A flow that is neither HTTP nor TCP. """ + + def __init__(self): + pass + + +class TestMatchingDummyFlow: + + def flow(self): + return DummyFlow() + + def q(self, q, o): + return filt.parse(q)(o) + + def test_filters(self): + f = self.flow() + + assert not self.q("~a", f) + + assert not self.q("~b whatever", f) + assert not self.q("~bq whatever", f) + assert not self.q("~bs whatever", f) + + assert not self.q("~dst whatever", f) + + assert not self.q("~c 0", f) + + assert not self.q("~d whatever", f) + + assert not self.q("~e", f) + + assert not self.q("~http", f) + + assert not self.q("~h whatever", f) + assert not self.q("~hq whatever", f) + assert not self.q("~hs whatever", f) + + assert not self.q("~m whatever", f) + + assert not self.q("~s", f) + + assert not self.q("~src whatever", f) + + assert not self.q("~tcp", f) + + assert not self.q("~t whatever", f) + assert not self.q("~tq whatever", f) + assert not self.q("~ts whatever", f) + + assert not self.q("~u whatever", f) + + assert not self.q("~q", f) + + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): """https://github.com/mitmproxy/mitmproxy/issues/1087""" From 5082dc6cbdf8b5964ab47747f9b834f47a13fddc Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 15 Jul 2016 17:31:29 +0530 Subject: [PATCH 13/16] Sort filt items alphabetically --- mitmproxy/filt.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 11765dcd3..51e9cc307 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -175,7 +175,7 @@ class FContentType(_Rex): return False -class FRequestContentType(_Rex): +class FContentTypeRequest(_Rex): code = "tq" help = "Request Content-Type header" @@ -184,7 +184,7 @@ class FRequestContentType(_Rex): return _check_content_type(self.re, f.request) -class FResponseContentType(_Rex): +class FContentTypeResponse(_Rex): code = "ts" help = "Response Content-Type header" @@ -399,28 +399,28 @@ class FNot(_Token): filt_unary = [ - FReq, - FResp, FAsset, FErr, FHTTP, + FReq, + FResp, FTCP, ] filt_rex = [ - FHeadRequest, - FHeadResponse, - FHead, + FBod, FBodRequest, FBodResponse, - FBod, - FMethod, - FDomain, - FUrl, - FRequestContentType, - FResponseContentType, FContentType, - FSrc, + FContentTypeRequest, + FContentTypeResponse, + FDomain, FDst, + FHead, + FHeadRequest, + FHeadResponse, + FMethod, + FSrc, + FUrl, ] filt_int = [ FCode From fea4435dacf09797dd018f1dcf9f8280a54a2ce6 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 16 Jul 2016 09:59:40 +0530 Subject: [PATCH 14/16] Allow DummyFlow to match ~e, ~src and ~dst --- mitmproxy/filt.py | 3 --- test/mitmproxy/test_filt.py | 24 ++++++++++++------------ test/mitmproxy/tutils.py | 28 +++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 51e9cc307..f67698e50 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -76,7 +76,6 @@ class FErr(_Action): code = "e" help = "Match error" - @only(HTTPFlow, TCPFlow) def __call__(self, f): return True if f.error else False @@ -326,7 +325,6 @@ class FSrc(_Rex): help = "Match source address" is_binary = False - @only(HTTPFlow, TCPFlow) def __call__(self, f): return f.client_conn.address and self.re.search(repr(f.client_conn.address)) @@ -336,7 +334,6 @@ class FDst(_Rex): help = "Match destination address" is_binary = False - @only(HTTPFlow, TCPFlow) def __call__(self, f): return f.server_conn.address and self.re.search(repr(f.server_conn.address)) diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 92a48cd2e..69f042bb0 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -2,7 +2,6 @@ from six.moves import cStringIO as StringIO from mock import patch from mitmproxy import filt -from mitmproxy.models.flow import Flow from . import tutils @@ -379,23 +378,21 @@ class TestMatchingTCPFlow: assert not self.q("~u whatever", f) -class DummyFlow(Flow): - """ A flow that is neither HTTP nor TCP. """ - - def __init__(self): - pass - - class TestMatchingDummyFlow: def flow(self): - return DummyFlow() + return tutils.tdummyflow() + + def err(self): + return tutils.tdummyflow(err=True) def q(self, q, o): return filt.parse(q)(o) def test_filters(self): + e = self.err() f = self.flow() + f.server_conn = tutils.tserver_conn() assert not self.q("~a", f) @@ -403,12 +400,14 @@ class TestMatchingDummyFlow: assert not self.q("~bq whatever", f) assert not self.q("~bs whatever", f) - assert not self.q("~dst whatever", f) - assert not self.q("~c 0", f) assert not self.q("~d whatever", f) + assert self.q("~dst address", f) + assert not self.q("~dst nonexistent", f) + + assert self.q("~e", e) assert not self.q("~e", f) assert not self.q("~http", f) @@ -421,7 +420,8 @@ class TestMatchingDummyFlow: assert not self.q("~s", f) - assert not self.q("~src whatever", f) + assert self.q("~src address", f) + assert not self.q("~src nonexistent", f) assert not self.q("~tcp", f) diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index d0a09035e..d743a9e62 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -4,18 +4,19 @@ import tempfile import argparse import sys -from mitmproxy.models.tcp import TCPMessage -from six.moves import cStringIO as StringIO from contextlib import contextmanager - from unittest.case import SkipTest +from six.moves import cStringIO as StringIO + import netlib.utils import netlib.tutils from mitmproxy import controller from mitmproxy.models import ( ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow ) +from mitmproxy.models.tcp import TCPMessage +from mitmproxy.models.flow import Flow def _skip_windows(*args): @@ -47,6 +48,27 @@ def skip_appveyor(fn): return fn +class DummyFlow(Flow): + """A flow that is neither HTTP nor TCP.""" + + def __init__(self, client_conn, server_conn, live=None): + super(DummyFlow, self).__init__("dummy", client_conn, server_conn, live) + + +def tdummyflow(client_conn=True, server_conn=True, err=None): + if client_conn is True: + client_conn = tclient_conn() + if server_conn is True: + server_conn = tserver_conn() + if err is True: + err = terr() + + f = DummyFlow(client_conn, server_conn) + f.error = err + f.reply = controller.DummyReply() + return f + + def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None): if client_conn is True: client_conn = tclient_conn() From f0c15aa09c977ba89a2bd6b0b2371501f082828a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 16 Jul 2016 00:36:26 -0700 Subject: [PATCH 15/16] @mhils can't merge --- mitmproxy/filt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 2774f8d13..8b647b225 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -241,7 +241,7 @@ class FBod(_Rex): if self.re.search(f.request.get_content(strict=False)): return True if f.response and f.response.raw_content: - if self.re.search(f.request.get_content(strict=False)): + if self.re.search(f.response.get_content(strict=False)): return True elif isinstance(f, TCPFlow): for msg in f.messages: From 2fde1d0b5338fb75de782a7066093e94a25fab9c Mon Sep 17 00:00:00 2001 From: Drake Caraker Date: Sat, 16 Jul 2016 15:24:56 -0700 Subject: [PATCH 16/16] Update gamecenter.rst Typo fix: changed plist score value from '0' to '55', aligning with the subsequent explanatory text. --- docs/tutorials/gamecenter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/gamecenter.rst b/docs/tutorials/gamecenter.rst index 9dce5df8b..d0d73b736 100644 --- a/docs/tutorials/gamecenter.rst +++ b/docs/tutorials/gamecenter.rst @@ -51,7 +51,7 @@ The contents of the submission are particularly interesting: context 0 score-value - 0 + 55 timestamp 1363515361321