From b05bf588e68ba8be08e27ccdf40f97a59b17250a Mon Sep 17 00:00:00 2001 From: Ujjwal Verma Date: Sun, 3 Sep 2017 01:20:30 +0530 Subject: [PATCH 1/6] JS beautifier --- mitmproxy/contentviews/javascript.py | 49 +++++++++++-- .../test_js_data/simple-formatted.js | 68 +++++++++++++++++++ .../contentviews/test_js_data/simple.js | 8 +++ 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 test/mitmproxy/contentviews/test_js_data/simple-formatted.js create mode 100644 test/mitmproxy/contentviews/test_js_data/simple.js diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index c2fab8755..1d671fe68 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -1,6 +1,47 @@ -import jsbeautifier +import io +import re -from . import base +from mitmproxy.utils import strutils +from mitmproxy.contentviews import base + +DELIMITERS = '{};\n' +SPECIAL_AREAS = ( + r"(?<=[^\w\s)])\s*/(?:[^\n/]|(? Date: Sat, 26 Aug 2017 20:24:40 +0530 Subject: [PATCH 2/6] Added test --- .../mitmproxy/contentviews/test_javascript.py | 24 +++- .../test_js_data/simple-formatted.js | 136 +++++++++--------- .../contentviews/test_js_data/simple.js | 2 +- 3 files changed, 92 insertions(+), 70 deletions(-) diff --git a/test/mitmproxy/contentviews/test_javascript.py b/test/mitmproxy/contentviews/test_javascript.py index 43039c933..23dd106ec 100644 --- a/test/mitmproxy/contentviews/test_javascript.py +++ b/test/mitmproxy/contentviews/test_javascript.py @@ -1,10 +1,32 @@ +import pytest + from mitmproxy.contentviews import javascript +from mitmproxy.test import tutils from . import full_eval +data = tutils.test_data.push("mitmproxy/contentviews/test_js_data/") + def test_view_javascript(): v = full_eval(javascript.ViewJavaScript()) assert v(b"[1, 2, 3]") assert v(b"[1, 2, 3") - assert v(b"function(a){[1, 2, 3]}") + assert v(b"function(a){[1, 2, 3]}") == ("JavaScript", [ + [('text', 'function(a) {')], + [('text', ' [1, 2, 3]')], + [('text', '}')] + ]) assert v(b"\xfe") # invalid utf-8 + + +@pytest.mark.parametrize("filename", [ + "simple.js", +]) +def test_format_xml(filename): + path = data.path(filename) + with open(path) as f: + input = f.read() + with open("-formatted.".join(path.rsplit(".", 1))) as f: + expected = f.read() + js = javascript.beautify(input) + assert js == expected diff --git a/test/mitmproxy/contentviews/test_js_data/simple-formatted.js b/test/mitmproxy/contentviews/test_js_data/simple-formatted.js index 7ffd7b08c..2b665f026 100644 --- a/test/mitmproxy/contentviews/test_js_data/simple-formatted.js +++ b/test/mitmproxy/contentviews/test_js_data/simple-formatted.js @@ -1,68 +1,68 @@ -/* _GlobalPrefix_ */ -this.gbar_=this.gbar_||{}; -(function(_) { - var window=this; - -/* _Module_:sy25 */ - try { - var Mn=function(){}; - _.y(Mn,Error); - _.Nn=function() { - this.b="pending"; - this.B=[]; - this.w=this.C=void 0 - }; - _.fe(_.Nn); - var On=function() { - _.qa.call(this,"Multiple attempts to set the state of this Result") - }; - _.y(On,_.qa); - _.Nn.prototype.ta=function() { - return this.C - }; - _.Pn=function(a,c,d) { - "pending"==a.b?a.B.push( { - hb:c,scope:d||null - } - ):c.call(d,a) - }; - _.Nn.prototype.A=function(a) { - if("pending"==this.b)this.C=a,this.b="success",Qn(this); - else if(!Rn(this))throw new On; - }; - _.Nn.prototype.o=function(a) { - if("pending"==this.b)this.w=a,this.b="error",Qn(this); - else if(!Rn(this))throw new On; - }; - var Qn=function(a) { - var c=a.B; - a.B=[]; - for(var d=0;d Date: Thu, 31 Aug 2017 02:14:22 +0200 Subject: [PATCH 3/6] make split_special_areas more flexible, refs #2537 (cherry picked from commit 31ef7f149e4553eb9403634c0eec6de4d0123386) --- mitmproxy/contentviews/css.py | 8 ++++---- mitmproxy/utils/strutils.py | 22 ++++++++++------------ test/mitmproxy/utils/test_strutils.py | 8 ++++---- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index 8fa09ed38..30a583da5 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -14,10 +14,10 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are: """ CSS_SPECIAL_AREAS = ( - ("'", strutils.NO_ESCAPE + "'"), - ('"', strutils.NO_ESCAPE + '"'), - (r"/\*", r"\*/"), - ("//", "$") + "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", + '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"', + r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + "//" + strutils.SINGLELINE_CONTENT + "$" ) CSS_SPECIAL_CHARS = "{};:" diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 37bed7ded..71d1c54c4 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,7 +1,7 @@ import io import re import codecs -from typing import AnyStr, Optional, cast, Iterable, Tuple +from typing import AnyStr, Optional, cast, Iterable def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: @@ -153,11 +153,14 @@ def _restore_from_private_code_plane(matchobj): NO_ESCAPE = r"(?>> split_special_areas( >>> "test /* don't modify me */ foo", - >>> [(r"/\*", r"\*/")]) # (left delimiter regex, right delimiter regex) + >>> [r"/\*[\s\S]*?\*/"]) # (regex matching comments) ["test ", "/* don't modify me */", " foo"] "".join(split_special_areas(x, ...)) == x always holds true. """ - patterns = "|".join( - r"{lchar}[\s\S]*?{rchar}".format( - lchar=a, - rchar=b, - ) for (a, b) in area_delimiter) return re.split( - "({})".format(patterns), + "({})".format("|".join(area_delimiter)), data, flags=re.MULTILINE ) @@ -185,7 +183,7 @@ def split_special_areas( def escape_special_areas( data: str, - area_delimiter: Iterable[Tuple[str, str]], + area_delimiter: Iterable[str], control_characters, ): """ @@ -200,11 +198,11 @@ def escape_special_areas( >>> print(x) if (true) { console.log('{}'); } - >>> x = escape_special_areas(x, "{", [("'", "'")]) + >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"]) >>> print(x) if (true) { console.log('�}'); } >>> x = re.sub(r"\s*{\s*", " {\n ", x) - >>> x = unescape_special_areas(x, "{", [("'", "'")]) + >>> x = unescape_special_areas(x) >>> print(x) if (true) { console.log('{}'); } diff --git a/test/mitmproxy/utils/test_strutils.py b/test/mitmproxy/utils/test_strutils.py index 7ec72e4ec..dfe2c6203 100644 --- a/test/mitmproxy/utils/test_strutils.py +++ b/test/mitmproxy/utils/test_strutils.py @@ -99,8 +99,8 @@ def test_hexdump(): ESCAPE_QUOTES = [ - ("'", strutils.NO_ESCAPE + "'"), - ('"', strutils.NO_ESCAPE + '"') + "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", + '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"' ] @@ -113,11 +113,11 @@ def test_split_special_areas(): ) == ["foo ", "'b\\'a\"r'", " baz"] assert strutils.split_special_areas( "foo\n/*bar\nbaz*/\nqux", - [(r'/\*', r'\*/')] + [r'/\*[\s\S]+?\*/'] ) == ["foo\n", "/*bar\nbaz*/", "\nqux"] assert strutils.split_special_areas( "foo\n//bar\nbaz", - [(r'//', r'$')] + [r'//.+$'] ) == ["foo\n", "//bar", "\nbaz"] From f46e5e667ffa6c5c32b2568d4a5e2bb461852fe2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 3 Sep 2017 21:01:52 +0200 Subject: [PATCH 4/6] remove jsbeautifier dependency --- mitmproxy/contrib/README | 7 +------ setup.py | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/mitmproxy/contrib/README b/mitmproxy/contrib/README index e5ce11daf..fc44eed38 100644 --- a/mitmproxy/contrib/README +++ b/mitmproxy/contrib/README @@ -1,14 +1,9 @@ Contribs: -jsbeautifier, git checkout 25/03/12, MIT license - - Removed test directories - - Disabled packers through a single-line modification (see "# CORTESI" - comment) - wbxml - https://github.com/davidpshaw/PyWBXMLDecoder tls, BSD license - https://github.com/mhils/tls/tree/mitmproxy - - limited to required files. \ No newline at end of file + - limited to required files. diff --git a/setup.py b/setup.py index fefb799e9..52ab18402 100644 --- a/setup.py +++ b/setup.py @@ -68,7 +68,6 @@ setup( "h2>=3.0, <4", "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=5.0, <6", - "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.7, <0.8", "ldap3>=2.2.0, <2.4", "passlib>=1.6.5, <1.8", From f6f994416082cdbadd0b21669216180a3b647d21 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Sep 2017 17:53:55 +0200 Subject: [PATCH 5/6] pyinstaller: add passlib hook --- release/hooks/hook-passlib.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 release/hooks/hook-passlib.py diff --git a/release/hooks/hook-passlib.py b/release/hooks/hook-passlib.py new file mode 100644 index 000000000..4f1efbb83 --- /dev/null +++ b/release/hooks/hook-passlib.py @@ -0,0 +1 @@ +hiddenimports = ["configparser"] From ca91780ff4482df13f1456801583321dd4c7f5b1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Sep 2017 18:08:00 +0200 Subject: [PATCH 6/6] do not fail CI for missing newlines this regularly stalls pull requests and serves no further purpose, so it needs to go. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 42be63dba..02a048d99 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,C901,W503 +ignore = E251,C901,W503,W292 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* addons = file,open,basestring,xrange,unicode,long,cmp