Merge pull request #2537 from ujjwal96/content-view

JS beautifier

mitmproxy/contentviews/css.py

@@ -14,10 +14,10 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are:
 """
 
 CSS_SPECIAL_AREAS = (
-    ("'", strutils.NO_ESCAPE + "'"),
-    ('"', strutils.NO_ESCAPE + '"'),
-    (r"/\*", r"\*/"),
-    ("//", "$")
+    "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'",
+    '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"',
+    r"/\*" + strutils.MULTILINE_CONTENT + "\*/",
+    "//" + strutils.SINGLELINE_CONTENT + "$"
 )
 CSS_SPECIAL_CHARS = "{};:"
 

mitmproxy/contentviews/javascript.py

@@ -1,6 +1,47 @@
-import jsbeautifier
+import io
+import re
 
-from . import base
+from mitmproxy.utils import strutils
+from mitmproxy.contentviews import base
+
+DELIMITERS = '{};\n'
+SPECIAL_AREAS = (
+    r"(?<=[^\w\s)])\s*/(?:[^\n/]|(?<!\\)(?:\\\\)*\\/)+?/(?=[gimsuy]{0,6}\s*(?:[;,).\n]|$))",
+    r"'" + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + "'",
+    r'"' + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + '"',
+    r'`' + strutils.MULTILINE_CONTENT + strutils.NO_ESCAPE + '`',
+    r"/\*" + strutils.MULTILINE_CONTENT + "\*/",
+    r"//" + strutils.SINGLELINE_CONTENT + "$",
+    r"for\(" + strutils.SINGLELINE_CONTENT + "\)",
+)
+
+
+def beautify(data):
+    data = strutils.escape_special_areas(
+        data,
+        SPECIAL_AREAS,
+        DELIMITERS
+    )
+
+    data = re.sub(r"\s*{\s*(?!};)", " {\n", data)
+    data = re.sub(r"\s*;\s*", ";\n", data)
+    data = re.sub(r"(?<!{)\s*}(;)?\s*", r"\n}\1\n", data)
+
+    beautified = io.StringIO()
+    indent_level = 0
+
+    for line in data.splitlines(True):
+        if line.endswith("{\n"):
+            beautified.write(" " * 2 * indent_level + line)
+            indent_level += 1
+        elif line.startswith("}"):
+            indent_level -= 1
+            beautified.write(" " * 2 * indent_level + line)
+        else:
+            beautified.write(" " * 2 * indent_level + line)
+
+    data = strutils.unescape_special_areas(beautified.getvalue())
+    return data
 
 
 class ViewJavaScript(base.View):
@@ -13,8 +54,6 @@ class ViewJavaScript(base.View):
     ]
 
     def __call__(self, data, **metadata):
-        opts = jsbeautifier.default_options()
-        opts.indent_size = 2
         data = data.decode("utf-8", "replace")
-        res = jsbeautifier.beautify(data, opts)
+        res = beautify(data)
         return "JavaScript", base.format_text(res)

mitmproxy/contrib/README

@@ -1,14 +1,9 @@
 
 Contribs:
 
-jsbeautifier, git checkout 25/03/12, MIT license
-    - Removed test directories
-    - Disabled packers through a single-line modification (see "# CORTESI"
-    comment)
-
 wbxml
     - https://github.com/davidpshaw/PyWBXMLDecoder
 
 tls, BSD license
     - https://github.com/mhils/tls/tree/mitmproxy
-    - limited to required files.
+    - limited to required files.

mitmproxy/utils/strutils.py

@@ -1,7 +1,7 @@
 import io
 import re
 import codecs
-from typing import AnyStr, Optional, cast, Iterable, Tuple
+from typing import AnyStr, Optional, cast, Iterable
 
 
 def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]:
@@ -153,11 +153,14 @@ def _restore_from_private_code_plane(matchobj):
 
 
 NO_ESCAPE = r"(?<!\\)(?:\\\\)*"
+MULTILINE_CONTENT = r"[\s\S]*?"
+SINGLELINE_CONTENT = r".*?"
+MULTILINE_CONTENT_LINE_CONTINUATION = r"(?:.|(?<=\\)\n)*?"
 
 
 def split_special_areas(
         data: str,
-        area_delimiter: Iterable[Tuple[str, str]],
+        area_delimiter: Iterable[str],
 ):
     """
     Split a string of code into a [code, special area, code, special area, ..., code] list.
@@ -166,18 +169,13 @@ def split_special_areas(
 
     >>> split_special_areas(
     >>>     "test /* don't modify me */ foo",
-    >>>     [(r"/\*", r"\*/")])  # (left delimiter regex, right delimiter regex)
+    >>>     [r"/\*[\s\S]*?\*/"])  # (regex matching comments)
     ["test ", "/* don't modify me */", " foo"]
 
     "".join(split_special_areas(x, ...)) == x always holds true.
     """
-    patterns = "|".join(
-        r"{lchar}[\s\S]*?{rchar}".format(
-            lchar=a,
-            rchar=b,
-        ) for (a, b) in area_delimiter)
     return re.split(
-        "({})".format(patterns),
+        "({})".format("|".join(area_delimiter)),
         data,
         flags=re.MULTILINE
     )
@@ -185,7 +183,7 @@ def split_special_areas(
 
 def escape_special_areas(
         data: str,
-        area_delimiter: Iterable[Tuple[str, str]],
+        area_delimiter: Iterable[str],
         control_characters,
 ):
     """
@@ -200,11 +198,11 @@ def escape_special_areas(
 
     >>> print(x)
     if (true) { console.log('{}'); }
-    >>> x = escape_special_areas(x, "{", [("'", "'")])
+    >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"])
     >>> print(x)
     if (true) { console.log('<EFBFBD>}'); }
     >>> x = re.sub(r"\s*{\s*", " {\n    ", x)
-    >>> x = unescape_special_areas(x, "{", [("'", "'")])
+    >>> x = unescape_special_areas(x)
     >>> print(x)
     if (true) {
         console.log('{}'); }

release/hooks/hook-passlib.py

@@ -0,0 +1 @@
+hiddenimports = ["configparser"]

setup.cfg

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 140
 max-complexity = 25
-ignore = E251,C901,W503
+ignore = E251,C901,W503,W292
 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*
 addons = file,open,basestring,xrange,unicode,long,cmp
 

setup.py

@@ -68,7 +68,6 @@ setup(
         "h2>=3.0, <4",
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=5.0, <6",
-        "jsbeautifier>=1.6.3, <1.7",
         "kaitaistruct>=0.7, <0.8",
         "ldap3>=2.2.0, <2.4",
         "passlib>=1.6.5, <1.8",

test/mitmproxy/contentviews/test_javascript.py

@@ -1,10 +1,32 @@
+import pytest
+
 from mitmproxy.contentviews import javascript
+from mitmproxy.test import tutils
 from . import full_eval
 
+data = tutils.test_data.push("mitmproxy/contentviews/test_js_data/")
+
 
 def test_view_javascript():
     v = full_eval(javascript.ViewJavaScript())
     assert v(b"[1, 2, 3]")
     assert v(b"[1, 2, 3")
-    assert v(b"function(a){[1, 2, 3]}")
+    assert v(b"function(a){[1, 2, 3]}") == ("JavaScript", [
+        [('text', 'function(a) {')],
+        [('text', '  [1, 2, 3]')],
+        [('text', '}')]
+    ])
     assert v(b"\xfe")  # invalid utf-8
+
+
+@pytest.mark.parametrize("filename", [
+    "simple.js",
+])
+def test_format_xml(filename):
+    path = data.path(filename)
+    with open(path) as f:
+        input = f.read()
+    with open("-formatted.".join(path.rsplit(".", 1))) as f:
+        expected = f.read()
+    js = javascript.beautify(input)
+    assert js == expected

test/mitmproxy/contentviews/test_js_data/simple-formatted.js

@@ -0,0 +1,68 @@
+/* _GlobalPrefix_ */
+this.gbar_=this.gbar_||{};
+(function(_) {
+  var window=this;
+  
+/* _Module_:sy25 */
+  try {
+    var Mn=function(){};
+    _.y(Mn,Error);
+    _.Nn=function() {
+      this.b="pending";
+      this.B=[];
+      this.w=this.C=void 0
+    };
+    _.fe(_.Nn);
+    var On=function() {
+      _.qa.call(this,"Multiple attempts to set the state of this Result")
+    };
+    _.y(On,_.qa);
+    _.Nn.prototype.ta=function() {
+      return this.C
+    };
+    _.Pn=function(a,c,d) {
+      "pending"==a.b?a.B.push( {
+        hb:c,scope:d||null
+      }
+      ):c.call(d,a)
+    };
+    _.Nn.prototype.A=function(a) {
+      if("pending"==this.b)this.C=a,this.b="success",Qn(this);
+      else if(!Rn(this))throw new On;
+    };
+    _.Nn.prototype.o=function(a) {
+      if("pending"==this.b)this.w=a,this.b="error",Qn(this);
+      else if(!Rn(this))throw new On;
+    };
+    var Qn=function(a) {
+      var c=a.B;
+      a.B=[];
+      for(var d=0;d<c.length;d++) {
+        var e=c[d];
+        e.hb.call(e.scope,a)
+      }
+      
+    };
+    _.Nn.prototype.cancel=function() {
+      return"pending"==this.b?(this.o(new Mn),!0):!1
+    };
+    var Rn=function(a) {
+      return"error"==a.b&&a.w instanceof Mn
+    };
+    _.Nn.prototype.then=function(a,c,d) {
+      var e,f,g=new _.ie(function(a,c) {
+        e=a;
+        f=c
+      }
+      );
+      _.Pn(this,function(a) {
+        Rn(a)?g.cancel():"success"==a.b?e(a.ta()):"error"==a.b&&f(a.w)
+      }
+      );
+      return g.then(a,c,d)
+    };
+    
+  }
+  catch(e) {
+    _._DumpException(e)
+  }

test/mitmproxy/contentviews/test_js_data/simple.js

@@ -0,0 +1,8 @@
+/* _GlobalPrefix_ */
+this.gbar_=this.gbar_||{};(function(_){var window=this;
+/* _Module_:sy25 */
+try{
+var Mn=function(){};_.y(Mn,Error);_.Nn=function(){this.b="pending";this.B=[];this.w=this.C=void 0};_.fe(_.Nn);var On=function(){_.qa.call(this,"Multiple attempts to set the state of this Result")};_.y(On,_.qa);_.Nn.prototype.ta=function(){return this.C};_.Pn=function(a,c,d){"pending"==a.b?a.B.push({hb:c,scope:d||null}):c.call(d,a)};_.Nn.prototype.A=function(a){if("pending"==this.b)this.C=a,this.b="success",Qn(this);else if(!Rn(this))throw new On;};
+_.Nn.prototype.o=function(a){if("pending"==this.b)this.w=a,this.b="error",Qn(this);else if(!Rn(this))throw new On;};var Qn=function(a){var c=a.B;a.B=[];for(var d=0;d<c.length;d++){var e=c[d];e.hb.call(e.scope,a)}};_.Nn.prototype.cancel=function(){return"pending"==this.b?(this.o(new Mn),!0):!1};var Rn=function(a){return"error"==a.b&&a.w instanceof Mn}; _.Nn.prototype.then=function(a,c,d){var e,f,g=new _.ie(function(a,c){e=a;f=c});_.Pn(this,function(a){Rn(a)?g.cancel():"success"==a.b?e(a.ta()):"error"==a.b&&f(a.w)});return g.then(a,c,d)};
+
+}catch(e){_._DumpException(e)}

test/mitmproxy/utils/test_strutils.py

@@ -99,8 +99,8 @@ def test_hexdump():
 
 
 ESCAPE_QUOTES = [
-    ("'", strutils.NO_ESCAPE + "'"),
-    ('"', strutils.NO_ESCAPE + '"')
+    "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'",
+    '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"'
 ]
 
 
@@ -113,11 +113,11 @@ def test_split_special_areas():
     ) == ["foo ", "'b\\'a\"r'", " baz"]
     assert strutils.split_special_areas(
         "foo\n/*bar\nbaz*/\nqux",
-        [(r'/\*', r'\*/')]
+        [r'/\*[\s\S]+?\*/']
     ) == ["foo\n", "/*bar\nbaz*/", "\nqux"]
     assert strutils.split_special_areas(
         "foo\n//bar\nbaz",
-        [(r'//', r'$')]
+        [r'//.+$']
     ) == ["foo\n", "//bar", "\nbaz"]