Try not to hang when user views large request & response bodies

Two different strategies here:

    - Use a simple heuristic to detect if we're looking at XML data when indent
    mode is used. On non-XML data we can hang even on small documents.

    - Only view partial data for large bodies. At the moment the cutoff is
    100k. I might finetune this later.

libmproxy/console.py

@@ -20,6 +20,8 @@ import urwid.raw_display
 import urwid
 import controller, utils, filt, proxy, flow
 
+VIEW_CUTOFF = 1024*100
+
 
 class Stop(Exception): pass
 
@@ -879,14 +881,27 @@ class ConsoleMaster(flow.FlowMaster):
             self.spawn_external_viewer(serr, None)
         self.refresh_connection(f)
 
+    def _trailer(self, content, txt):
+        rem = len(content) - VIEW_CUTOFF
+        if rem > 0:
+            txt.append(urwid.Text(""))
+            txt.append(
+                urwid.Text(
+                    [
+                        ("highlight", "... %s of data not shown"%utils.pretty_size(rem))
+                    ]
+                )
+            )
+
     def _view_conn_normal(self, content, txt):
-        for i in content.splitlines():
+        for i in content[:VIEW_CUTOFF].splitlines():
             txt.append(
                 urwid.Text(("text", i))
             )
+        self._trailer(content, txt)
 
     def _view_conn_binary(self, content, txt):
-        for offset, hex, s in utils.hexdump(content):
+        for offset, hex, s in utils.hexdump(content[:VIEW_CUTOFF]):
             txt.append(urwid.Text([
                 ("offset", offset),
                 " ",
@@ -894,12 +909,14 @@ class ConsoleMaster(flow.FlowMaster):
                 "   ",
                 ("text", s),
             ]))
+        self._trailer(content, txt)
 
     def _view_conn_pretty(self, content, txt):
-        for i in utils.pretty_xmlish(content):
+        for i in utils.pretty_xmlish(content[:VIEW_CUTOFF]):
             txt.append(
                 urwid.Text(("text", i)),
             )
+        self._trailer(content, txt)
 
     @utils.LRUCache(20)
     def _cached_conn_text(self, content, hdrItems, viewmode):
@@ -918,7 +935,13 @@ class ConsoleMaster(flow.FlowMaster):
             if viewmode == VIEW_BODY_BINARY:
                 self._view_conn_binary(content, txt)
             elif viewmode == VIEW_BODY_INDENT:
-                self._view_conn_pretty(content, txt)
+                if utils.isXML(content):
+                    self._view_conn_pretty(content, txt)
+                else:
+                    if utils.isBin(content):
+                        self._view_conn_binary(content, txt)
+                    else:
+                        self._view_conn_normal(content, txt)
             else:
                 if utils.isBin(content):
                     self._view_conn_binary(content, txt)

libmproxy/utils.py

@@ -44,6 +44,16 @@ def isBin(s):
     return False
 
 
+def isXML(s):
+    for i in s:
+        if i in "\n \t":
+            continue
+        elif i == "<":
+            return True
+        else:
+            return False
+
+
 def cleanBin(s):
     parts = []
     for i in s:
@@ -51,7 +61,8 @@ def cleanBin(s):
         if o > 31 and o < 127:
             parts.append(i)
         else:
-            parts.append(".")
+            if i not in "\n\r\t":
+                parts.append(".")
     return "".join(parts)
     
 

test/test_utils.py

@@ -18,6 +18,13 @@ class uisBin(libpry.AutoTree):
         assert utils.isBin("testing\x7f")
 
 
+class uisXML(libpry.AutoTree):
+    def test_simple(self):
+        assert not utils.isXML("foo")
+        assert utils.isXML("<foo")
+        assert utils.isXML("  \n<foo")
+
+
 class uhexdump(libpry.AutoTree):
     def test_simple(self):
         assert utils.hexdump("one\0"*10)
@@ -352,6 +359,7 @@ class uLRUCache(libpry.AutoTree):
 tests = [
     uformat_timestamp(),
     uisBin(),
+    uisXML(),
     uhexdump(),
     upretty_size(),
     uisStringLike(),