Merge branch 'master' into tabular

.travis.yml

@@ -52,7 +52,6 @@ matrix:
         - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb
         - sudo dpkg -i hugo*.deb
         - pip install tox virtualenv setuptools
-        - pyenv global system 3.6
       script:
         - tox
       after_success:

examples/complex/har_dump.py

@@ -87,7 +87,10 @@ def response(flow):
     }
 
     # HAR timings are integers in ms, so we re-encode the raw timings to that format.
-    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
+    timings = {
+        k: int(1000 * v) if v != -1 else -1 
+        for k, v in timings_raw.items()
+    }
 
     # full_time is the sum of all timings.
     # Timings set to -1 will be ignored as per spec.

mitmproxy/addons/core.py

@@ -289,7 +289,7 @@ class Core:
         """
             The possible values for an encoding specification.
         """
-        return ["gzip", "deflate", "br"]
+        return ["gzip", "deflate", "br", "zstd"]
 
     @command.command("options.load")
     def options_load(self, path: mitmproxy.types.Path) -> None:

mitmproxy/contentviews/base.py

@@ -37,7 +37,7 @@ class View:
 
 def format_pairs(
         items: typing.Iterable[typing.Tuple[TTextType, TTextType]]
-)-> typing.Iterator[TViewLine]:
+) -> typing.Iterator[TViewLine]:
 
     """
     Helper function that accepts a list of (k,v) pairs into a list of

mitmproxy/contentviews/image/image_parser.py

@@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata:
         entries = block.body.body.entries
         for entry in entries:
             comment = entry.bytes
-            if comment is not b'':
+            if comment != b'':
                 parts.append(('comment', str(comment)))
     return parts
 

mitmproxy/net/http/encoding.py

@@ -9,6 +9,7 @@ from io import BytesIO
 import gzip
 import zlib
 import brotli
+import zstandard as zstd
 
 from typing import Union, Optional, AnyStr  # noqa
 
@@ -52,7 +53,7 @@ def decode(
             decoded = custom_decode[encoding](encoded)
         except KeyError:
             decoded = codecs.decode(encoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return decoded
     except TypeError:
@@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio
             encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return encoded
     except TypeError:
@@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes:
     return brotli.compress(content)
 
 
+def decode_zstd(content: bytes) -> bytes:
+    if not content:
+        return b""
+    zstd_ctx = zstd.ZstdDecompressor()
+    try:
+        return zstd_ctx.decompress(content)
+    except zstd.ZstdError:
+        # If the zstd stream is streamed without a size header,
+        # try decoding with a 10MiB output buffer
+        return zstd_ctx.decompress(content, max_output_size=10 * 2**20)
+
+
+def encode_zstd(content: bytes) -> bytes:
+    zstd_ctx = zstd.ZstdCompressor()
+    return zstd_ctx.compress(content)
+
+
 def decode_deflate(content: bytes) -> bytes:
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
@@ -170,6 +188,7 @@ custom_decode = {
     "gzip": decode_gzip,
     "deflate": decode_deflate,
     "br": decode_brotli,
+    "zstd": decode_zstd,
 }
 custom_encode = {
     "none": identity,
@@ -177,6 +196,7 @@ custom_encode = {
     "gzip": encode_gzip,
     "deflate": encode_deflate,
     "br": encode_brotli,
+    "zstd": encode_zstd,
 }
 
 __all__ = ["encode", "decode"]

mitmproxy/net/http/message.py

@@ -236,7 +236,7 @@ class Message(serializable.Serializable):
 
     def encode(self, e):
         """
-        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br".
+        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd".
         Any existing content-encodings are overwritten,
         the content is not decoded beforehand.
 

mitmproxy/net/http/request.py

@@ -423,7 +423,7 @@ class Request(message.Message):
             self.headers["accept-encoding"] = (
                 ', '.join(
                     e
-                    for e in {"gzip", "identity", "deflate", "br"}
+                    for e in {"gzip", "identity", "deflate", "br", "zstd"}
                     if e in accept_encoding
                 )
             )

setup.py

@@ -82,6 +82,7 @@ setup(
         "urwid>=2.0.1,<2.1",
         "wsproto>=0.13.0,<0.14.0",
         "publicsuffix2~=2.20"
+        "zstandard>=0.11.0,<0.13.0",
     ],
     extras_require={
         ':sys_platform == "win32"': [
@@ -89,7 +90,7 @@ setup(
         ],
         'dev': [
             "asynctest>=0.12.0",
-            "flake8>=3.5,<3.7",
+            "flake8>=3.5,<=3.7.8",
             "Flask>=1.0,<1.1",
             "mypy>=0.590,<0.591",
             "parver>=0.1,<2.0",

test/mitmproxy/addons/test_view.py

@@ -471,7 +471,7 @@ def test_focus():
     v = view.View()
     v.add([tft()])
     f = view.Focus(v)
-    assert f.index is 0
+    assert f.index == 0
     assert f.flow is v[0]
 
     # Start empty

test/mitmproxy/net/http/test_encoding.py

@@ -19,6 +19,7 @@ def test_identity(encoder):
     'gzip',
     'br',
     'deflate',
+    'zstd',
 ])
 def test_encoders(encoder):
     """

test/mitmproxy/script/test_concurrent.py

@@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest):
             assert await tctx.master.await_log("decorator not supported")
 
     def test_concurrent_class(self, tdata):
-            with taddons.context() as tctx:
-                sc = tctx.script(
-                    tdata.path(
-                        "mitmproxy/data/addonscripts/concurrent_decorator_class.py"
-                    )
+        with taddons.context() as tctx:
+            sc = tctx.script(
+                tdata.path(
+                    "mitmproxy/data/addonscripts/concurrent_decorator_class.py"
                 )
-                f1, f2 = tflow.tflow(), tflow.tflow()
-                tctx.cycle(sc, f1)
-                tctx.cycle(sc, f2)
-                start = time.time()
-                while time.time() - start < 5:
-                    if f1.reply.state == f2.reply.state == "committed":
-                        return
-                raise ValueError("Script never acked")
+            )
+            f1, f2 = tflow.tflow(), tflow.tflow()
+            tctx.cycle(sc, f1)
+            tctx.cycle(sc, f2)
+            start = time.time()
+            while time.time() - start < 5:
+                if f1.reply.state == f2.reply.state == "committed":
+                    return
+            raise ValueError("Script never acked")