From 4ce5e1386c7d065f4c4f8b68aa57b0e18d6945ca Mon Sep 17 00:00:00 2001
From: RamiBerm <54766858+RamiBerm@users.noreply.github.com>
Date: Tue, 3 Sep 2019 17:19:50 +0300
Subject: [PATCH 1/5] Updated har_dump,py timings dictionary function

the HAR file spec (http://www.softwareishard.com/blog/har-12-spec/#timings) states that timings that do not apply for a certain requests should be set to -1, this example may set -1000 as a timings value for certain requests.
This ends up producing invalid HAR files in many cases.

My proposed fix is to assign -1 into the dic and only multiply by 1000 for other values
---
 examples/complex/har_dump.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
index 33a2f79f8..e09646017 100644
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -87,7 +87,7 @@ def response(flow):
     }
 
     # HAR timings are integers in ms, so we re-encode the raw timings to that format.
-    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
+    timings = dict([(k, -1 if v is -1 else int(1000 * v)) for k, v in timings_raw.items()])
 
     # full_time is the sum of all timings.
     # Timings set to -1 will be ignored as per spec.

From dd3589ce345bcabe51563e1e1ac83797353baf8c Mon Sep 17 00:00:00 2001
From: Tero Saaristo <tero.saaristo@gmail.com>
Date: Thu, 5 Sep 2019 17:52:04 +0300
Subject: [PATCH 2/5] encoding: add support for zstd (zstandard)

Handles zstandard-compressed bodies labeled as zstd.
---
 mitmproxy/addons/core.py                 |  2 +-
 mitmproxy/net/http/encoding.py           | 24 ++++++++++++++++++++++--
 mitmproxy/net/http/message.py            |  2 +-
 mitmproxy/net/http/request.py            |  2 +-
 setup.py                                 |  1 +
 test/mitmproxy/net/http/test_encoding.py |  1 +
 6 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py
index a908dbb31..5c9bbcd09 100644
--- a/mitmproxy/addons/core.py
+++ b/mitmproxy/addons/core.py
@@ -289,7 +289,7 @@ class Core:
         """
             The possible values for an encoding specification.
         """
-        return ["gzip", "deflate", "br"]
+        return ["gzip", "deflate", "br", "zstd"]
 
     @command.command("options.load")
     def options_load(self, path: mitmproxy.types.Path) -> None:
diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py
index 8cb96e5c3..16d399ca6 100644
--- a/mitmproxy/net/http/encoding.py
+++ b/mitmproxy/net/http/encoding.py
@@ -9,6 +9,7 @@ from io import BytesIO
 import gzip
 import zlib
 import brotli
+import zstandard as zstd
 
 from typing import Union, Optional, AnyStr  # noqa
 
@@ -52,7 +53,7 @@ def decode(
             decoded = custom_decode[encoding](encoded)
         except KeyError:
             decoded = codecs.decode(encoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return decoded
     except TypeError:
@@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio
             encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return encoded
     except TypeError:
@@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes:
     return brotli.compress(content)
 
 
+def decode_zstd(content: bytes) -> bytes:
+    if not content:
+        return b""
+    zstd_ctx = zstd.ZstdDecompressor()
+    try:
+        return zstd_ctx.decompress(content)
+    except zstd.ZstdError:
+        # If the zstd stream is streamed without a size header,
+        # try decoding with a 10MiB output buffer
+        return zstd_ctx.decompress(content, max_output_size=10 * 2**20)
+
+
+def encode_zstd(content: bytes) -> bytes:
+    zstd_ctx = zstd.ZstdCompressor()
+    return zstd_ctx.compress(content)
+
+
 def decode_deflate(content: bytes) -> bytes:
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
@@ -170,6 +188,7 @@ custom_decode = {
     "gzip": decode_gzip,
     "deflate": decode_deflate,
     "br": decode_brotli,
+    "zstd": decode_zstd,
 }
 custom_encode = {
     "none": identity,
@@ -177,6 +196,7 @@ custom_encode = {
     "gzip": encode_gzip,
     "deflate": encode_deflate,
     "br": encode_brotli,
+    "zstd": encode_zstd,
 }
 
 __all__ = ["encode", "decode"]
diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py
index 86782e8ac..6830c6cd3 100644
--- a/mitmproxy/net/http/message.py
+++ b/mitmproxy/net/http/message.py
@@ -236,7 +236,7 @@ class Message(serializable.Serializable):
 
     def encode(self, e):
         """
-        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br".
+        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd".
         Any existing content-encodings are overwritten,
         the content is not decoded beforehand.
 
diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py
index 959fdd339..3516e9496 100644
--- a/mitmproxy/net/http/request.py
+++ b/mitmproxy/net/http/request.py
@@ -421,7 +421,7 @@ class Request(message.Message):
             self.headers["accept-encoding"] = (
                 ', '.join(
                     e
-                    for e in {"gzip", "identity", "deflate", "br"}
+                    for e in {"gzip", "identity", "deflate", "br", "zstd"}
                     if e in accept_encoding
                 )
             )
diff --git a/setup.py b/setup.py
index 7f83de631..12439f4eb 100644
--- a/setup.py
+++ b/setup.py
@@ -81,6 +81,7 @@ setup(
         "tornado>=4.3,<5.2",
         "urwid>=2.0.1,<2.1",
         "wsproto>=0.13.0,<0.14.0",
+        "zstandard>=0.11.0,<0.13.0",
     ],
     extras_require={
         ':sys_platform == "win32"': [
diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py
index 8dac12cbc..7f768f39d 100644
--- a/test/mitmproxy/net/http/test_encoding.py
+++ b/test/mitmproxy/net/http/test_encoding.py
@@ -19,6 +19,7 @@ def test_identity(encoder):
     'gzip',
     'br',
     'deflate',
+    'zstd',
 ])
 def test_encoders(encoder):
     """

From ebfff8db761bf101ae46ac9ca0983408897baad2 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Thu, 5 Sep 2019 21:36:28 +0200
Subject: [PATCH 3/5] lint

---
 mitmproxy/tools/cmdline.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py
index 21369a1ff..eb4a984dc 100644
--- a/mitmproxy/tools/cmdline.py
+++ b/mitmproxy/tools/cmdline.py
@@ -1,5 +1,6 @@
 import argparse
 
+
 def common_options(parser, opts):
     parser.add_argument(
         '--version',

From e77f375186403cee05dc66c069ed1155059b2d63 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Thu, 5 Sep 2019 22:08:18 +0200
Subject: [PATCH 4/5] lint

---
 .travis.yml                                  |  1 -
 mitmproxy/contentviews/base.py               |  2 +-
 mitmproxy/contentviews/image/image_parser.py |  2 +-
 setup.py                                     |  2 +-
 test/mitmproxy/addons/test_view.py           |  2 +-
 test/mitmproxy/script/test_concurrent.py     | 26 ++++++++++----------
 6 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 20afc279e..44d452c27 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -52,7 +52,6 @@ matrix:
         - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb
         - sudo dpkg -i hugo*.deb
         - pip install tox virtualenv setuptools
-        - pyenv global system 3.6
       script:
         - tox
       after_success:
diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py
index 6072dfb76..9b34f3d4a 100644
--- a/mitmproxy/contentviews/base.py
+++ b/mitmproxy/contentviews/base.py
@@ -37,7 +37,7 @@ class View:
 
 def format_pairs(
         items: typing.Iterable[typing.Tuple[TTextType, TTextType]]
-)-> typing.Iterator[TViewLine]:
+) -> typing.Iterator[TViewLine]:
 
     """
     Helper function that accepts a list of (k,v) pairs into a list of
diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py
index fcc50cb5b..d5bb404f9 100644
--- a/mitmproxy/contentviews/image/image_parser.py
+++ b/mitmproxy/contentviews/image/image_parser.py
@@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata:
         entries = block.body.body.entries
         for entry in entries:
             comment = entry.bytes
-            if comment is not b'':
+            if comment != b'':
                 parts.append(('comment', str(comment)))
     return parts
 
diff --git a/setup.py b/setup.py
index 7f83de631..50d02a55a 100644
--- a/setup.py
+++ b/setup.py
@@ -88,7 +88,7 @@ setup(
         ],
         'dev': [
             "asynctest>=0.12.0",
-            "flake8>=3.5,<3.7",
+            "flake8>=3.5,<=3.7.8",
             "Flask>=1.0,<1.1",
             "mypy>=0.590,<0.591",
             "parver>=0.1,<2.0",
diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py
index 976c14b79..f5088a689 100644
--- a/test/mitmproxy/addons/test_view.py
+++ b/test/mitmproxy/addons/test_view.py
@@ -471,7 +471,7 @@ def test_focus():
     v = view.View()
     v.add([tft()])
     f = view.Focus(v)
-    assert f.index is 0
+    assert f.index == 0
     assert f.flow is v[0]
 
     # Start empty
diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py
index 3ec58760f..70d41511e 100644
--- a/test/mitmproxy/script/test_concurrent.py
+++ b/test/mitmproxy/script/test_concurrent.py
@@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest):
             assert await tctx.master.await_log("decorator not supported")
 
     def test_concurrent_class(self, tdata):
-            with taddons.context() as tctx:
-                sc = tctx.script(
-                    tdata.path(
-                        "mitmproxy/data/addonscripts/concurrent_decorator_class.py"
-                    )
+        with taddons.context() as tctx:
+            sc = tctx.script(
+                tdata.path(
+                    "mitmproxy/data/addonscripts/concurrent_decorator_class.py"
                 )
-                f1, f2 = tflow.tflow(), tflow.tflow()
-                tctx.cycle(sc, f1)
-                tctx.cycle(sc, f2)
-                start = time.time()
-                while time.time() - start < 5:
-                    if f1.reply.state == f2.reply.state == "committed":
-                        return
-                raise ValueError("Script never acked")
+            )
+            f1, f2 = tflow.tflow(), tflow.tflow()
+            tctx.cycle(sc, f1)
+            tctx.cycle(sc, f2)
+            start = time.time()
+            while time.time() - start < 5:
+                if f1.reply.state == f2.reply.state == "committed":
+                    return
+            raise ValueError("Script never acked")

From e97a804e89454f5f5f546f3f99635ca8b99d75d3 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Thu, 5 Sep 2019 22:13:49 +0200
Subject: [PATCH 5/5] make dict comprehension more readable

---
 examples/complex/har_dump.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
index e09646017..414b4f61f 100644
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -87,7 +87,10 @@ def response(flow):
     }
 
     # HAR timings are integers in ms, so we re-encode the raw timings to that format.
-    timings = dict([(k, -1 if v is -1 else int(1000 * v)) for k, v in timings_raw.items()])
+    timings = {
+        k: int(1000 * v) if v != -1 else -1 
+        for k, v in timings_raw.items()
+    }
 
     # full_time is the sum of all timings.
     # Timings set to -1 will be ignored as per spec.