From 92c556afbe631f1ff065d8ffc612f8e04ce9f9d8 Mon Sep 17 00:00:00 2001 From: Gaurav Jain <64748057+errorxyz@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:20:25 +0530 Subject: [PATCH] Add `content_view_lines_cutoff` option to mitmdump (#6692) #### Description Fixes #4476 This PR adds the already existing `content_view_lines_cutoff` option to `mitmdump` to filter the number of content lines shown when `flow_detail` option is set to `3`. By default this option's value is set to `512` Updated tests that check for `cutoff` in responses that cross `512` lines(with the default value set) Added tests to ensure that custom values for `content_view_lines_cutoff` works as intended #### Verification 1. run `mitmdump --set flow_detail=3 --set content_view_lines_cutoff=3` 2. run `curl --proxy http:/127.0.0.1:8080 "https://google.com/"` #### Before
Output ``` [15:00:41.441] HTTP(S) proxy listening at *:8080. [15:00:53.236][127.0.0.1:38826] client connect [15:00:53.520][127.0.0.1:38826] server connect google.com:443 (142.250.183.238:443) 127.0.0.1:38826: GET https://google.com/ HTTP/2.0 user-agent: curl/8.5.0 accept: */* << HTTP/2.0 301 Moved Permanently 220b location: https://www.google.com/ content-type: text/html; charset=UTF-8 content-security-policy-report-only: object-src 'none';base-uri 'self';script-src 'nonce-7rNdfc7B14cr_4JZskk3AQ' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/gws/other-hp date: Tue, 27 Feb 2024 09:31:02 GMT expires: Thu, 28 Mar 2024 09:31:02 GMT cache-control: public, max-age=2592000 server: gws content-length: 220 x-xss-protection: 0 x-frame-options: SAMEORIGIN alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 301 Moved

301 Moved

The document has moved here . [15:00:53.897][127.0.0.1:38826] client disconnect [15:00:53.897][127.0.0.1:38826] server disconnect google.com:443 (142.250.183.238:443) ```
Content is not truncated within `3` lines #### After
Output ``` [14:55:26.042] HTTP(S) proxy listening at *:8080. [14:55:33.630][127.0.0.1:40554] client connect [14:55:33.775][127.0.0.1:40554] server connect google.com:443 (142.250.182.142:443) 127.0.0.1:40554: GET https://google.com/ HTTP/2.0 user-agent: curl/8.5.0 accept: */* << HTTP/2.0 301 Moved Permanently 220b location: https://www.google.com/ content-type: text/html; charset=UTF-8 content-security-policy-report-only: object-src 'none';base-uri 'self';script-src 'nonce-LCd_ThPYwSImYoighASUFQ' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/gws/other-hp date: Tue, 27 Feb 2024 09:25:42 GMT expires: Thu, 28 Mar 2024 09:25:42 GMT cache-control: public, max-age=2592000 server: gws content-length: 220 x-xss-protection: 0 x-frame-options: SAMEORIGIN alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 (cut off) [14:55:34.021][127.0.0.1:40554] client disconnect [14:55:34.022][127.0.0.1:40554] server disconnect google.com:443 (142.250.182.142:443) ```
Content is truncated within `3` lines #### Checklist - [x] I have updated tests where applicable. - [x] I have added an entry to the CHANGELOG. --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 ++ mitmproxy/addons/dumper.py | 9 ++++++--- test/mitmproxy/addons/test_dumper.py | 19 +++++++++++++++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 051202a58..f5a8f3ca8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ ([#6659](https://github.com/mitmproxy/mitmproxy/pull/6659), @basedBaba) * Fix a regression when leaf cert creation would fail with intermediate CAs in `ca_file`. ([#6666](https://github.com/mitmproxy/mitmproxy/pull/6666), @manselmi) +* Add `content_view_lines_cutoff` option to mitmdump + ([#6692](https://github.com/mitmproxy/mitmproxy/pull/6692), @errorxyz) ## 21 January 2024: mitmproxy 10.2.2 diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index b76c34832..1fdfe8bcf 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -18,6 +18,7 @@ from mitmproxy import flowfilter from mitmproxy import http from mitmproxy.contrib import click as miniclick from mitmproxy.net.dns import response_codes +from mitmproxy.options import CONTENT_VIEW_LINES_CUTOFF from mitmproxy.tcp import TCPFlow from mitmproxy.tcp import TCPMessage from mitmproxy.udp import UDPFlow @@ -54,12 +55,12 @@ class Dumper: "flow_detail", int, 1, - """ + f""" The display detail level for flows in mitmdump: 0 (quiet) to 4 (very verbose). 0: no output 1: shortened request URL with response status code 2: full request URL with response status code and HTTP headers - 3: 2 + truncated response content, content of WebSocket and TCP messages + 3: 2 + truncated response content, content of WebSocket and TCP messages (content_view_lines_cutoff: {CONTENT_VIEW_LINES_CUTOFF}) 4: 3 + nothing is truncated """, ) @@ -125,7 +126,9 @@ class Dumper: logging.debug(error) if ctx.options.flow_detail == 3: - lines_to_echo = itertools.islice(lines, 70) + lines_to_echo = itertools.islice( + lines, ctx.options.content_view_lines_cutoff + ) else: lines_to_echo = lines diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index c7f2e85d4..e9b6a92bc 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -97,7 +97,7 @@ def test_simple(): def test_echo_body(): f = tflow.tflow(resp=True) f.response.headers["content-type"] = "text/html" - f.response.content = b"foo bar voing\n" * 100 + f.response.content = b"foo bar voing\n" * 600 sio = io.StringIO() d = dumper.Dumper(sio) @@ -108,6 +108,21 @@ def test_echo_body(): assert "cut off" in t +def test_echo_body_custom_cutoff(): + f = tflow.tflow(resp=True) + f.response.headers["content-type"] = "text/html" + f.response.content = b"foo bar voing\n" * 4 + + sio = io.StringIO() + d = dumper.Dumper(sio) + with taddons.context(d) as ctx: + ctx.configure(d, flow_detail=3) + ctx.configure(d, content_view_lines_cutoff=3) + d._echo_message(f.response, f) + t = sio.getvalue() + assert "cut off" in t + + def test_echo_trailer(): sio = io.StringIO() d = dumper.Dumper(sio) @@ -118,7 +133,7 @@ def test_echo_trailer(): f.request.headers["content-type"] = "text/html" f.request.headers["transfer-encoding"] = "chunked" f.request.headers["trailer"] = "my-little-request-trailer" - f.request.content = b"some request content\n" * 100 + f.request.content = b"some request content\n" * 600 f.request.trailers = Headers( [(b"my-little-request-trailer", b"foobar-request-trailer")] )