diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index e018b813b..725848ef8 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -15,10 +15,18 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: install-pinned/ruff@f91b0bd5d5680f7ecf60fcd37860121a4b6dadf5 + - uses: actions/setup-python@v5 + with: + python-version-file: .github/python-version.txt + - run: pip install -e .[dev] - run: ruff --fix-only . - run: ruff format . + - run: web/gen/all + + - uses: actions/setup-node@v4 + with: + node-version-file: .github/node-version.txt - name: Run prettier run: | npm ci diff --git a/CHANGELOG.md b/CHANGELOG.md index 618d43c9e..fdb895348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,6 @@ * Releases now come with a Sigstore attestations file to demonstrate build provenance. ([f05c050](https://github.com/mitmproxy/mitmproxy/commit/f05c050f615b9ab9963707944c893bc94e738525), @mhils) - ## 17 April 2024: mitmproxy 10.3.0 * Add support for editing non text files in a hex editor diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 7ea24351a..578af252f 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -637,16 +637,17 @@ class DnsRebind(RequestHandler): class State(RequestHandler): + # Separate method for testability. + @staticmethod + def get_json(master: mitmproxy.tools.web.master.WebMaster): + return { + "version": version.VERSION, + "contentViews": [v.name for v in contentviews.views if v.name != "Query"], + "servers": [s.to_json() for s in master.proxyserver.servers], + } + def get(self): - self.write( - { - "version": version.VERSION, - "contentViews": [ - v.name for v in contentviews.views if v.name != "Query" - ], - "servers": [s.to_json() for s in self.master.proxyserver.servers], - } - ) + self.write(State.get_json(self.master)) class GZipContentAndFlowFiles(tornado.web.GZipContentEncoding): diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 83f599940..2e9b7f36e 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -1,30 +1,23 @@ import gzip -import io +import importlib import json import logging -import textwrap -from collections.abc import Sequence -from contextlib import redirect_stdout from pathlib import Path -from typing import Optional from unittest import mock -from unittest.mock import Mock import pytest import tornado.testing from tornado import httpclient from tornado import websocket -from mitmproxy import certs from mitmproxy import log from mitmproxy import options -from mitmproxy import optmanager -from mitmproxy.http import Headers -from mitmproxy.proxy.mode_servers import ServerInstance from mitmproxy.test import tflow from mitmproxy.tools.web import app from mitmproxy.tools.web import master as webmaster +here = Path(__file__).parent.absolute() + @pytest.fixture(scope="module") def no_tornado_logging(): @@ -41,118 +34,14 @@ def get_json(resp: httpclient.HTTPResponse): return json.loads(resp.body.decode()) -def test_generate_tflow_js(tdata): - tf_http = tflow.tflow(resp=True, err=True, ws=True) - tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29" - tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939" - tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8" - tf_http.server_conn.certificate_list = [ - certs.Cert.from_pem( - Path( - tdata.path("mitmproxy/net/data/verificationcerts/self-signed.pem") - ).read_bytes() - ) - ] - tf_http.request.trailers = Headers(trailer="qvalue") - tf_http.response.trailers = Headers(trailer="qvalue") - tf_http.comment = "I'm a comment!" - - tf_tcp = tflow.ttcpflow(err=True) - tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001" - tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322" - tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200" - - tf_udp = tflow.tudpflow(err=True) - tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951" - tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c" - tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b" - - tf_dns = tflow.tdnsflow(resp=True, err=True) - tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4" - tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc" - tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f" - - # language=TypeScript - content = ( - "/** Auto-generated by test_app.py:test_generate_tflow_js */\n" - "import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n" - "export function THTTPFlow(): Required {\n" - " return %s\n" - "}\n" - "export function TTCPFlow(): Required {\n" - " return %s\n" - "}\n" - "export function TUDPFlow(): Required {\n" - " return %s\n" - "}\n" - "export function TDNSFlow(): Required {\n" - " return %s\n" - "}\n" - % ( - textwrap.indent( - json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " " - ), - textwrap.indent( - json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " " - ), - textwrap.indent( - json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " " - ), - textwrap.indent( - json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " " - ), - ) - ) - content = content.replace(": null", ": undefined") - - ( - Path(__file__).parent / "../../../../web/src/js/__tests__/ducks/_tflow.ts" - ).write_bytes(content.encode()) - - -async def test_generate_options_js(): - o = options.Options() - m = webmaster.WebMaster(o) - opt: optmanager._Option - - def ts_type(t): - if t == bool: - return "boolean" - if t == str: - return "string" - if t == int: - return "number" - if t == Sequence[str]: - return "string[]" - if t == Optional[str]: - return "string | undefined" - if t == Optional[int]: - return "number | undefined" - raise RuntimeError(t) - - with redirect_stdout(io.StringIO()) as s: - print("/** Auto-generated by test_app.py:test_generate_options_js */") - - print("export interface OptionsState {") - for _, opt in sorted(m.options.items()): - print(f" {opt.name}: {ts_type(opt.typespec)};") - print("}") - print("") - print("export type Option = keyof OptionsState;") - print("") - print("export const defaultState: OptionsState = {") - for _, opt in sorted(m.options.items()): - print( - f" {opt.name}: {json.dumps(opt.default)},".replace( - ": null", ": undefined" - ) - ) - print("};") - - ( - Path(__file__).parent / "../../../../web/src/js/ducks/_options_gen.ts" - ).write_bytes(s.getvalue().encode()) - await m.done() +@pytest.mark.parametrize("filename", list((here / "../../../../web/gen").glob("*.py"))) +async def test_generated_files(filename): + mod = importlib.import_module(f"web.gen.{filename.stem}") + expected = await mod.make() + actual = mod.filename.read_text().replace("\r\n", "\n") + assert ( + actual == expected + ), f"{mod.filename} must be regenerated by running {filename.resolve()}." @pytest.mark.usefixtures("no_tornado_logging", "tdata") @@ -176,24 +65,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): m.view.add([tflow.tflow(err=True)]) m.events._add_log(log.LogEntry("test log", "info")) m.events.done() - si1 = ServerInstance.make("regular", m.proxyserver) - sock1 = Mock() - sock1.getsockname.return_value = ("127.0.0.1", 8080) - sock2 = Mock() - sock2.getsockname.return_value = ("::1", 8080) - server = Mock() - server.sockets = [sock1, sock2] - si1._servers = [server] - si2 = ServerInstance.make("reverse:example.com", m.proxyserver) - si2.last_exception = RuntimeError("I failed somehow.") - si3 = ServerInstance.make("socks5", m.proxyserver) - m.proxyserver.servers._instances.update( - { - si1.mode: si1, - si2.mode: si2, - si3.mode: si3, - } - ) self.master = m self.view = m.view self.events = m.events @@ -500,31 +371,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): def test_option_save(self): assert self.fetch("/options/save", method="POST").code == 200 - def test_generate_state_js(self): - resp = self.fetch("/state") - assert resp.code == 200 - data = json.loads(resp.body) - data.update(available=True) - data["contentViews"] = ["Auto", "Raw"] - data["version"] = "1.2.3" - - # language=TypeScript - content = ( - "/** Auto-generated by test_app.py:test_generate_state_js */\n" - "import {BackendState} from '../../ducks/backendState';\n" - "export function TBackendState(): Required {\n" - " return %s\n" - "}\n" - % textwrap.indent( - json.dumps(data, indent=4, sort_keys=True), " " - ).lstrip() - ) - - ( - Path(__file__).parent - / "../../../../web/src/js/__tests__/ducks/_tbackendstate.ts" - ).write_bytes(content.encode()) - def test_err(self): with mock.patch("mitmproxy.tools.web.app.IndexHandler.get") as f: f.side_effect = RuntimeError diff --git a/web/gen/all b/web/gen/all new file mode 100755 index 000000000..b95734da8 --- /dev/null +++ b/web/gen/all @@ -0,0 +1,8 @@ +#!/bin/bash + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +for file in "$script_dir"/*.py; do + echo "$file..." + "$file" +done diff --git a/web/gen/options_js.py b/web/gen/options_js.py new file mode 100755 index 000000000..d9ea7c66c --- /dev/null +++ b/web/gen/options_js.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +import asyncio +import io +import json +from collections.abc import Sequence +from contextlib import redirect_stdout +from pathlib import Path + +from mitmproxy import options +from mitmproxy import optmanager +from mitmproxy.tools.web import master + +here = Path(__file__).parent.absolute() + +filename = here / "../src/js/ducks/_options_gen.ts" + + +def _ts_type(t): + if t == bool: + return "boolean" + if t == str: + return "string" + if t == int: + return "number" + if t == Sequence[str]: + return "string[]" + if t == str | None: + return "string | undefined" + if t == int | None: + return "number | undefined" + raise RuntimeError(t) + + +async def make() -> str: + o = options.Options() + m = master.WebMaster(o) + opt: optmanager._Option + + with redirect_stdout(io.StringIO()) as s: + print("/** Auto-generated by web/gen/options_js.py */") + + print("export interface OptionsState {") + for _, opt in sorted(m.options.items()): + print(f" {opt.name}: {_ts_type(opt.typespec)};") + print("}") + print("") + print("export type Option = keyof OptionsState;") + print("") + print("export const defaultState: OptionsState = {") + for _, opt in sorted(m.options.items()): + print( + f" {opt.name}: {json.dumps(opt.default)},".replace( + ": null", ": undefined" + ) + ) + print("};") + + await m.done() + return s.getvalue() + + +if __name__ == "__main__": + filename.write_bytes(asyncio.run(make()).encode()) diff --git a/web/gen/state_js.py b/web/gen/state_js.py new file mode 100755 index 000000000..6988a0528 --- /dev/null +++ b/web/gen/state_js.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import asyncio +import json +import textwrap +from pathlib import Path +from unittest.mock import Mock + +from mitmproxy import options +from mitmproxy.proxy.mode_servers import ServerInstance +from mitmproxy.tools.web import app +from mitmproxy.tools.web import master + +here = Path(__file__).parent.absolute() + +filename = here / "../src/js/__tests__/ducks/_tbackendstate.ts" + + +async def make() -> str: + o = options.Options() + m = master.WebMaster(o) + + si1 = ServerInstance.make("regular", m.proxyserver) + sock1 = Mock() + sock1.getsockname.return_value = ("127.0.0.1", 8080) + sock2 = Mock() + sock2.getsockname.return_value = ("::1", 8080) + server = Mock() + server.sockets = [sock1, sock2] + si1._servers = [server] + si2 = ServerInstance.make("reverse:example.com", m.proxyserver) + si2.last_exception = RuntimeError("I failed somehow.") + si3 = ServerInstance.make("socks5", m.proxyserver) + m.proxyserver.servers._instances.update( + { + si1.mode: si1, + si2.mode: si2, + si3.mode: si3, + } + ) + + data = app.State.get_json(m) + await m.done() + + data.update(available=True) + data["contentViews"] = ["Auto", "Raw"] + data["version"] = "1.2.3" + + # language=TypeScript + content = ( + "/** Auto-generated by web/gen/state_js.py */\n" + "import {BackendState} from '../../ducks/backendState';\n" + "export function TBackendState(): Required {\n" + " return %s\n" + "}\n" + % textwrap.indent(json.dumps(data, indent=4, sort_keys=True), " ").lstrip() + ) + + return content + + +if __name__ == "__main__": + filename.write_bytes(asyncio.run(make()).encode()) diff --git a/web/gen/tflow_js.py b/web/gen/tflow_js.py new file mode 100755 index 000000000..32d71c84d --- /dev/null +++ b/web/gen/tflow_js.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +import asyncio +import json +import textwrap +from pathlib import Path + +from mitmproxy import certs +from mitmproxy.http import Headers +from mitmproxy.test import tflow +from mitmproxy.tools.web import app + +here = Path(__file__).parent.absolute() + +filename = here / "../src/js/__tests__/ducks/_tflow.ts" + + +async def make() -> str: + tf_http = tflow.tflow(resp=True, err=True, ws=True) + tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29" + tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939" + tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8" + tf_http.server_conn.certificate_list = [ + certs.Cert.from_pem( + ( + here / "../../test/mitmproxy/net/data/verificationcerts/self-signed.pem" + ).read_bytes() + ) + ] + tf_http.request.trailers = Headers(trailer="qvalue") + tf_http.response.trailers = Headers(trailer="qvalue") + tf_http.comment = "I'm a comment!" + + tf_tcp = tflow.ttcpflow(err=True) + tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001" + tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322" + tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200" + + tf_udp = tflow.tudpflow(err=True) + tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951" + tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c" + tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b" + + tf_dns = tflow.tdnsflow(resp=True, err=True) + tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4" + tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc" + tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f" + + # language=TypeScript + content = ( + "/** Auto-generated by web/gen/tflow_js.py */\n" + "import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n" + "export function THTTPFlow(): Required {\n" + " return %s\n" + "}\n" + "export function TTCPFlow(): Required {\n" + " return %s\n" + "}\n" + "export function TUDPFlow(): Required {\n" + " return %s\n" + "}\n" + "export function TDNSFlow(): Required {\n" + " return %s\n" + "}\n" + % ( + textwrap.indent( + json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " " + ), + textwrap.indent( + json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " " + ), + textwrap.indent( + json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " " + ), + textwrap.indent( + json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " " + ), + ) + ) + content = content.replace(": null", ": undefined") + return content + + +if __name__ == "__main__": + filename.write_bytes(asyncio.run(make()).encode()) diff --git a/web/src/js/__tests__/ducks/_tbackendstate.ts b/web/src/js/__tests__/ducks/_tbackendstate.ts index a847f318d..2552b52dc 100644 --- a/web/src/js/__tests__/ducks/_tbackendstate.ts +++ b/web/src/js/__tests__/ducks/_tbackendstate.ts @@ -1,4 +1,4 @@ -/** Auto-generated by test_app.py:test_generate_state_js */ +/** Auto-generated by web/gen/state_js.py */ import {BackendState} from '../../ducks/backendState'; export function TBackendState(): Required { return { diff --git a/web/src/js/__tests__/ducks/_tflow.ts b/web/src/js/__tests__/ducks/_tflow.ts index 025fccdfe..9fc122d9a 100644 --- a/web/src/js/__tests__/ducks/_tflow.ts +++ b/web/src/js/__tests__/ducks/_tflow.ts @@ -1,4 +1,4 @@ -/** Auto-generated by test_app.py:test_generate_tflow_js */ +/** Auto-generated by web/gen/tflow_js.py */ import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow'; export function THTTPFlow(): Required { return { diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index d94b9fc62..cb1281064 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -1,10 +1,9 @@ -/** Auto-generated by test_app.py:test_generate_options_js */ +/** Auto-generated by web/gen/options_js.py */ export interface OptionsState { add_upstream_certs_to_client_chain: boolean; allow_hosts: string[]; anticache: boolean; anticomp: boolean; - block_ech: boolean; block_global: boolean; block_list: string[]; block_private: boolean; @@ -70,6 +69,7 @@ export interface OptionsState { stickyauth: string | undefined; stickycookie: string | undefined; stream_large_bodies: string | undefined; + strip_ech: boolean; tcp_hosts: string[]; termlog_verbosity: string; tls_ecdh_curve_client: string | undefined; @@ -101,7 +101,6 @@ export const defaultState: OptionsState = { allow_hosts: [], anticache: false, anticomp: false, - block_ech: true, block_global: true, block_list: [], block_private: false, @@ -167,6 +166,7 @@ export const defaultState: OptionsState = { stickyauth: undefined, stickycookie: undefined, stream_large_bodies: undefined, + strip_ech: true, tcp_hosts: [], termlog_verbosity: "info", tls_ecdh_curve_client: undefined,