Autofix generated JS files and do not patch them in tests (#6910)

* autofix generated JS files and do not patch them in tests

* autofix: setup python

* [autofix.ci] apply automated fixes

* autofix: setup node

* add missing newline

* fixup

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Maximilian Hils 2024-06-12 01:25:46 +02:00 committed by GitHub
parent 968a169077
commit 460789b7be
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 255 additions and 181 deletions

View File

@ -15,10 +15,18 @@ jobs:
steps:
- uses: actions/checkout@v4
- uses: install-pinned/ruff@f91b0bd5d5680f7ecf60fcd37860121a4b6dadf5
- uses: actions/setup-python@v5
with:
python-version-file: .github/python-version.txt
- run: pip install -e .[dev]
- run: ruff --fix-only .
- run: ruff format .
- run: web/gen/all
- uses: actions/setup-node@v4
with:
node-version-file: .github/node-version.txt
- name: Run prettier
run: |
npm ci

View File

@ -24,7 +24,6 @@
* Releases now come with a Sigstore attestations file to demonstrate build provenance.
([f05c050](https://github.com/mitmproxy/mitmproxy/commit/f05c050f615b9ab9963707944c893bc94e738525), @mhils)
## 17 April 2024: mitmproxy 10.3.0
* Add support for editing non text files in a hex editor

View File

@ -637,16 +637,17 @@ class DnsRebind(RequestHandler):
class State(RequestHandler):
# Separate method for testability.
@staticmethod
def get_json(master: mitmproxy.tools.web.master.WebMaster):
return {
"version": version.VERSION,
"contentViews": [v.name for v in contentviews.views if v.name != "Query"],
"servers": [s.to_json() for s in master.proxyserver.servers],
}
def get(self):
self.write(
{
"version": version.VERSION,
"contentViews": [
v.name for v in contentviews.views if v.name != "Query"
],
"servers": [s.to_json() for s in self.master.proxyserver.servers],
}
)
self.write(State.get_json(self.master))
class GZipContentAndFlowFiles(tornado.web.GZipContentEncoding):

View File

@ -1,30 +1,23 @@
import gzip
import io
import importlib
import json
import logging
import textwrap
from collections.abc import Sequence
from contextlib import redirect_stdout
from pathlib import Path
from typing import Optional
from unittest import mock
from unittest.mock import Mock
import pytest
import tornado.testing
from tornado import httpclient
from tornado import websocket
from mitmproxy import certs
from mitmproxy import log
from mitmproxy import options
from mitmproxy import optmanager
from mitmproxy.http import Headers
from mitmproxy.proxy.mode_servers import ServerInstance
from mitmproxy.test import tflow
from mitmproxy.tools.web import app
from mitmproxy.tools.web import master as webmaster
here = Path(__file__).parent.absolute()
@pytest.fixture(scope="module")
def no_tornado_logging():
@ -41,118 +34,14 @@ def get_json(resp: httpclient.HTTPResponse):
return json.loads(resp.body.decode())
def test_generate_tflow_js(tdata):
tf_http = tflow.tflow(resp=True, err=True, ws=True)
tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29"
tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939"
tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8"
tf_http.server_conn.certificate_list = [
certs.Cert.from_pem(
Path(
tdata.path("mitmproxy/net/data/verificationcerts/self-signed.pem")
).read_bytes()
)
]
tf_http.request.trailers = Headers(trailer="qvalue")
tf_http.response.trailers = Headers(trailer="qvalue")
tf_http.comment = "I'm a comment!"
tf_tcp = tflow.ttcpflow(err=True)
tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001"
tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322"
tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200"
tf_udp = tflow.tudpflow(err=True)
tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951"
tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c"
tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b"
tf_dns = tflow.tdnsflow(resp=True, err=True)
tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4"
tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc"
tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f"
# language=TypeScript
content = (
"/** Auto-generated by test_app.py:test_generate_tflow_js */\n"
"import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n"
"export function THTTPFlow(): Required<HTTPFlow> {\n"
" return %s\n"
"}\n"
"export function TTCPFlow(): Required<TCPFlow> {\n"
" return %s\n"
"}\n"
"export function TUDPFlow(): Required<UDPFlow> {\n"
" return %s\n"
"}\n"
"export function TDNSFlow(): Required<DNSFlow> {\n"
" return %s\n"
"}\n"
% (
textwrap.indent(
json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " "
),
)
)
content = content.replace(": null", ": undefined")
(
Path(__file__).parent / "../../../../web/src/js/__tests__/ducks/_tflow.ts"
).write_bytes(content.encode())
async def test_generate_options_js():
o = options.Options()
m = webmaster.WebMaster(o)
opt: optmanager._Option
def ts_type(t):
if t == bool:
return "boolean"
if t == str:
return "string"
if t == int:
return "number"
if t == Sequence[str]:
return "string[]"
if t == Optional[str]:
return "string | undefined"
if t == Optional[int]:
return "number | undefined"
raise RuntimeError(t)
with redirect_stdout(io.StringIO()) as s:
print("/** Auto-generated by test_app.py:test_generate_options_js */")
print("export interface OptionsState {")
for _, opt in sorted(m.options.items()):
print(f" {opt.name}: {ts_type(opt.typespec)};")
print("}")
print("")
print("export type Option = keyof OptionsState;")
print("")
print("export const defaultState: OptionsState = {")
for _, opt in sorted(m.options.items()):
print(
f" {opt.name}: {json.dumps(opt.default)},".replace(
": null", ": undefined"
)
)
print("};")
(
Path(__file__).parent / "../../../../web/src/js/ducks/_options_gen.ts"
).write_bytes(s.getvalue().encode())
await m.done()
@pytest.mark.parametrize("filename", list((here / "../../../../web/gen").glob("*.py")))
async def test_generated_files(filename):
mod = importlib.import_module(f"web.gen.{filename.stem}")
expected = await mod.make()
actual = mod.filename.read_text().replace("\r\n", "\n")
assert (
actual == expected
), f"{mod.filename} must be regenerated by running {filename.resolve()}."
@pytest.mark.usefixtures("no_tornado_logging", "tdata")
@ -176,24 +65,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
m.view.add([tflow.tflow(err=True)])
m.events._add_log(log.LogEntry("test log", "info"))
m.events.done()
si1 = ServerInstance.make("regular", m.proxyserver)
sock1 = Mock()
sock1.getsockname.return_value = ("127.0.0.1", 8080)
sock2 = Mock()
sock2.getsockname.return_value = ("::1", 8080)
server = Mock()
server.sockets = [sock1, sock2]
si1._servers = [server]
si2 = ServerInstance.make("reverse:example.com", m.proxyserver)
si2.last_exception = RuntimeError("I failed somehow.")
si3 = ServerInstance.make("socks5", m.proxyserver)
m.proxyserver.servers._instances.update(
{
si1.mode: si1,
si2.mode: si2,
si3.mode: si3,
}
)
self.master = m
self.view = m.view
self.events = m.events
@ -500,31 +371,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
def test_option_save(self):
assert self.fetch("/options/save", method="POST").code == 200
def test_generate_state_js(self):
resp = self.fetch("/state")
assert resp.code == 200
data = json.loads(resp.body)
data.update(available=True)
data["contentViews"] = ["Auto", "Raw"]
data["version"] = "1.2.3"
# language=TypeScript
content = (
"/** Auto-generated by test_app.py:test_generate_state_js */\n"
"import {BackendState} from '../../ducks/backendState';\n"
"export function TBackendState(): Required<BackendState> {\n"
" return %s\n"
"}\n"
% textwrap.indent(
json.dumps(data, indent=4, sort_keys=True), " "
).lstrip()
)
(
Path(__file__).parent
/ "../../../../web/src/js/__tests__/ducks/_tbackendstate.ts"
).write_bytes(content.encode())
def test_err(self):
with mock.patch("mitmproxy.tools.web.app.IndexHandler.get") as f:
f.side_effect = RuntimeError

8
web/gen/all Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
for file in "$script_dir"/*.py; do
echo "$file..."
"$file"
done

64
web/gen/options_js.py Executable file
View File

@ -0,0 +1,64 @@
#!/usr/bin/env python3
import asyncio
import io
import json
from collections.abc import Sequence
from contextlib import redirect_stdout
from pathlib import Path
from mitmproxy import options
from mitmproxy import optmanager
from mitmproxy.tools.web import master
here = Path(__file__).parent.absolute()
filename = here / "../src/js/ducks/_options_gen.ts"
def _ts_type(t):
if t == bool:
return "boolean"
if t == str:
return "string"
if t == int:
return "number"
if t == Sequence[str]:
return "string[]"
if t == str | None:
return "string | undefined"
if t == int | None:
return "number | undefined"
raise RuntimeError(t)
async def make() -> str:
o = options.Options()
m = master.WebMaster(o)
opt: optmanager._Option
with redirect_stdout(io.StringIO()) as s:
print("/** Auto-generated by web/gen/options_js.py */")
print("export interface OptionsState {")
for _, opt in sorted(m.options.items()):
print(f" {opt.name}: {_ts_type(opt.typespec)};")
print("}")
print("")
print("export type Option = keyof OptionsState;")
print("")
print("export const defaultState: OptionsState = {")
for _, opt in sorted(m.options.items()):
print(
f" {opt.name}: {json.dumps(opt.default)},".replace(
": null", ": undefined"
)
)
print("};")
await m.done()
return s.getvalue()
if __name__ == "__main__":
filename.write_bytes(asyncio.run(make()).encode())

63
web/gen/state_js.py Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python3
import asyncio
import json
import textwrap
from pathlib import Path
from unittest.mock import Mock
from mitmproxy import options
from mitmproxy.proxy.mode_servers import ServerInstance
from mitmproxy.tools.web import app
from mitmproxy.tools.web import master
here = Path(__file__).parent.absolute()
filename = here / "../src/js/__tests__/ducks/_tbackendstate.ts"
async def make() -> str:
o = options.Options()
m = master.WebMaster(o)
si1 = ServerInstance.make("regular", m.proxyserver)
sock1 = Mock()
sock1.getsockname.return_value = ("127.0.0.1", 8080)
sock2 = Mock()
sock2.getsockname.return_value = ("::1", 8080)
server = Mock()
server.sockets = [sock1, sock2]
si1._servers = [server]
si2 = ServerInstance.make("reverse:example.com", m.proxyserver)
si2.last_exception = RuntimeError("I failed somehow.")
si3 = ServerInstance.make("socks5", m.proxyserver)
m.proxyserver.servers._instances.update(
{
si1.mode: si1,
si2.mode: si2,
si3.mode: si3,
}
)
data = app.State.get_json(m)
await m.done()
data.update(available=True)
data["contentViews"] = ["Auto", "Raw"]
data["version"] = "1.2.3"
# language=TypeScript
content = (
"/** Auto-generated by web/gen/state_js.py */\n"
"import {BackendState} from '../../ducks/backendState';\n"
"export function TBackendState(): Required<BackendState> {\n"
" return %s\n"
"}\n"
% textwrap.indent(json.dumps(data, indent=4, sort_keys=True), " ").lstrip()
)
return content
if __name__ == "__main__":
filename.write_bytes(asyncio.run(make()).encode())

85
web/gen/tflow_js.py Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env python3
import asyncio
import json
import textwrap
from pathlib import Path
from mitmproxy import certs
from mitmproxy.http import Headers
from mitmproxy.test import tflow
from mitmproxy.tools.web import app
here = Path(__file__).parent.absolute()
filename = here / "../src/js/__tests__/ducks/_tflow.ts"
async def make() -> str:
tf_http = tflow.tflow(resp=True, err=True, ws=True)
tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29"
tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939"
tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8"
tf_http.server_conn.certificate_list = [
certs.Cert.from_pem(
(
here / "../../test/mitmproxy/net/data/verificationcerts/self-signed.pem"
).read_bytes()
)
]
tf_http.request.trailers = Headers(trailer="qvalue")
tf_http.response.trailers = Headers(trailer="qvalue")
tf_http.comment = "I'm a comment!"
tf_tcp = tflow.ttcpflow(err=True)
tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001"
tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322"
tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200"
tf_udp = tflow.tudpflow(err=True)
tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951"
tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c"
tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b"
tf_dns = tflow.tdnsflow(resp=True, err=True)
tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4"
tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc"
tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f"
# language=TypeScript
content = (
"/** Auto-generated by web/gen/tflow_js.py */\n"
"import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n"
"export function THTTPFlow(): Required<HTTPFlow> {\n"
" return %s\n"
"}\n"
"export function TTCPFlow(): Required<TCPFlow> {\n"
" return %s\n"
"}\n"
"export function TUDPFlow(): Required<UDPFlow> {\n"
" return %s\n"
"}\n"
"export function TDNSFlow(): Required<DNSFlow> {\n"
" return %s\n"
"}\n"
% (
textwrap.indent(
json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " "
),
textwrap.indent(
json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " "
),
)
)
content = content.replace(": null", ": undefined")
return content
if __name__ == "__main__":
filename.write_bytes(asyncio.run(make()).encode())

View File

@ -1,4 +1,4 @@
/** Auto-generated by test_app.py:test_generate_state_js */
/** Auto-generated by web/gen/state_js.py */
import {BackendState} from '../../ducks/backendState';
export function TBackendState(): Required<BackendState> {
return {

View File

@ -1,4 +1,4 @@
/** Auto-generated by test_app.py:test_generate_tflow_js */
/** Auto-generated by web/gen/tflow_js.py */
import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';
export function THTTPFlow(): Required<HTTPFlow> {
return {

View File

@ -1,10 +1,9 @@
/** Auto-generated by test_app.py:test_generate_options_js */
/** Auto-generated by web/gen/options_js.py */
export interface OptionsState {
add_upstream_certs_to_client_chain: boolean;
allow_hosts: string[];
anticache: boolean;
anticomp: boolean;
block_ech: boolean;
block_global: boolean;
block_list: string[];
block_private: boolean;
@ -70,6 +69,7 @@ export interface OptionsState {
stickyauth: string | undefined;
stickycookie: string | undefined;
stream_large_bodies: string | undefined;
strip_ech: boolean;
tcp_hosts: string[];
termlog_verbosity: string;
tls_ecdh_curve_client: string | undefined;
@ -101,7 +101,6 @@ export const defaultState: OptionsState = {
allow_hosts: [],
anticache: false,
anticomp: false,
block_ech: true,
block_global: true,
block_list: [],
block_private: false,
@ -167,6 +166,7 @@ export const defaultState: OptionsState = {
stickyauth: undefined,
stickycookie: undefined,
stream_large_bodies: undefined,
strip_ech: true,
tcp_hosts: [],
termlog_verbosity: "info",
tls_ecdh_curve_client: undefined,