Autofix generated JS files and do not patch them in tests (#6910)
* autofix generated JS files and do not patch them in tests * autofix: setup python * [autofix.ci] apply automated fixes * autofix: setup node * add missing newline * fixup --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
parent
968a169077
commit
460789b7be
|
@ -15,10 +15,18 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: install-pinned/ruff@f91b0bd5d5680f7ecf60fcd37860121a4b6dadf5
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version-file: .github/python-version.txt
|
||||
- run: pip install -e .[dev]
|
||||
- run: ruff --fix-only .
|
||||
- run: ruff format .
|
||||
|
||||
- run: web/gen/all
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .github/node-version.txt
|
||||
- name: Run prettier
|
||||
run: |
|
||||
npm ci
|
||||
|
|
|
@ -24,7 +24,6 @@
|
|||
* Releases now come with a Sigstore attestations file to demonstrate build provenance.
|
||||
([f05c050](https://github.com/mitmproxy/mitmproxy/commit/f05c050f615b9ab9963707944c893bc94e738525), @mhils)
|
||||
|
||||
|
||||
## 17 April 2024: mitmproxy 10.3.0
|
||||
|
||||
* Add support for editing non text files in a hex editor
|
||||
|
|
|
@ -637,16 +637,17 @@ class DnsRebind(RequestHandler):
|
|||
|
||||
|
||||
class State(RequestHandler):
|
||||
# Separate method for testability.
|
||||
@staticmethod
|
||||
def get_json(master: mitmproxy.tools.web.master.WebMaster):
|
||||
return {
|
||||
"version": version.VERSION,
|
||||
"contentViews": [v.name for v in contentviews.views if v.name != "Query"],
|
||||
"servers": [s.to_json() for s in master.proxyserver.servers],
|
||||
}
|
||||
|
||||
def get(self):
|
||||
self.write(
|
||||
{
|
||||
"version": version.VERSION,
|
||||
"contentViews": [
|
||||
v.name for v in contentviews.views if v.name != "Query"
|
||||
],
|
||||
"servers": [s.to_json() for s in self.master.proxyserver.servers],
|
||||
}
|
||||
)
|
||||
self.write(State.get_json(self.master))
|
||||
|
||||
|
||||
class GZipContentAndFlowFiles(tornado.web.GZipContentEncoding):
|
||||
|
|
|
@ -1,30 +1,23 @@
|
|||
import gzip
|
||||
import io
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import textwrap
|
||||
from collections.abc import Sequence
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import tornado.testing
|
||||
from tornado import httpclient
|
||||
from tornado import websocket
|
||||
|
||||
from mitmproxy import certs
|
||||
from mitmproxy import log
|
||||
from mitmproxy import options
|
||||
from mitmproxy import optmanager
|
||||
from mitmproxy.http import Headers
|
||||
from mitmproxy.proxy.mode_servers import ServerInstance
|
||||
from mitmproxy.test import tflow
|
||||
from mitmproxy.tools.web import app
|
||||
from mitmproxy.tools.web import master as webmaster
|
||||
|
||||
here = Path(__file__).parent.absolute()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def no_tornado_logging():
|
||||
|
@ -41,118 +34,14 @@ def get_json(resp: httpclient.HTTPResponse):
|
|||
return json.loads(resp.body.decode())
|
||||
|
||||
|
||||
def test_generate_tflow_js(tdata):
|
||||
tf_http = tflow.tflow(resp=True, err=True, ws=True)
|
||||
tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29"
|
||||
tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939"
|
||||
tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8"
|
||||
tf_http.server_conn.certificate_list = [
|
||||
certs.Cert.from_pem(
|
||||
Path(
|
||||
tdata.path("mitmproxy/net/data/verificationcerts/self-signed.pem")
|
||||
).read_bytes()
|
||||
)
|
||||
]
|
||||
tf_http.request.trailers = Headers(trailer="qvalue")
|
||||
tf_http.response.trailers = Headers(trailer="qvalue")
|
||||
tf_http.comment = "I'm a comment!"
|
||||
|
||||
tf_tcp = tflow.ttcpflow(err=True)
|
||||
tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001"
|
||||
tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322"
|
||||
tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200"
|
||||
|
||||
tf_udp = tflow.tudpflow(err=True)
|
||||
tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951"
|
||||
tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c"
|
||||
tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b"
|
||||
|
||||
tf_dns = tflow.tdnsflow(resp=True, err=True)
|
||||
tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4"
|
||||
tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc"
|
||||
tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f"
|
||||
|
||||
# language=TypeScript
|
||||
content = (
|
||||
"/** Auto-generated by test_app.py:test_generate_tflow_js */\n"
|
||||
"import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n"
|
||||
"export function THTTPFlow(): Required<HTTPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TTCPFlow(): Required<TCPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TUDPFlow(): Required<UDPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TDNSFlow(): Required<DNSFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
% (
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " "
|
||||
),
|
||||
)
|
||||
)
|
||||
content = content.replace(": null", ": undefined")
|
||||
|
||||
(
|
||||
Path(__file__).parent / "../../../../web/src/js/__tests__/ducks/_tflow.ts"
|
||||
).write_bytes(content.encode())
|
||||
|
||||
|
||||
async def test_generate_options_js():
|
||||
o = options.Options()
|
||||
m = webmaster.WebMaster(o)
|
||||
opt: optmanager._Option
|
||||
|
||||
def ts_type(t):
|
||||
if t == bool:
|
||||
return "boolean"
|
||||
if t == str:
|
||||
return "string"
|
||||
if t == int:
|
||||
return "number"
|
||||
if t == Sequence[str]:
|
||||
return "string[]"
|
||||
if t == Optional[str]:
|
||||
return "string | undefined"
|
||||
if t == Optional[int]:
|
||||
return "number | undefined"
|
||||
raise RuntimeError(t)
|
||||
|
||||
with redirect_stdout(io.StringIO()) as s:
|
||||
print("/** Auto-generated by test_app.py:test_generate_options_js */")
|
||||
|
||||
print("export interface OptionsState {")
|
||||
for _, opt in sorted(m.options.items()):
|
||||
print(f" {opt.name}: {ts_type(opt.typespec)};")
|
||||
print("}")
|
||||
print("")
|
||||
print("export type Option = keyof OptionsState;")
|
||||
print("")
|
||||
print("export const defaultState: OptionsState = {")
|
||||
for _, opt in sorted(m.options.items()):
|
||||
print(
|
||||
f" {opt.name}: {json.dumps(opt.default)},".replace(
|
||||
": null", ": undefined"
|
||||
)
|
||||
)
|
||||
print("};")
|
||||
|
||||
(
|
||||
Path(__file__).parent / "../../../../web/src/js/ducks/_options_gen.ts"
|
||||
).write_bytes(s.getvalue().encode())
|
||||
await m.done()
|
||||
@pytest.mark.parametrize("filename", list((here / "../../../../web/gen").glob("*.py")))
|
||||
async def test_generated_files(filename):
|
||||
mod = importlib.import_module(f"web.gen.{filename.stem}")
|
||||
expected = await mod.make()
|
||||
actual = mod.filename.read_text().replace("\r\n", "\n")
|
||||
assert (
|
||||
actual == expected
|
||||
), f"{mod.filename} must be regenerated by running {filename.resolve()}."
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("no_tornado_logging", "tdata")
|
||||
|
@ -176,24 +65,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
|
|||
m.view.add([tflow.tflow(err=True)])
|
||||
m.events._add_log(log.LogEntry("test log", "info"))
|
||||
m.events.done()
|
||||
si1 = ServerInstance.make("regular", m.proxyserver)
|
||||
sock1 = Mock()
|
||||
sock1.getsockname.return_value = ("127.0.0.1", 8080)
|
||||
sock2 = Mock()
|
||||
sock2.getsockname.return_value = ("::1", 8080)
|
||||
server = Mock()
|
||||
server.sockets = [sock1, sock2]
|
||||
si1._servers = [server]
|
||||
si2 = ServerInstance.make("reverse:example.com", m.proxyserver)
|
||||
si2.last_exception = RuntimeError("I failed somehow.")
|
||||
si3 = ServerInstance.make("socks5", m.proxyserver)
|
||||
m.proxyserver.servers._instances.update(
|
||||
{
|
||||
si1.mode: si1,
|
||||
si2.mode: si2,
|
||||
si3.mode: si3,
|
||||
}
|
||||
)
|
||||
self.master = m
|
||||
self.view = m.view
|
||||
self.events = m.events
|
||||
|
@ -500,31 +371,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
|
|||
def test_option_save(self):
|
||||
assert self.fetch("/options/save", method="POST").code == 200
|
||||
|
||||
def test_generate_state_js(self):
|
||||
resp = self.fetch("/state")
|
||||
assert resp.code == 200
|
||||
data = json.loads(resp.body)
|
||||
data.update(available=True)
|
||||
data["contentViews"] = ["Auto", "Raw"]
|
||||
data["version"] = "1.2.3"
|
||||
|
||||
# language=TypeScript
|
||||
content = (
|
||||
"/** Auto-generated by test_app.py:test_generate_state_js */\n"
|
||||
"import {BackendState} from '../../ducks/backendState';\n"
|
||||
"export function TBackendState(): Required<BackendState> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
% textwrap.indent(
|
||||
json.dumps(data, indent=4, sort_keys=True), " "
|
||||
).lstrip()
|
||||
)
|
||||
|
||||
(
|
||||
Path(__file__).parent
|
||||
/ "../../../../web/src/js/__tests__/ducks/_tbackendstate.ts"
|
||||
).write_bytes(content.encode())
|
||||
|
||||
def test_err(self):
|
||||
with mock.patch("mitmproxy.tools.web.app.IndexHandler.get") as f:
|
||||
f.side_effect = RuntimeError
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
for file in "$script_dir"/*.py; do
|
||||
echo "$file..."
|
||||
"$file"
|
||||
done
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
|
||||
from mitmproxy import options
|
||||
from mitmproxy import optmanager
|
||||
from mitmproxy.tools.web import master
|
||||
|
||||
here = Path(__file__).parent.absolute()
|
||||
|
||||
filename = here / "../src/js/ducks/_options_gen.ts"
|
||||
|
||||
|
||||
def _ts_type(t):
|
||||
if t == bool:
|
||||
return "boolean"
|
||||
if t == str:
|
||||
return "string"
|
||||
if t == int:
|
||||
return "number"
|
||||
if t == Sequence[str]:
|
||||
return "string[]"
|
||||
if t == str | None:
|
||||
return "string | undefined"
|
||||
if t == int | None:
|
||||
return "number | undefined"
|
||||
raise RuntimeError(t)
|
||||
|
||||
|
||||
async def make() -> str:
|
||||
o = options.Options()
|
||||
m = master.WebMaster(o)
|
||||
opt: optmanager._Option
|
||||
|
||||
with redirect_stdout(io.StringIO()) as s:
|
||||
print("/** Auto-generated by web/gen/options_js.py */")
|
||||
|
||||
print("export interface OptionsState {")
|
||||
for _, opt in sorted(m.options.items()):
|
||||
print(f" {opt.name}: {_ts_type(opt.typespec)};")
|
||||
print("}")
|
||||
print("")
|
||||
print("export type Option = keyof OptionsState;")
|
||||
print("")
|
||||
print("export const defaultState: OptionsState = {")
|
||||
for _, opt in sorted(m.options.items()):
|
||||
print(
|
||||
f" {opt.name}: {json.dumps(opt.default)},".replace(
|
||||
": null", ": undefined"
|
||||
)
|
||||
)
|
||||
print("};")
|
||||
|
||||
await m.done()
|
||||
return s.getvalue()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
filename.write_bytes(asyncio.run(make()).encode())
|
|
@ -0,0 +1,63 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
from mitmproxy import options
|
||||
from mitmproxy.proxy.mode_servers import ServerInstance
|
||||
from mitmproxy.tools.web import app
|
||||
from mitmproxy.tools.web import master
|
||||
|
||||
here = Path(__file__).parent.absolute()
|
||||
|
||||
filename = here / "../src/js/__tests__/ducks/_tbackendstate.ts"
|
||||
|
||||
|
||||
async def make() -> str:
|
||||
o = options.Options()
|
||||
m = master.WebMaster(o)
|
||||
|
||||
si1 = ServerInstance.make("regular", m.proxyserver)
|
||||
sock1 = Mock()
|
||||
sock1.getsockname.return_value = ("127.0.0.1", 8080)
|
||||
sock2 = Mock()
|
||||
sock2.getsockname.return_value = ("::1", 8080)
|
||||
server = Mock()
|
||||
server.sockets = [sock1, sock2]
|
||||
si1._servers = [server]
|
||||
si2 = ServerInstance.make("reverse:example.com", m.proxyserver)
|
||||
si2.last_exception = RuntimeError("I failed somehow.")
|
||||
si3 = ServerInstance.make("socks5", m.proxyserver)
|
||||
m.proxyserver.servers._instances.update(
|
||||
{
|
||||
si1.mode: si1,
|
||||
si2.mode: si2,
|
||||
si3.mode: si3,
|
||||
}
|
||||
)
|
||||
|
||||
data = app.State.get_json(m)
|
||||
await m.done()
|
||||
|
||||
data.update(available=True)
|
||||
data["contentViews"] = ["Auto", "Raw"]
|
||||
data["version"] = "1.2.3"
|
||||
|
||||
# language=TypeScript
|
||||
content = (
|
||||
"/** Auto-generated by web/gen/state_js.py */\n"
|
||||
"import {BackendState} from '../../ducks/backendState';\n"
|
||||
"export function TBackendState(): Required<BackendState> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
% textwrap.indent(json.dumps(data, indent=4, sort_keys=True), " ").lstrip()
|
||||
)
|
||||
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
filename.write_bytes(asyncio.run(make()).encode())
|
|
@ -0,0 +1,85 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
from mitmproxy import certs
|
||||
from mitmproxy.http import Headers
|
||||
from mitmproxy.test import tflow
|
||||
from mitmproxy.tools.web import app
|
||||
|
||||
here = Path(__file__).parent.absolute()
|
||||
|
||||
filename = here / "../src/js/__tests__/ducks/_tflow.ts"
|
||||
|
||||
|
||||
async def make() -> str:
|
||||
tf_http = tflow.tflow(resp=True, err=True, ws=True)
|
||||
tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29"
|
||||
tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939"
|
||||
tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8"
|
||||
tf_http.server_conn.certificate_list = [
|
||||
certs.Cert.from_pem(
|
||||
(
|
||||
here / "../../test/mitmproxy/net/data/verificationcerts/self-signed.pem"
|
||||
).read_bytes()
|
||||
)
|
||||
]
|
||||
tf_http.request.trailers = Headers(trailer="qvalue")
|
||||
tf_http.response.trailers = Headers(trailer="qvalue")
|
||||
tf_http.comment = "I'm a comment!"
|
||||
|
||||
tf_tcp = tflow.ttcpflow(err=True)
|
||||
tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001"
|
||||
tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322"
|
||||
tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200"
|
||||
|
||||
tf_udp = tflow.tudpflow(err=True)
|
||||
tf_udp.id = "f9f7b2b9-7727-4477-822d-d3526e5b8951"
|
||||
tf_udp.client_conn.id = "0a8833da-88e4-429d-ac54-61cda8a7f91c"
|
||||
tf_udp.server_conn.id = "c49f9c2b-a729-4b16-9212-d181717e294b"
|
||||
|
||||
tf_dns = tflow.tdnsflow(resp=True, err=True)
|
||||
tf_dns.id = "5434da94-1017-42fa-872d-a189508d48e4"
|
||||
tf_dns.client_conn.id = "0b4cc0a3-6acb-4880-81c0-1644084126fc"
|
||||
tf_dns.server_conn.id = "db5294af-c008-4098-a320-a94f901eaf2f"
|
||||
|
||||
# language=TypeScript
|
||||
content = (
|
||||
"/** Auto-generated by web/gen/tflow_js.py */\n"
|
||||
"import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';\n"
|
||||
"export function THTTPFlow(): Required<HTTPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TTCPFlow(): Required<TCPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TUDPFlow(): Required<UDPFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
"export function TDNSFlow(): Required<DNSFlow> {\n"
|
||||
" return %s\n"
|
||||
"}\n"
|
||||
% (
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_udp), indent=4, sort_keys=True), " "
|
||||
),
|
||||
textwrap.indent(
|
||||
json.dumps(app.flow_to_json(tf_dns), indent=4, sort_keys=True), " "
|
||||
),
|
||||
)
|
||||
)
|
||||
content = content.replace(": null", ": undefined")
|
||||
return content
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
filename.write_bytes(asyncio.run(make()).encode())
|
|
@ -1,4 +1,4 @@
|
|||
/** Auto-generated by test_app.py:test_generate_state_js */
|
||||
/** Auto-generated by web/gen/state_js.py */
|
||||
import {BackendState} from '../../ducks/backendState';
|
||||
export function TBackendState(): Required<BackendState> {
|
||||
return {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
/** Auto-generated by test_app.py:test_generate_tflow_js */
|
||||
/** Auto-generated by web/gen/tflow_js.py */
|
||||
import {HTTPFlow, TCPFlow, UDPFlow, DNSFlow} from '../../flow';
|
||||
export function THTTPFlow(): Required<HTTPFlow> {
|
||||
return {
|
||||
|
|
|
@ -1,10 +1,9 @@
|
|||
/** Auto-generated by test_app.py:test_generate_options_js */
|
||||
/** Auto-generated by web/gen/options_js.py */
|
||||
export interface OptionsState {
|
||||
add_upstream_certs_to_client_chain: boolean;
|
||||
allow_hosts: string[];
|
||||
anticache: boolean;
|
||||
anticomp: boolean;
|
||||
block_ech: boolean;
|
||||
block_global: boolean;
|
||||
block_list: string[];
|
||||
block_private: boolean;
|
||||
|
@ -70,6 +69,7 @@ export interface OptionsState {
|
|||
stickyauth: string | undefined;
|
||||
stickycookie: string | undefined;
|
||||
stream_large_bodies: string | undefined;
|
||||
strip_ech: boolean;
|
||||
tcp_hosts: string[];
|
||||
termlog_verbosity: string;
|
||||
tls_ecdh_curve_client: string | undefined;
|
||||
|
@ -101,7 +101,6 @@ export const defaultState: OptionsState = {
|
|||
allow_hosts: [],
|
||||
anticache: false,
|
||||
anticomp: false,
|
||||
block_ech: true,
|
||||
block_global: true,
|
||||
block_list: [],
|
||||
block_private: false,
|
||||
|
@ -167,6 +166,7 @@ export const defaultState: OptionsState = {
|
|||
stickyauth: undefined,
|
||||
stickycookie: undefined,
|
||||
stream_large_bodies: undefined,
|
||||
strip_ech: true,
|
||||
tcp_hosts: [],
|
||||
termlog_verbosity: "info",
|
||||
tls_ecdh_curve_client: undefined,
|
||||
|
|
Loading…
Reference in New Issue