From b6d52fc8abf58a12128d4c1dba23e20655da0c52 Mon Sep 17 00:00:00 2001 From: Martin Plattner Date: Thu, 3 Sep 2020 17:14:54 +0200 Subject: [PATCH] add cli recording scripts --- docs/build.sh | 2 +- docs/scripts/clirecording/Dockerfile | 45 +++ docs/scripts/clirecording/clidirector.py | 170 +++++++++++ docs/scripts/clirecording/docker/tmux.conf | 21 ++ .../clirecording/generate_recordings.sh | 6 + docs/scripts/clirecording/record.py | 12 + docs/scripts/clirecording/screenplays.py | 288 ++++++++++++++++++ 7 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 docs/scripts/clirecording/Dockerfile create mode 100644 docs/scripts/clirecording/clidirector.py create mode 100644 docs/scripts/clirecording/docker/tmux.conf create mode 100644 docs/scripts/clirecording/generate_recordings.sh create mode 100644 docs/scripts/clirecording/record.py create mode 100644 docs/scripts/clirecording/screenplays.py diff --git a/docs/build.sh b/docs/build.sh index afa73d9c7..05d5141a8 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -8,7 +8,7 @@ set -o nounset SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" pushd ${SCRIPTPATH} -for script in scripts/* ; do +for script in scripts/*.py ; do output="${script##*/}" output="src/generated/${output%.*}.html" echo "Generating output for ${script} into ${output} ..." diff --git a/docs/scripts/clirecording/Dockerfile b/docs/scripts/clirecording/Dockerfile new file mode 100644 index 000000000..5314add50 --- /dev/null +++ b/docs/scripts/clirecording/Dockerfile @@ -0,0 +1,45 @@ +# todo: use a more lightweight base, e.g., Alpine Linux +FROM ubuntu:18.04 + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV TERM screen-256color + +# install mitmproxy, asciinema, and dependencies +RUN apt-get update && apt-get install -y \ + asciinema \ + autoconf \ + automake \ + autotools-dev \ + bison \ + curl \ + git \ + libevent-dev \ + libtool \ + locales \ + m4 \ + make \ + ncurses-dev \ + pkg-config \ + python3-pip \ + python3 \ + wget \ + xterm \ + && locale-gen --purge "en_US.UTF-8" \ + && update-locale "LANG=en_US.UTF-8" \ + && pip3 install libtmux curl requests mitmproxy + +# install latest tmux (to support popups) +RUN git clone https://github.com/tmux/tmux.git \ + && cd tmux \ + && sh autogen.sh \ + && ./configure && make && make install + +WORKDIR /root/clidirector + +COPY ./docker/tmux.conf ../.tmux.conf +COPY clidirector.py screenplays.py record.py ./ + +RUN echo 'PS1="[tutorial@mitmproxy] $ "' >> /root/.bashrc + +ENTRYPOINT [ "./record.py" ] diff --git a/docs/scripts/clirecording/clidirector.py b/docs/scripts/clirecording/clidirector.py new file mode 100644 index 000000000..2dbb0dbe6 --- /dev/null +++ b/docs/scripts/clirecording/clidirector.py @@ -0,0 +1,170 @@ +import datetime +import json +import libtmux +import random +import requests +import subprocess +import threading +import time +import typing + + +class CliDirector: + def __init__(self): + self.record_start = None + self.pause_between_keys = 0.1 + self.pause_between_keys_rand = 0.25 + + def start(self, filename: str, width: int = 0, height: int = 0) -> libtmux.Session: + self.start_session(width, height) + self.start_recording(filename) + return self.tmux_session + + def start_session(self, width: int = 0, height: int = 0) -> libtmux.Session: + self.tmux_server = libtmux.Server() + self.tmux_session = self.tmux_server.new_session(session_name="asciinema_recorder", kill_session=True) + self.tmux_pane = self.tmux_session.attached_window.attached_pane + self.tmux_version = self.tmux_pane.display_message("#{version}", True) + if width and height: + self.resize_window(width, height) + self.pause(3) + return self.tmux_session + + def start_recording(self, filename: str) -> None: + self.asciinema_proc = subprocess.Popen([ + "asciinema", "rec", "-y", "--overwrite", "-c", "tmux attach -t asciinema_recorder", filename]) + self.pause(1.5) + self.record_start = datetime.datetime.now() + + def resize_window(self, width: int, height: int) -> None: + subprocess.Popen(["resize", "-s", str(height), str(width)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def end(self) -> None: + self.end_recording() + self.end_session() + + def end_recording(self) -> None: + self.asciinema_proc.terminate() + self.asciinema_proc.wait(timeout=5) + + def end_session(self) -> None: + self.tmux_session.kill_session() + + def press_key(self, keys: str, count=1, pause: typing.Optional[float] = None, target = None) -> None: + if pause is None: + pause = self.pause_between_keys + if target is None: + target = self.tmux_pane + for i in range(count): + if keys == " ": + keys = "Space" + target.send_keys(cmd=keys, enter=False, suppress_history=False) + self.pause(pause + random.uniform(0, self.pause_between_keys_rand)) + + def type(self, keys: str, pause: typing.Optional[float] = None, target = None) -> None: + if pause is None: + pause = self.pause_between_keys + if target is None: + target = self.tmux_pane + target.select_pane() + for key in keys: + self.press_key(key, pause=pause, target=target) + + def exec(self, keys: str, target = None) -> None: + if target is None: + target = self.tmux_pane + self.type(keys, target=target) + self.pause(1.25) + self.press_key("Enter", target=target) + self.pause(0.5) + + def focus_pane(self, pane: libtmux.Pane, set_active_pane: bool = True) -> None: + pane.select_pane() + if set_active_pane: + self.tmux_pane = pane + + def pause(self, seconds: float) -> None: + time.sleep(seconds) + + def run_external(self, command: str) -> None: + subprocess.run(command, shell=True) + + def message(self, msg: str, duration: typing.Optional[int] = None, add_instruction: bool = True, instruction_html: str = "") -> None: + if duration is None: + duration = len(msg) * 0.1 # seconds + self.tmux_session.set_option("display-time", int(duration * 1000)) # milliseconds + self.tmux_pane.display_message(" " + msg) + + # todo: this is a hack and needs refactoring (instruction() is only defined in MitmCliDirector) + if add_instruction or instruction_html: + if not instruction_html: + instruction_html = msg + self.instruction(instruction=instruction_html, duration=duration) + self.pause(duration + 0.5) + + def popup(self, content: str, duration: int = 4) -> None: + # todo: check if installed tmux version supports display-popup + + # tmux's display-popup is blocking, so we close it in a separate thread + t=threading.Thread(target=self.close_popup, args=[duration]) + t.start() + + lines = content.splitlines() + self.tmux_pane.cmd("display-popup", "", *lines) + t.join() + + def close_popup(self, duration: float = 0) -> None: + self.pause(duration) + self.tmux_pane.cmd("display-popup", "-C") + + @property + def current_time(self) -> float: + now = datetime.datetime.now() + return round((now - self.record_start).total_seconds(), 2) + + @property + def current_pane(self) -> libtmux.Pane: + return self.tmux_pane + + +class InstructionSpec(typing.NamedTuple): + instruction: str + time_from: float + time_from_str: str + time_to: float + + +# todo: merge with CliDirector +class MitmCliDirector(CliDirector): + def __init__(self): + super().__init__() + self.instructions: typing.List[InstructionSpec] = [] + + def instruction(self, instruction: str, duration: float = 3, time_from: typing.Optional[float] = None, correction: float = 0) -> None: + if time_from is None: + time_from = self.current_time + time_from_str = str(datetime.timedelta(seconds = int(time_from + correction)))[2:] + + self.instructions.append(InstructionSpec( + str(len(self.instructions)+1) + ". " + instruction, + time_from=time_from + correction, + time_from_str=time_from_str, + time_to=time_from - correction + duration + )) + + def save_instructions(self, output_path: str) -> None: + instr_as_dicts = [] + for instr in self.instructions: + instr_as_dicts.append(instr._asdict()) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(instr_as_dicts, f, ensure_ascii=False, indent=4) + + def request(self, url: str, threaded: bool = False) -> None: + if threaded: + threading.Thread(target=lambda: requests.get(url, verify=False)).start() + else: + requests.get(url, verify=False) + + def end_recording(self) -> None: + self.instructions = [] + super().end_recording() diff --git a/docs/scripts/clirecording/docker/tmux.conf b/docs/scripts/clirecording/docker/tmux.conf new file mode 100644 index 000000000..c8b9db556 --- /dev/null +++ b/docs/scripts/clirecording/docker/tmux.conf @@ -0,0 +1,21 @@ +set -g default-terminal "screen-256color" + +set-option -g status-position top + +set -g status-style "bg=#000000,fg=#ffffff" +set -g message-style "bg=#3273dc,fg=#ffffff" + +set -g status-justify left +set -g status-left "" +set -g status-right "" + +setw -g window-status-current-format "" + + +# pane options +setw -g pane-base-index 1 +setw -g pane-border-format " Terminal Window #P --------------------------------------------------------------------------------------------------------" +setw -g pane-border-status top +setw -g pane-border-lines simple +setw -g pane-border-style "fg=#cccccc" +setw -g pane-active-border-style "fg=#ffffff" diff --git a/docs/scripts/clirecording/generate_recordings.sh b/docs/scripts/clirecording/generate_recordings.sh new file mode 100644 index 000000000..b352ed136 --- /dev/null +++ b/docs/scripts/clirecording/generate_recordings.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +docker build --pull --rm -t mitmproxy-clirecorder:latest . +docker run -i -t --rm \ + -v "$(pwd)"/../../src/static/recordings:/root/clidirector/recordings \ + mitmproxy-clirecorder:latest diff --git a/docs/scripts/clirecording/record.py b/docs/scripts/clirecording/record.py new file mode 100644 index 000000000..7eeddf73f --- /dev/null +++ b/docs/scripts/clirecording/record.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 + +from clidirector import MitmCliDirector +import screenplays + + +if __name__ == '__main__': + director = MitmCliDirector() + screenplays.record_user_interface(director) + screenplays.record_intercept_requests(director) + screenplays.record_modify_requests(director) + screenplays.record_replay_requests(director) diff --git a/docs/scripts/clirecording/screenplays.py b/docs/scripts/clirecording/screenplays.py new file mode 100644 index 000000000..3c15c45eb --- /dev/null +++ b/docs/scripts/clirecording/screenplays.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 + +from clidirector import MitmCliDirector + + +def record_user_interface(d: MitmCliDirector): + tmux = d.start_session(width=120, height=36) + window = tmux.attached_window + + d.start_recording("recordings/mitmproxy_user_interface.cast") + d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the user interface.") + d.pause(1) + d.exec("mitmproxy") + d.pause(3) + + d.message("This is the default view of mitmproxy.") + d.message("mitmproxy adds rows to the view as new requests come in.") + d.message("Let’s generate some requests using `curl` in a separate terminal.") + + pane_top = d.current_pane + pane_bottom = window.split_window(attach=True) + pane_bottom.resize_pane(height=12) + + d.focus_pane(pane_bottom) + d.pause(2) + + d.type("curl") + d.message("Use curl’s `-x` option to specify a proxy, e.g., `curl -x http://127.0.0.1:8080` to use mitmproxy.") + + d.type(" -x http://127.0.0.1:8080") + + d.message("We use the text-based weather service `wttr.in`.") + d.exec(" \"http://wttr.in/Paris?0\"") + + d.pause(2) + d.press_key("Up") + d.press_key("Left", count=3) + d.press_key("BSpace", count=5) + d.exec("Miami") + + d.pause(2) + d.press_key("Up") + d.press_key("Left", count=3) + d.press_key("BSpace", count=5) + d.exec("Tokio") + + d.pause(2) + d.press_key("Up") + d.press_key("Left", count=3) + d.press_key("BSpace", count=5) + d.exec("London") + + d.pause(2) + d.exec("exit", target=pane_bottom) + + d.focus_pane(pane_top) + + d.message("You see the requests to `wttr.in` in the list of flows.") + + d.message("mitmproxy is controlled using keyboard shortcuts.") + d.message("Use your arrow keys `↑` and `↓` to change the focused flow (`>>`).") + d.press_key("Down", count=3, pause=0.5) + d.press_key("Up", count=2, pause=0.5) + d.press_key("Down", count=2, pause=0.5) + + d.message("The focused flow (`>>`) is used as a target for various commands.") + + d.message("One such command shows the flow details, it is bound to `↵`.") + + d.message("Press `↵` to view the details of the focused flow.") + d.press_key("Enter") + + d.message("The flow details view has 3 panes: request, response, and detail.") + d.message("Use your arrow keys `←` and `→` to switch between panes.") + d.press_key("Right", count=2, pause=2.5) + d.press_key("Left", count=2, pause=1) + + d.message("Press `q` to exit the current view.",) + d.type("q") + + d.message("Press `?` to get a list of all available keyboard shortcuts.") + d.type("?") + d.pause(2) + d.press_key("Down", count=20, pause=0.25) + + d.message("Press `q` to exit the current view.") + d.type("q") + + d.message("Each shortcut is internally bound to a command.") + d.message("You can also execute commands directly (without using shortcuts).") + d.message("Press `:` to open the command prompt at the bottom.") + d.type(":") + + d.message("Enter `console.view.flow @focus`.") + d.type("console.view.flow @focus") + + d.message("The command `console.view.flow` opens the details view for a flow.") + + d.message("The argument `@focus` defines the target flow.") + + d.message("Press `↵` to execute the command.") + d.press_key("Enter") + + d.message("Commands unleash the full power of mitmproxy, i.e., to configure interceptions.") + + d.message("You now know basics of mitmproxy’s UI and how to control it.") + d.pause(1) + d.save_instructions("recordings/mitmproxy_user_interface_instructions.json") + d.end() + + +def record_intercept_requests(d: MitmCliDirector): + tmux = d.start_session(width=120, height=36) + window = tmux.attached_window + + d.start_recording("recordings/mitmproxy_intercept_requests.cast") + d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the interception of requests.") + d.pause(1) + d.exec("mitmproxy") + d.pause(3) + + d.message("We first need to configure mitmproxy to intercept requests.") + + d.message("Press `i` to prepopulate mitmproxy’s command prompt with `set intercept ''`.") + d.type("i") + d.pause(2) + + d.message("We use the flow filter expression `~u ` to only intercept specific URLs.") + d.message("Additionally, we use the filter `~q` to only intercept requests, but not responses.") + d.message("We combine both flow filters using `&`.") + + d.message("Enter `~u /Paris & ~q` between the quotes of the `set intercept` command and press `↵`.") + d.exec("~u /Paris & ~q") + d.message("The bottom bar shows that the interception has been configured.") + + d.message("Let’s generate a request using `curl` in a separate terminal.") + + pane_top = d.current_pane + pane_bottom = window.split_window(attach=True) + pane_bottom.resize_pane(height=12) + + d.focus_pane(pane_bottom) + d.pause(2) + + d.type("curl") + d.message("Use curl’s `-x` option to specify a proxy, e.g., `curl -x http://127.0.0.1:8080` to use mitmproxy.") + + d.type(" -x http://127.0.0.1:8080") + + d.message("We use the text-based weather service `wttr.in`.") + d.exec(" \"http://wttr.in/Paris?0\"") + d.pause(2) + + d.focus_pane(pane_top) + + d.message("You will see a new line in in the list of flows.") + d.message("The new flow is displayed in red to indicate that it has been intercepted.") + d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.") + d.message("Press `a` to resume this flow without making any changes.") + d.type("a") + + d.focus_pane(pane_bottom) + + d.message("Submit another request and focus its flow.") + d.press_key("Up") + d.press_key("Enter") + d.pause(2) + + d.focus_pane(pane_top) + d.press_key("Down") + + d.message("Press `X` to kill this flow, i.e., discard it without forwarding it to its final destination `wttr.in`.") + d.type("X") + d.pause(3) + d.save_instructions("recordings/mitmproxy_intercept_requests_instructions.json") + d.end() + + +def record_modify_requests(d: MitmCliDirector): + tmux = d.start_session(width=120, height=36) + window = tmux.attached_window + + d.start_recording("recordings/mitmproxy_modify_requests.cast") + d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the modification of intercepted requests.") + d.pause(1) + d.exec("mitmproxy") + d.pause(3) + + d.message("We configure and use the same interception rule as in the last tutorial.") + d.message("Press `i` to prepopulate mitmproxy’s command prompt, enter the flow filter `~u /Paris & ~q`, and press `↵`.") + d.type("i") + d.pause(2) + d.exec("~u /Paris & ~q") + + d.message("Let’s generate a request using `curl` in a separate terminal.") + + pane_top = d.current_pane + pane_bottom = window.split_window(attach=True) + pane_bottom.resize_pane(height=12) + + d.focus_pane(pane_bottom) + d.pause(2) + + d.type("curl -x http://127.0.0.1:8080") + d.exec(" \"http://wttr.in/Paris?0\"") + d.pause(2) + + d.focus_pane(pane_top) + + d.message("We now want to modify the intercepted request.") + d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.") + + d.message("Press `↵` to open the details view for the intercepted flow.") + d.press_key("Enter") + + d.message("Press `e` to edit the intercepted flow.") + d.type("e") + + d.message("mitmproxy asks which part to modify.") + + d.message("Select `path` by using your arrow keys and press `↵`.") + d.press_key("Down", count=3, pause=0.5) + d.pause(1) + d.press_key("Enter") + + d.message("mitmproxy shows all path components line by line, in our example its just one element: `Paris`.") + d.message("Press `↵` to modify the selected path component.") + d.press_key("Down", pause=2) + d.press_key("Enter") + + d.message("Replace `Paris` with `Tokio`.") + d.press_key("BSpace", count=5, pause=0.5) + d.type("Tokio", pause=0.5) + + d.message("Press `ESC` to confirm your change.") + d.press_key("Escape") + + d.message("Press `q` to go back to the flow details view.") + d.type("q") + + d.message("Press `a` to resume the intercepted flow.") + d.type("a") + d.pause(2) + + d.message("You see that the request URL was modified and `wttr.in` replied with the weather report for `Tokio`.") + + d.save_instructions("recordings/mitmproxy_modify_requests_instructions.json") + d.end() + + +def record_replay_requests(d: MitmCliDirector): + tmux = d.start_session(width=120, height=36) + window = tmux.attached_window + + d.start_recording("recordings/mitmproxy_replay_requests.cast") + d.message("Welcome to the mitmproxy tutorial. In this lesson we cover replaying requests.") + d.pause(1) + d.exec("mitmproxy") + d.pause(3) + + d.message("Let’s generate a request that we can replay. We use `curl` in a separate terminal.") + + pane_top = d.current_pane + pane_bottom = window.split_window(attach=True) + pane_bottom.resize_pane(height=12) + + d.focus_pane(pane_bottom) + d.pause(2) + + d.exec("curl -x http://127.0.0.1:8080 \"http://wttr.in/Paris?0\"") + d.pause(2) + + d.focus_pane(pane_top) + + d.message("We now want to replay the intercepted request.") + d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.") + d.message("Press `r` to replay this flow.") + d.type("r") + + d.message("Note that no new rows are added for replayed flows, but the existing row is updated.") + d.message("Every time you press `r`, mitmproxy sends this request to the server again and updates the flow.") + d.press_key("r", count=6, pause=0.75) + + d.message("You can also modify a flow before replaying it.") + d.message("It works as shown in the previous tutorial by pressing `e`.") + + d.save_instructions("recordings/mitmproxy_replay_requests_instructions.json") + d.end()