diff --git a/.travis.yml b/.travis.yml index 9fd4fbd94..cdc578519 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,10 @@ +sudo: false language: python -sudo: false - -python: - - "2.7" - - pypy - matrix: + fast_finish: true include: + - python: 2.7 - python: 2.7 env: OPENSSL=1.0.2 addons: @@ -18,6 +15,7 @@ matrix: - debian-sid packages: - libssl-dev + - python: pypy - python: pypy env: OPENSSL=1.0.2 addons: @@ -28,6 +26,10 @@ matrix: - debian-sid packages: - libssl-dev + allow_failures: + # We allow pypy to fail until Travis fixes their infrastructure to a pypy + # with a recent enought CFFI library to run cryptography 1.0+. + - python: pypy install: - "pip install --src . -r requirements.txt" @@ -50,7 +52,7 @@ notifications: slack: rooms: - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu - on_success: :change + on_success: change on_failure: always # exclude cryptography from cache @@ -58,13 +60,10 @@ notifications: # which needs to be compiled specifically to each version before_cache: - pip uninstall -y cryptography - - rm -rf /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages/cryptography/ - - rm -rf /home/travis/virtualenv/pypy-2.5.0/site-packages/cryptography/ - - rm /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages/pip/_vendor/requests/packages/urllib3/contrib/pyopenssl.py - - rm /home/travis/virtualenv/pypy-2.5.0/site-packages/pip/_vendor/requests/packages/urllib3/contrib/pyopenssl.py cache: directories: + - $HOME/.cache/pip - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages - /home/travis/virtualenv/python2.7.9/bin - /home/travis/virtualenv/pypy-2.5.0/site-packages diff --git a/README.mkd b/README.mkd index d686e9334..9b4944e32 100644 --- a/README.mkd +++ b/README.mkd @@ -20,8 +20,7 @@ mitmproxy.org website: Installation Instructions are available at [mitmproxy.org/doc/install.html](http://mitmproxy.org/doc/install.html). You can join our developer chat on Slack: -[![Slack](https://mitmproxy-slack.herokuapp.com/badge.svg)](https://mitmproxy-slack.herokuapp.com/) - +[![Slack](http://slack.mitmproxy.org/badge.svg)](http://slack.mitmproxy.org/) Features @@ -53,11 +52,11 @@ installation instructions for virtualenv following: ``` -$ git clone https://github.com/mitmproxy/mitmproxy.git -$ git clone https://github.com/mitmproxy/netlib.git -$ git clone https://github.com/mitmproxy/pathod.git -$ cd mitmproxy -$ source ./dev +git clone https://github.com/mitmproxy/mitmproxy.git +git clone https://github.com/mitmproxy/netlib.git +git clone https://github.com/mitmproxy/pathod.git +cd mitmproxy +source ./dev ``` The *dev* script will create a virtualenv environment in a directory called @@ -70,7 +69,7 @@ To confirm that you're up and running, activate the virtualenv, and run the mitmproxy test suite: ```shell -$ source ../venv.mitmproxy/bin/activate # ..\venv.mitmproxy\Scripts\activate.bat on Windows +$ . ../venv.mitmproxy/bin/activate # ..\venv.mitmproxy\Scripts\activate.bat on Windows $ nosetests ./test ``` Note that the main executables for the project - **mitmdump**, **mitmproxy** and diff --git a/dev b/dev index 47a66cc4d..248a3d894 100755 --- a/dev +++ b/dev @@ -3,7 +3,7 @@ set -e VENV=../venv.mitmproxy python -m virtualenv $VENV --always-copy || { echo 'virtualenv is not installed. Exiting.' ; exit 1; } -source $VENV/bin/activate +. $VENV/bin/activate pip install --src .. -r requirements.txt echo "" diff --git a/doc-src/certinstall.html b/doc-src/certinstall.html index fb3cfd42e..594049d99 100644 --- a/doc-src/certinstall.html +++ b/doc-src/certinstall.html @@ -31,8 +31,8 @@ you're on, and you are good to go. ## Installing the mitmproxy CA certificate manually -Sometimes using the quick install app is not an option - Java or the IOS -similator spring to mind - or you just need to do it manually for some other +Sometimes using the quick install app is not an option - Java or the iOS +Simulator spring to mind - or you just need to do it manually for some other reason. Below is a list of pointers to manual certificate installation documentation for some common platforms: diff --git a/examples/read_dumpfile b/examples/read_dumpfile old mode 100644 new mode 100755 index 2238d2d28..eb1c93bbf --- a/examples/read_dumpfile +++ b/examples/read_dumpfile @@ -5,15 +5,17 @@ from libmproxy import flow import json +import pprint import sys -with open("logfile", "rb") as logfile: +with open(sys.argv[1], "rb") as logfile: freader = flow.FlowReader(logfile) + pp = pprint.PrettyPrinter(indent=4) try: for f in freader.stream(): print(f) print(f.request.host) - json.dump(f.get_state(), sys.stdout, indent=4) + pp.pprint(f.get_state()) print("") except flow.FlowReadError as v: print "Flow file corrupted. Stopped loading." diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 55377af26..7f6f69ef0 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -650,6 +650,11 @@ def mitmproxy(): action="store_true", dest="eventlog", help="Show event log." ) + parser.add_argument( + "--no-mouse", + action="store_true", dest="no_mouse", + help="Disable mouse interaction." + ) group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 3d20947b9..2133f97f8 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -154,7 +154,8 @@ class Options(object): "wfile", "nopop", "palette", - "palette_transparent" + "palette_transparent", + "no_mouse" ] def __init__(self, **kwargs): @@ -455,12 +456,12 @@ class ConsoleMaster(flow.FlowMaster): def run(self): self.ui = urwid.raw_display.Screen() - self.ui.set_mouse_tracking() self.ui.set_terminal_properties(256) self.set_palette(self.palette) self.loop = urwid.MainLoop( urwid.SolidFill("x"), screen = self.ui, + handle_mouse = not self.options.no_mouse, ) self.server.start_slave( diff --git a/libmproxy/main.py b/libmproxy/main.py index faef8c82d..23cb487c6 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -55,6 +55,7 @@ def mitmproxy(args=None): # pragma: nocover console_options.eventlog = options.eventlog console_options.intercept = options.intercept console_options.limit = options.limit + console_options.no_mouse = options.no_mouse server = get_server(console_options.no_server, proxy_config) diff --git a/release/build.py b/release/build.py deleted file mode 100755 index 7d0d52dc6..000000000 --- a/release/build.py +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env python - -from __future__ import (absolute_import, print_function, division) -from contextlib import contextmanager -from os.path import dirname, realpath, join, exists, normpath -import os -import shutil -import subprocess -import glob -import re -from shlex import split -import click - -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if os.name == "nt": - venv_bin = "Scripts" -else: - venv_bin = "bin" - -root_dir = join(dirname(realpath(__file__)), "..", "..") -mitmproxy_dir = join(root_dir, "mitmproxy") -dist_dir = join(mitmproxy_dir, "dist") -test_venv_dir = join(root_dir, "venv.mitmproxy-release") - -all_projects = ("netlib", "pathod", "mitmproxy") -tools = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathod", "pathoc"], - "netlib": [] -} -if os.name == "nt": - tools["mitmproxy"].remove("mitmproxy") -version_files = { - "mitmproxy": normpath(join(root_dir, "mitmproxy/libmproxy/version.py")), - "pathod": normpath(join(root_dir, "pathod/libpathod/version.py")), - "netlib": normpath(join(root_dir, "netlib/netlib/version.py")), -} - - -@contextmanager -def empty_pythonpath(): - """ - Make sure that the regular python installation is not on the python path, - which would give us access to modules installed outside of our virtualenv. - """ - pythonpath = os.environ["PYTHONPATH"] - os.environ["PYTHONPATH"] = "" - yield - os.environ["PYTHONPATH"] = pythonpath - - -@contextmanager -def chdir(path): - old_dir = os.getcwd() - os.chdir(path) - yield - os.chdir(old_dir) - - -@click.group(chain=True) -def cli(): - """ - mitmproxy build tool - """ - pass - - -@cli.command("contributors") -def contributors(): - """ - Update CONTRIBUTORS.md - """ - print("Updating CONTRIBUTORS.md...") - contributors_data = subprocess.check_output(split("git shortlog -n -s")) - with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f: - f.write(contributors_data) - - -@cli.command("docs") -def docs(): - """ - Render the docs - """ - print("Rendering the docs...") - subprocess.check_call([ - "cshape", - join(mitmproxy_dir, "doc-src"), - join(mitmproxy_dir, "doc") - ]) - - -@cli.command("set-version") -@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) -@click.argument('version') -def set_version(projects, version): - """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - for project, version_file in version_files.items(): - if project not in projects: - continue - print("Update %s..." % version_file) - with open(version_file, "rb") as f: - content = f.read() - new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content) - with open(version_file, "wb") as f: - f.write(new_content) - - -@cli.command("git") -@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) -@click.argument('args', nargs=-1, required=True) -def git(projects, args): - """ - Run a git command on every project - """ - args = ["git"] + list(args) - for project in projects: - print("%s> %s..." % (project, " ".join(args))) - subprocess.check_call( - args, - cwd=join(root_dir, project) - ) - - -@cli.command("sdist") -@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) -def sdist(projects): - """ - Build a source distribution - """ - with empty_pythonpath(): - print("Building release...") - if exists(dist_dir): - shutil.rmtree(dist_dir) - for project in projects: - print("Creating %s source distribution..." % project) - subprocess.check_call( - ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"], - cwd=join(root_dir, project) - ) - - -@cli.command("test") -@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) -@click.pass_context -def test(ctx, projects): - """ - Test the source distribution - """ - if not exists(dist_dir): - ctx.invoke(sdist) - - with empty_pythonpath(): - print("Creating virtualenv for test install...") - if exists(test_venv_dir): - shutil.rmtree(test_venv_dir) - subprocess.check_call(["virtualenv", "-q", test_venv_dir]) - - pip = join(test_venv_dir, venv_bin, "pip") - with chdir(dist_dir): - for project in projects: - print("Installing %s..." % project) - dist = glob.glob("./%s*" % project)[0] - subprocess.check_call([pip, "install", "-q", dist]) - - print("Running binaries...") - for project in projects: - for tool in tools[project]: - tool = join(test_venv_dir, venv_bin, tool) - print(tool) - print(subprocess.check_output([tool, "--version"])) - - print("Virtualenv available for further testing:") - print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate"))) - - -@cli.command("upload") -@click.option('--username', prompt=True) -@click.password_option(confirmation_prompt=False) -@click.option('--repository', default="pypi") -def upload_release(username, password, repository): - """ - Upload source distributions to PyPI - """ - print("Uploading distributions...") - subprocess.check_call([ - "twine", - "upload", - "-u", username, - "-p", password, - "-r", repository, - "%s/*" % dist_dir - ]) - - -# TODO: Fully automate build process. -# This wizard is missing OSX builds and updating mitmproxy.org. -@cli.command("wizard") -@click.option('--version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) -@click.pass_context -def wizard(ctx, version, username, password, repository, projects): - """ - Interactive Release Wizard - """ - for project in projects: - if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)): - raise RuntimeError("%s repository is not clean." % project) - - # Build test release - ctx.invoke(sdist, projects=projects) - ctx.invoke(test, projects=projects) - click.confirm("Please test the release now. Is it ok?", abort=True) - - # bump version, update docs and contributors - ctx.invoke(set_version, version=version, projects=projects) - ctx.invoke(docs) - ctx.invoke(contributors) - - # version bump commit + tag - ctx.invoke(git, args=["commit", "-a", "-m", "bump version"], projects=projects) - ctx.invoke(git, args=["tag", "v" + version], projects=projects) - ctx.invoke(git, args=["push"], projects=projects) - ctx.invoke(git, args=["push", "--tags"], projects=projects) - - # Re-invoke sdist with bumped version - ctx.invoke(sdist, projects=projects) - click.confirm("All good, can upload to PyPI?", abort=True) - ctx.invoke(upload_release, username=username, password=password, repository=repository) - click.echo("All done!") - - -if __name__ == "__main__": - cli() diff --git a/release/mitmdump.spec b/release/mitmdump.spec index f9b0311c7..5ea6d0ce2 100644 --- a/release/mitmdump.spec +++ b/release/mitmdump.spec @@ -2,23 +2,28 @@ from glob import glob -a = Analysis(['./mitmdump'], +VENV = "../release/venv" + +a = Analysis(['../mitmproxy/mitmdump'], hiddenimports=[], hookspath=None, runtime_hooks=None, excludes=None, ) a.datas = Tree( - "./libmproxy/onboarding/templates", + "../mitmproxy/libmproxy/onboarding/templates", prefix="libmproxy/onboarding/templates" ) a.datas += Tree( - "./libmproxy/onboarding/static", + "../mitmproxy/libmproxy/onboarding/static", prefix="libmproxy/onboarding/static" ) a.datas += Tree( - "../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src", - prefix = "cryptography/hazmat/bindings/openssl/src" + os.path.join( + VENV, + "lib/python2.7/site-packages/certifi", + ), + prefix = "certifi" ) pyz = PYZ(a.pure) exe = EXE(pyz, diff --git a/release/mitmproxy.spec b/release/mitmproxy.spec index 9aab5ef46..a5a75d756 100644 --- a/release/mitmproxy.spec +++ b/release/mitmproxy.spec @@ -1,24 +1,29 @@ # -*- mode: python -*- - +import os.path from glob import glob -a = Analysis(['./mitmproxy'], +VENV = "../release/venv" + +a = Analysis(['../mitmproxy/mitmproxy'], hiddenimports=[], hookspath=None, runtime_hooks=None, excludes=None, ) a.datas = Tree( - "./libmproxy/onboarding/templates", + "../mitmproxy/libmproxy/onboarding/templates", prefix="libmproxy/onboarding/templates" ) a.datas += Tree( - "./libmproxy/onboarding/static", + "../mitmproxy/libmproxy/onboarding/static", prefix="libmproxy/onboarding/static" ) a.datas += Tree( - "../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src", - prefix = "cryptography/hazmat/bindings/openssl/src" + os.path.join( + VENV, + "lib/python2.7/site-packages/certifi", + ), + prefix = "certifi" ) pyz = PYZ(a.pure) exe = EXE(pyz, diff --git a/release/mitmweb.spec b/release/mitmweb.spec.disabled similarity index 65% rename from release/mitmweb.spec rename to release/mitmweb.spec.disabled index 565482e66..21ae28900 100644 --- a/release/mitmweb.spec +++ b/release/mitmweb.spec.disabled @@ -2,31 +2,36 @@ from glob import glob -a = Analysis(['./mitmweb'], +VENV = "../release/venv" + +a = Analysis(['../mitmproxy/mitmweb'], hiddenimports=[], hookspath=None, runtime_hooks=None, excludes=None, ) a.datas = Tree( - "./libmproxy/onboarding/templates", + "../mitmproxy/libmproxy/onboarding/templates", prefix="libmproxy/onboarding/templates" ) a.datas += Tree( - "./libmproxy/onboarding/static", + "../mitmproxy/libmproxy/onboarding/static", prefix="libmproxy/onboarding/static" ) a.datas += Tree( - "./libmproxy/web/templates", + "../mitmproxy/libmproxy/web/templates", prefix="libmproxy/web/templates" ) a.datas += Tree( - "./libmproxy/web/static", + "../mitmproxy/libmproxy/web/static", prefix="libmproxy/web/static" ) a.datas += Tree( - "../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src", - prefix = "cryptography/hazmat/bindings/openssl/src" + os.path.join( + VENV, + "lib/python2.7/site-packages/certifi", + ), + prefix = "certifi" ) pyz = PYZ(a.pure) exe = EXE(pyz, diff --git a/release/osx-binaries b/release/osx-binaries deleted file mode 100755 index 3cfe0913b..000000000 --- a/release/osx-binaries +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -# Quick and dangerous script for building OSX binaries. - -# At the moment, pyinstaller has no support for entry points, except for this -# hideous hack on the wiki: -# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point -# Once this is fixed, we can ditch the redundant command scripts. - -VENV=../venv.mitmproxy -PYINST_CMD="$VENV/bin/pyinstaller -F --clean" -TMPDIR=./tmp -CACHE="~/Library/Application Support/pyinstaller" - - -if [ ! -d $VENV ] -then - echo "Failed: set up a dev environment as described in the README" - echo "and run from the top-level mitmproxy directory." - exit -fi - -source $VENV/bin/activate - -if [ ! -f $VENV/bin/pyinstaller ] - then - echo "Installing pyinstaller..." - $VENV/bin/pip install \ - --force-reinstall \ - --upgrade \ - https://github.com/pyinstaller/pyinstaller/archive/develop.zip - $VENV/bin/pip install --upgrade macholib -fi - -# readline.so is actually a symlink to a Python file, which breaks PyInstaller -# (and readline itself). Why? Who knows. Re-address this when this stupidity -# ceases to be. -echo "Removing broken readline..." -rm -f $VENV/lib/python2.7/readline.so - - -echo "Clearing caches..." -rm -f dist/* -rm -rf $TMPDIR -rm -rf $CACHE - -$PYINST_CMD ./release/mitmdump.spec -echo "Running mitmdump..." -./dist/mitmdump --version || exit 1 - -$PYINST_CMD ./release/mitmproxy.spec -echo "Running mitmproxy..." -./dist/mitmproxy --version || exit 1 - -$PYINST_CMD ./release/mitmweb.spec -echo "Running mitmweb..." -./dist/mitmweb --version || exit 1 - -DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` -mkdir -p $TMPDIR/$DST -cp ./dist/mitmproxy $TMPDIR/$DST -cp ./dist/mitmdump $TMPDIR/$DST -cshape ./doc-src $TMPDIR/$DST/doc - -cd $TMPDIR -tar -czvf $DST.tar.gz $DST diff --git a/release/release-checklist.md b/release/release-checklist.md deleted file mode 100644 index e6d9ae1f4..000000000 --- a/release/release-checklist.md +++ /dev/null @@ -1,55 +0,0 @@ -# Release Checklist - -## Test - - - Create the source distributions, make sure the output is sensible: - `./release/build.py release` - All source distributions can be found in `./dist`. - - - Test the source distributions: - `./release/build.py test` - This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. - -## Release - - - Verify that repositories are in a clean state: - `./release/build.py git status` - - - Update the version number in `version.py` for all projects: - `./release/build.py set-version 0.13` - - - Ensure that the website style assets have been compiled for production, and synced to the docs. - - - Render the docs, update CONTRIBUTORS file: - `./release/build.py docs contributors` - - - Make version bump commit for all projects, tag and push it: - `./release/build.py git commit -am "bump version"` - `./release/build.py git tag v0.13` - `./release/build.py git push --tags` - - - Recreate the source distributions with updated version information: - `./release/build.py sdist` - - - Build the OSX binaries - - Follow instructions in osx-binaries - - Move to download dir: - `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` - - - Move all source distributions from `./dist` to the server: - `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` - - - Upload distributions in `./dist` to PyPI: - `./release/build.py upload` - You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. - ([more info](https://tom-christie.github.io/articles/pypi/)) - - - Now bump the version number to be ready for the next cycle: - - **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? - We should probably just leave it as-is and only bump once we actually do the next release. - - Also, we need a release policy. I propose the following: - - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. - - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. - This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` \ No newline at end of file diff --git a/release/test-release b/release/test-release deleted file mode 100755 index 8cbcea8c5..000000000 --- a/release/test-release +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -MITMPROXY_DIR=~/mitmproxy/mitmproxy -NETLIB_DIR=~/mitmproxy/netlib -PATHOD_DIR=~/mitmproxy/pathod -DST=/tmp/mitmproxy_release - -rm -rf $DST -mkdir -p $DST - -cd $NETLIB_DIR -echo "Creating netlib source distribution..." -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating mitmproxy source distribution..." -cd $MITMPROXY_DIR -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating pathod source distribution..." -cd $PATHOD_DIR -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating virtualenv for test install..." -virtualenv -q $DST/venv - -cd $DST -echo "Installing netlib..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./netlib* -echo "Installing pathod..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./pathod* -echo "Installing mitmproxy..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./mitmproxy* - -echo "Running binaries..." -./venv/bin/mitmproxy --version -./venv/bin/mitmdump --version -./venv/bin/pathod --version -./venv/bin/pathoc --version diff --git a/setup.py b/setup.py index 876b4f9cf..e28033ad1 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,6 @@ dev_deps = { "nose>=1.3.0", "nose-cov>=1.6", "coveralls>=0.4.1", - "click>=4.1", - "twine>=1.5.0", "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), "countershape" }