From f37bd89433af80b8d99dc8c5dce3f5aec34e476d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 18:47:42 +0100 Subject: [PATCH 01/10] update pyinstaller :pray: --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 0859ddae1..e53572b41 100644 --- a/tox.ini +++ b/tox.ini @@ -57,8 +57,7 @@ deps = # The 3.2 release is broken # the next commit after this updates the bootloaders, which then segfault! # https://github.com/pyinstaller/pyinstaller/issues/2232 - git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c; sys_platform == 'win32' - git+https://github.com/mhils/pyinstaller.git@d094401e4196b1a6a03818b80164a5f555861cef; sys_platform != 'win32' + pyinstaller==3.3.1 commands = rtool {posargs} From 127cfa689a8a8edcb30087d2f31aece96ec6e82e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 19:21:43 +0100 Subject: [PATCH 02/10] make sure that mitmproxy/rtool dependencies don't clash --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e53572b41..681e95f17 100644 --- a/tox.ini +++ b/tox.ini @@ -60,4 +60,5 @@ deps = pyinstaller==3.3.1 commands = + mitmdump --version rtool {posargs} From f6a308d31adbbeb16fa3378a59589afef7a4ee90 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 19:27:21 +0100 Subject: [PATCH 03/10] update rtool dependencies --- release/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/setup.py b/release/setup.py index 0c4e6605a..531faaaf0 100644 --- a/release/setup.py +++ b/release/setup.py @@ -8,7 +8,7 @@ setup( "click>=6.2, <7.0", "twine>=1.6.5, <1.10", "pysftp==0.2.8", - "cryptography>=2.0.0, <2.1", + "cryptography>=2.1.4, <2.2", ], entry_points={ "console_scripts": [ From 7075f4dfec749258f6f23c90f0dc2d1e3a11d0f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 19:42:47 +0100 Subject: [PATCH 04/10] remove rtool's setup.py, embrace tox --- release/setup.py | 18 ------------------ tox.ini | 8 +++----- 2 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 release/setup.py diff --git a/release/setup.py b/release/setup.py deleted file mode 100644 index 531faaaf0..000000000 --- a/release/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -from setuptools import setup - -setup( - name='mitmproxy-rtool', - version="1.0", - py_modules=["rtool"], - install_requires=[ - "click>=6.2, <7.0", - "twine>=1.6.5, <1.10", - "pysftp==0.2.8", - "cryptography>=2.1.4, <2.2", - ], - entry_points={ - "console_scripts": [ - "rtool=rtool:cli", - ], - }, -) diff --git a/tox.ini b/tox.ini index 681e95f17..f1e06a6c1 100644 --- a/tox.ini +++ b/tox.ini @@ -53,12 +53,10 @@ commands = [testenv:rtool] deps = -rrequirements.txt - -e./release - # The 3.2 release is broken - # the next commit after this updates the bootloaders, which then segfault! - # https://github.com/pyinstaller/pyinstaller/issues/2232 pyinstaller==3.3.1 + twine==1.9.1 + pysftp==0.2.8 commands = mitmdump --version - rtool {posargs} + python ./release/rtool.py {posargs} From 749099e2922a2852a4628978b564ade4927fafd3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 19:46:22 +0100 Subject: [PATCH 05/10] tox.ini: python3 -> python "python3" points to a different binary... on Windows, at least. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f1e06a6c1..5c2b25ae8 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html commands = mitmdump --version flake8 --jobs 8 mitmproxy pathod examples test release - python3 test/filename_matching.py + python test/filename_matching.py rstcheck README.rst mypy --ignore-missing-imports ./mitmproxy mypy --ignore-missing-imports ./pathod @@ -35,7 +35,7 @@ commands = deps = -rrequirements.txt commands = - python3 test/individual_coverage.py + python test/individual_coverage.py [testenv:wheel] recreate = True From 937a849c9363b1fb745b0948310533b535348819 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 26 Dec 2017 21:53:16 +0100 Subject: [PATCH 06/10] rtool: include dev version in binaries --- .appveyor.yml | 2 +- mitmproxy/version.py | 3 +- release/rtool.py | 110 +++++++++++++++++++++++++++++-------------- tox.ini | 1 + 4 files changed, 78 insertions(+), 38 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 160cdf736..7ef9d8b84 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -36,7 +36,7 @@ test_script: - ps: | $Env:VERSION = $(python mitmproxy/version.py) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" - tox -e wheel + tox -e rtool -- wheel tox -e rtool -- bdist - ps: | diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 3cae2a048..c901cb67f 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -1,5 +1,4 @@ -IVERSION = (3, 0, 0) -VERSION = ".".join(str(i) for i in IVERSION) +VERSION = "3.0.0" PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION diff --git a/release/rtool.py b/release/rtool.py index 271392baa..14a0a0784 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -4,6 +4,7 @@ import contextlib import fnmatch import os import platform +import re import runpy import shlex import shutil @@ -79,26 +80,38 @@ def git(args: str) -> str: return subprocess.check_output(["git"] + shlex.split(args)).decode() -def get_version() -> str: - return runpy.run_path(VERSION_FILE)["VERSION"] +def get_version(dev: bool = False, build: bool = False) -> str: + version = runpy.run_path(VERSION_FILE)["VERSION"] + version = re.sub(r"\.dev.+?$", "", version) # replace dev suffix if present. - -def get_snapshot_version() -> str: last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) + commit = commit.lstrip("g")[:7] tag_dist = int(tag_dist) - if tag_dist == 0: - return get_version() - else: - # remove the 'g' prefix added by recent git versions - if commit.startswith('g'): - commit = commit[1:] + if tag_dist > 0 and dev: + dev_tag = ".dev{tag_dist:04}".format(tag_dist=tag_dist) + else: + dev_tag = "" + + if tag_dist > 0 and build: # The wheel build tag (we use the commit) must start with a digit, so we include "0x" - return "{version}dev{tag_dist:04}-0x{commit}".format( - version=get_version(), # this should already be the next version - tag_dist=tag_dist, - commit=commit - ) + build_tag = "-0x{commit}".format(commit=commit) + else: + build_tag = "" + + return version + dev_tag + build_tag + + +def set_version(dev: bool) -> None: + """ + Update version information in mitmproxy's version.py to either include the dev version or not. + """ + v = get_version(dev) + with open(VERSION_FILE) as f: + content = f.read() + content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(v), content) + with open(VERSION_FILE, "w") as f: + f.write(content) def archive_name(bdist: str) -> str: @@ -116,7 +129,7 @@ def archive_name(bdist: str) -> str: def wheel_name() -> str: return "mitmproxy-{version}-py3-none-any.whl".format( - version=get_version(), + version=get_version(True), ) @@ -179,6 +192,23 @@ def contributors(): f.write(contributors_data.encode()) +@cli.command("wheel") +def make_wheel(): + """ + Build a Python wheel + """ + set_version(True) + try: + subprocess.check_call([ + "tox", "-e", "wheel", + ], env={ + **os.environ, + "VERSION": get_version(True), + }) + finally: + set_version(False) + + @cli.command("bdist") def make_bdist(): """ @@ -206,24 +236,30 @@ def make_bdist(): excludes.append("mitmproxy.tools.web") if tool != "mitmproxy_main": excludes.append("mitmproxy.tools.console") - subprocess.check_call( - [ - "pyinstaller", - "--clean", - "--workpath", PYINSTALLER_TEMP, - "--distpath", PYINSTALLER_DIST, - "--additional-hooks-dir", PYINSTALLER_HOOKS, - "--onefile", - "--console", - "--icon", "icon.ico", - # This is PyInstaller, so setting a - # different log level obviously breaks it :-) - # "--log-level", "WARN", - ] - + [x for e in excludes for x in ["--exclude-module", e]] - + PYINSTALLER_ARGS - + [tool] - ) + + # Overwrite mitmproxy/version.py to include commit info + set_version(True) + try: + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + finally: + set_version(False) # Delete the spec file - we're good without. os.remove("{}.spec".format(tool)) @@ -299,7 +335,11 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, for f in files: local_path = join(DIST_DIR, f) - remote_filename = f.replace(get_version(), get_snapshot_version()) + remote_filename = re.sub( + r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), + get_version(True, True), + f + ) symlink_path = "../{}".format(f.replace(get_version(), "latest")) # Upload new version diff --git a/tox.ini b/tox.ini index 5c2b25ae8..02d9a57bf 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,7 @@ commands = pathoc --version [testenv:rtool] +passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY deps = -rrequirements.txt pyinstaller==3.3.1 From 9dc3d16bf2554ca1fb6d191c232d096fe0774110 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 27 Dec 2017 21:46:52 +0100 Subject: [PATCH 07/10] single-source version processing --- mitmproxy/utils/debug.py | 41 ++++++----------------- mitmproxy/version.py | 54 ++++++++++++++++++++++++++++++ release/rtool.py | 29 ++++------------ setup.py | 2 +- test/mitmproxy/test_version.py | 23 +++++++++++++ test/mitmproxy/utils/test_debug.py | 13 ------- 6 files changed, 95 insertions(+), 67 deletions(-) diff --git a/mitmproxy/utils/debug.py b/mitmproxy/utils/debug.py index de01b12cd..e8eca9063 100644 --- a/mitmproxy/utils/debug.py +++ b/mitmproxy/utils/debug.py @@ -1,43 +1,24 @@ import gc import os +import platform +import re +import signal import sys import threading -import signal -import platform import traceback -import subprocess - -from mitmproxy import version from OpenSSL import SSL +from mitmproxy import version + def dump_system_info(): - mitmproxy_version = version.VERSION - here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - try: - git_describe = subprocess.check_output( - ['git', 'describe', '--tags', '--long'], - stderr=subprocess.STDOUT, - cwd=here, - ) - except: - pass - else: - last_tag, tag_dist, commit = git_describe.decode().strip().rsplit("-", 2) - - commit = commit.lstrip("g") # remove the 'g' prefix added by recent git versions - tag_dist = int(tag_dist) - - if tag_dist > 0: - tag_dist = "dev{:04}".format(tag_dist) - else: - tag_dist = "" - - mitmproxy_version += "{tag_dist} ({commit})".format( - tag_dist=tag_dist, - commit=commit, - ) + mitmproxy_version = version.get_version(True, True) + mitmproxy_version = re.sub( + r"-0x([0-9a-f]+)", + r" (commit \1)", + mitmproxy_version + ) # PyInstaller builds indicator, if using precompiled binary if getattr(sys, 'frozen', False): diff --git a/mitmproxy/version.py b/mitmproxy/version.py index c901cb67f..3073c3d3f 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -1,3 +1,8 @@ +import os +import subprocess + +# The actual version string. For precompiled binaries, this will be changed to include the build +# tag, e.g. "3.0.0.dev0042-0xcafeabc" VERSION = "3.0.0" PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION @@ -6,5 +11,54 @@ MITMPROXY = "mitmproxy " + VERSION # for each change in the file format. FLOW_FORMAT_VERSION = 5 + +def get_version(dev: bool = False, build: bool = False, refresh: bool = False) -> str: + """ + Return a detailed version string, sourced either from a hardcoded VERSION constant + or obtained dynamically using git. + + Args: + dev: If True, non-tagged releases will include a ".devXXXX" suffix, where XXXX is the number + of commits since the last tagged release. + build: If True, non-tagged releases will include a "-0xXXXXXXX" suffix, where XXXXXXX are + the first seven digits of the commit hash. + refresh: If True, always try to use git instead of a potentially hardcoded constant. + """ + + mitmproxy_version = VERSION + + if "dev" in VERSION and not refresh: + pass # There is a hardcoded build tag, so we just use what's there. + elif dev or build: + here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + try: + git_describe = subprocess.check_output( + ['git', 'describe', '--tags', '--long'], + stderr=subprocess.STDOUT, + cwd=here, + ) + last_tag, tag_dist, commit = git_describe.decode().strip().rsplit("-", 2) + commit = commit.lstrip("g")[:7] + tag_dist = int(tag_dist) + except Exception: + pass + else: + # Remove current suffix + mitmproxy_version = mitmproxy_version.split(".dev")[0] + + # Add suffix for non-tagged releases + if tag_dist > 0: + mitmproxy_version += ".dev{tag_dist:04}".format(tag_dist=tag_dist) + # The wheel build tag (we use the commit) must start with a digit, so we include "0x" + mitmproxy_version += "-0x{commit}".format(commit=commit) + + if not dev: + mitmproxy_version = mitmproxy_version.split(".dev")[0] + elif not build: + mitmproxy_version = mitmproxy_version.split("-0x")[0] + + return mitmproxy_version + + if __name__ == "__main__": print(VERSION) diff --git a/release/rtool.py b/release/rtool.py index 14a0a0784..4a07885cb 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -81,35 +81,18 @@ def git(args: str) -> str: def get_version(dev: bool = False, build: bool = False) -> str: - version = runpy.run_path(VERSION_FILE)["VERSION"] - version = re.sub(r"\.dev.+?$", "", version) # replace dev suffix if present. - - last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) - commit = commit.lstrip("g")[:7] - tag_dist = int(tag_dist) - - if tag_dist > 0 and dev: - dev_tag = ".dev{tag_dist:04}".format(tag_dist=tag_dist) - else: - dev_tag = "" - - if tag_dist > 0 and build: - # The wheel build tag (we use the commit) must start with a digit, so we include "0x" - build_tag = "-0x{commit}".format(commit=commit) - else: - build_tag = "" - - return version + dev_tag + build_tag + x = runpy.run_path(VERSION_FILE) + return x["get_version"](dev, build, True) def set_version(dev: bool) -> None: """ - Update version information in mitmproxy's version.py to either include the dev version or not. + Update version information in mitmproxy's version.py to either include hardcoded information or not. """ - v = get_version(dev) - with open(VERSION_FILE) as f: + version = get_version(dev, dev) + with open(VERSION_FILE, "r") as f: content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(v), content) + content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) with open(VERSION_FILE, "w") as f: f.write(content) diff --git a/setup.py b/setup.py index cfda3071c..cd1355970 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() -VERSION = runpy.run_path(os.path.join(here, "mitmproxy", "version.py"))["VERSION"] +VERSION = runpy.run_path(os.path.join(here, "mitmproxy", "version.py"))["VERSION"].split("-0x")[0] setup( name="mitmproxy", diff --git a/test/mitmproxy/test_version.py b/test/mitmproxy/test_version.py index f87b08517..f8d646dc3 100644 --- a/test/mitmproxy/test_version.py +++ b/test/mitmproxy/test_version.py @@ -1,4 +1,6 @@ import runpy +import subprocess +from unittest import mock from mitmproxy import version @@ -8,3 +10,24 @@ def test_version(capsys): stdout, stderr = capsys.readouterr() assert len(stdout) > 0 assert stdout.strip() == version.VERSION + + +def test_get_version_hardcoded(): + version.VERSION = "3.0.0.dev123-0xcafebabe" + assert version.get_version() == "3.0.0" + assert version.get_version(True) == "3.0.0.dev123" + assert version.get_version(True, True) == "3.0.0.dev123-0xcafebabe" + + +def test_get_version(): + version.VERSION = "3.0.0" + + with mock.patch('subprocess.check_output') as m: + m.return_value = b"tag-0-cafecafe" + assert version.get_version(True, True) == "3.0.0" + + m.return_value = b"tag-2-cafecafe" + assert version.get_version(True, True) == "3.0.0.dev0002-0xcafecaf" + + m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long') + assert version.get_version(True, True) == "3.0.0" diff --git a/test/mitmproxy/utils/test_debug.py b/test/mitmproxy/utils/test_debug.py index a8e1054da..0ca6ead0f 100644 --- a/test/mitmproxy/utils/test_debug.py +++ b/test/mitmproxy/utils/test_debug.py @@ -1,5 +1,4 @@ import io -import subprocess import sys from unittest import mock import pytest @@ -14,18 +13,6 @@ def test_dump_system_info_precompiled(precompiled): assert ("binary" in debug.dump_system_info()) == precompiled -def test_dump_system_info_version(): - with mock.patch('subprocess.check_output') as m: - m.return_value = b"v2.0.0-0-cafecafe" - x = debug.dump_system_info() - assert 'dev' not in x - assert 'cafecafe' in x - - with mock.patch('subprocess.check_output') as m: - m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long') - assert 'dev' not in debug.dump_system_info() - - def test_dump_info(): cs = io.StringIO() debug.dump_info(None, None, file=cs, testing=True) From d15bdf8d033b5ff2b5a465e74482e70a2a477069 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 29 Dec 2017 21:55:05 +0100 Subject: [PATCH 08/10] fix appveyor build script we now have a module named "types", which of course confuses Python 3.5. O:-) --- .appveyor.yml | 2 +- mitmproxy/__init__.py | 3 --- release/hooks/hook-mitmproxy.py | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 release/hooks/hook-mitmproxy.py diff --git a/.appveyor.yml b/.appveyor.yml index 7ef9d8b84..a05f33d6f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -34,7 +34,7 @@ install: test_script: - ps: "tox -- --verbose --cov-report=term" - ps: | - $Env:VERSION = $(python mitmproxy/version.py) + $Env:VERSION = $(python -m mitmproxy.version) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" tox -e rtool -- wheel tox -e rtool -- bdist diff --git a/mitmproxy/__init__.py b/mitmproxy/__init__.py index 9697de878..e69de29bb 100644 --- a/mitmproxy/__init__.py +++ b/mitmproxy/__init__.py @@ -1,3 +0,0 @@ -# https://github.com/mitmproxy/mitmproxy/issues/1809 -# import script here so that pyinstaller registers it. -from . import script # noqa diff --git a/release/hooks/hook-mitmproxy.py b/release/hooks/hook-mitmproxy.py new file mode 100644 index 000000000..21932507a --- /dev/null +++ b/release/hooks/hook-mitmproxy.py @@ -0,0 +1 @@ +hiddenimports = ["mitmproxy.script"] From 62455662631da3999344ca6581c93b2fed851e07 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 30 Dec 2017 17:54:23 +0100 Subject: [PATCH 09/10] fix #2470 --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cd1355970..4ae1974bf 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ import os -import runpy from codecs import open +import re from setuptools import setup, find_packages # Based on https://github.com/pypa/sampleproject/blob/master/setup.py @@ -12,7 +12,8 @@ here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() -VERSION = runpy.run_path(os.path.join(here, "mitmproxy", "version.py"))["VERSION"].split("-0x")[0] +with open(os.path.join(here, "mitmproxy", "version.py")) as f: + VERSION = re.search(r'VERSION = "(.+?)(?:-0x|")', f.read()).group(1) setup( name="mitmproxy", From 2f150220e6da495fecf3ed66d5fedeed62d0f915 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 30 Dec 2017 21:27:19 +0100 Subject: [PATCH 10/10] update installbuilder --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index a05f33d6f..3ef985be5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -46,7 +46,7 @@ test_script: ) { echo "Decrypt license..." tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml - $ibVersion = "17.9.0" + $ibVersion = "17.12.0" $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe" $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe" if (!(Test-Path $ibSetup)) {