Rooba
/
mitmproxy
mirror of https://github.com/mitmproxy/mitmproxy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix pyinstaller 

amongst other things, fix #1746
This commit is contained in:
Maximilian Hils 2016-11-28 00:51:52 +01:00 committed by GitHub
parent 568f40c810
commit 3154dc87fd
16 changed files with 141 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.appveyor.yml
View File

@ -24,17 +24,20 @@ install:
test_script:
- ps: "tox -- --cov mitmproxy --cov pathod -v"
- ps: |
$Env:VERSION = $(python mitmproxy/version.py)
$Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
tox -e wheel -- https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl
tox -e rtool -- bdist
deploy_script:
# we build binaries on every run, but we only upload them for master snapshots or tags.
ps: |
if(
($Env:TOXENV -match "py35") -and
(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
) {
pip install -U virtualenv
.\dev.ps1
cmd /c "python -u .\release\rtool.py bdist 2>&1"
python -u .\release\rtool.py upload-snapshot --bdist --wheel
(($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
) {
tox -e rtool -- upload-snapshot --bdist --wheel
}
cache:

19
.travis.yml
View File

@ -51,18 +51,21 @@ install:
fi
- pip install tox
script: tox -- --cov mitmproxy --cov pathod -v
script:
- |
tox -- --cov mitmproxy --cov pathod -v
if [[ $BDIST == "1" ]]
then
git fetch --unshallow --tags
tox -e rtool -- bdist
fi
after_success:
# we build binaries on every run, but we only upload them for master snapshots or tags.
- |
if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
then
git fetch --unshallow
./dev.sh 3.5
source venv3.5/bin/activate
pip install -e ./release
python -u ./release/rtool.py bdist
python -u ./release/rtool.py upload-snapshot --bdist
tox -e rtool -- upload-snapshot --bdist
fi
notifications:

2
dev.ps1
View File

@ -1,7 +1,7 @@
$ErrorActionPreference = "Stop"
$VENV = ".\venv"
virtualenv $VENV --always-copy
python3 -m venv $VENV --copies
& $VENV\Scripts\activate.ps1
python -m pip install --disable-pip-version-check -U pip

5
mitmproxy/tools/__init__.py
View File

@ -1,5 +0,0 @@
from mitmproxy.tools import web
from mitmproxy.tools import console
from mitmproxy.tools import dump
__all__ = ["web", "console", "dump"]

4
mitmproxy/utils/data.py
View File

@ -15,7 +15,7 @@ class Data:
"""
Change the data object to a path relative to the module.
"""
dirname = os.path.join(self.dirname, subpath)
dirname = os.path.normpath(os.path.join(self.dirname, subpath))
ret = Data(self.name)
ret.dirname = dirname
return ret
@ -27,7 +27,7 @@ class Data:
This function will raise ValueError if the path does not exist.
"""
fullpath = os.path.join(self.dirname, path)
fullpath = os.path.normpath(os.path.join(self.dirname, path))
if not os.path.exists(fullpath):
raise ValueError("dataPath: %s does not exist." % fullpath)
return fullpath

3
mitmproxy/version.py
View File

@ -2,3 +2,6 @@ IVERSION = (0, 19)
VERSION = ".".join(str(i) for i in IVERSION)
PATHOD = "pathod " + VERSION
MITMPROXY = "mitmproxy " + VERSION
if __name__ == "__main__":
print(VERSION)

43
release/hooks/hook-cryptography.py Normal file
View File

@ -0,0 +1,43 @@
# Taken from the latest pyinstaller master on 2016-11-27 (0729a2b).
#-----------------------------------------------------------------------------
# Copyright (c) 2005-2016, PyInstaller Development Team.
#
# Distributed under the terms of the GNU General Public License with exception
# for distributing bootloader.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------
"""
Hook for cryptography module from the Python Cryptography Authority.
"""
import os.path
import glob
from PyInstaller.compat import EXTENSION_SUFFIXES
from PyInstaller.utils.hooks import collect_submodules, get_module_file_attribute
from PyInstaller.utils.hooks import copy_metadata
# get the package data so we can load the backends
datas = copy_metadata('cryptography')
# Add the backends as hidden imports
hiddenimports = collect_submodules('cryptography.hazmat.backends')
# Add the OpenSSL FFI binding modules as hidden imports
hiddenimports += collect_submodules('cryptography.hazmat.bindings.openssl') + ['_cffi_backend']
# Include the cffi extensions as binaries in a subfolder named like the package.
# The cffi verifier expects to find them inside the package directory for
# the main module. We cannot use hiddenimports because that would add the modules
# outside the package.
binaries = []
cryptography_dir = os.path.dirname(get_module_file_attribute('cryptography'))
for ext in EXTENSION_SUFFIXES:
ffimods = glob.glob(os.path.join(cryptography_dir, '*_cffi_*%s*' % ext))
for f in ffimods:
binaries.append((f, 'cryptography'))

2
release/hooks/hook-mitmproxy.addons.onboardingapp.py
View File

@ -1,3 +1,3 @@
from PyInstaller.utils.hooks import collect_data_files
datas = collect_data_files('mitmproxy.addons.onboardingapp')
datas = collect_data_files('mitmproxy.addons.onboardingapp')

3
release/hooks/hook-mitmproxy.tools.web.py Normal file
View File

@ -0,0 +1,3 @@
from PyInstaller.utils.hooks import collect_data_files
datas = collect_data_files('mitmproxy.tools.web')

126
release/rtool.py
View File

@ -8,10 +8,9 @@ import runpy
import shlex
import shutil
import subprocess
import sys
import tarfile
import zipfile
from os.path import join, abspath, normpath, dirname, exists, basename
from os.path import join, abspath, dirname, exists, basename
import click
import pysftp
@ -20,8 +19,13 @@ import pysftp
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
VENV_BIN = "Scripts"
PYINSTALLER_ARGS = [
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
]
else:
VENV_BIN = "bin"
PYINSTALLER_ARGS = []
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
@ -46,13 +50,9 @@ PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
VENV_DIR = join(BUILD_DIR, "venv")
VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
PROJECT_NAME = "mitmproxy"
PYTHON_VERSION = "py2.py3"
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
@ -62,20 +62,20 @@ if platform.system() == "Windows":
TOOLS = [
tool
for tools in BDISTS.values()
for tools in sorted(BDISTS.values())
for tool in tools
]
def get_version() -> str:
return runpy.run_path(VERSION_FILE)["VERSION"]
def git(args: str) -> str:
with chdir(ROOT_DIR):
return subprocess.check_output(["git"] + shlex.split(args)).decode()
def get_version() -> str:
return runpy.run_path(VERSION_FILE)["VERSION"]
def get_snapshot_version() -> str:
last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
tag_dist = int(tag_dist)
@ -109,25 +109,11 @@ def archive_name(bdist: str) -> str:
def wheel_name() -> str:
return "{project}-{version}-{py_version}-none-any.whl".format(
project=PROJECT_NAME,
return "mitmproxy-{version}-py3-none-any.whl".format(
version=get_version(),
py_version=PYTHON_VERSION
)
@contextlib.contextmanager
def empty_pythonpath():
"""
Make sure that the regular python installation is not on the python path,
which would give us access to modules installed outside of our virtualenv.
"""
pythonpath = os.environ.get("PYTHONPATH", "")
os.environ["PYTHONPATH"] = ""
yield
os.environ["PYTHONPATH"] = pythonpath
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
@ -156,58 +142,8 @@ def contributors():
f.write(contributors_data.encode())
@cli.command("wheel")
def make_wheel():
"""
Build wheel
"""
with empty_pythonpath():
if exists(DIST_DIR):
shutil.rmtree(DIST_DIR)
print("Creating wheel...")
subprocess.check_call(
[
"python3", "./setup.py", "-q",
"bdist_wheel", "--dist-dir", DIST_DIR, "--universal"
],
cwd=ROOT_DIR
)
print("Creating virtualenv for test install...")
if exists(VENV_DIR):
shutil.rmtree(VENV_DIR)
subprocess.check_call(["python3", "-m", "virtualenv", "-q", VENV_DIR])
with chdir(DIST_DIR):
print("Install wheel into virtualenv...")
# lxml...
if platform.system() == "Windows" and sys.version_info[0] == 3:
subprocess.check_call(
[VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]
)
subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()])
print("Running tools...")
for tool in TOOLS:
tool = join(VENV_DIR, VENV_BIN, tool)
print("> %s --version" % tool)
print(subprocess.check_output([tool, "--version"]).decode())
print("Virtualenv available for further testing:")
print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate")))
@cli.command("bdist")
@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False)
@click.argument(
"pyinstaller_version",
envvar="PYINSTALLER_VERSION",
default="git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c"
)
@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1")
@click.pass_context
def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version):
def make_bdist():
"""
Build a binary distribution
"""
@ -216,37 +152,40 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
if not use_existing_wheel:
ctx.invoke(make_wheel)
os.makedirs(DIST_DIR, exist_ok=True)
print("Installing PyInstaller and setuptools...")
subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version])
print(subprocess.check_output([VENV_PIP, "freeze"]).decode())
for bdist, tools in BDISTS.items():
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
# We can't have a folder and a file with the same name.
if tool == "mitmproxy":
tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
excludes = []
if tool != "mitmweb":
excludes.append("mitmproxy.tools.web")
if tool != "mitmproxy_main":
excludes.append("mitmproxy.tools.console")
subprocess.check_call(
[
VENV_PYINSTALLER,
"pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"--additional-hooks-dir", PYINSTALLER_HOOKS,
# PyInstaller 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
"--onefile",
"--console",
"--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
tool
]
+ [x for e in excludes for x in ["--exclude-module", e]]
+ PYINSTALLER_ARGS
+ [tool]
)
# Delete the spec file - we're good without.
os.remove("{}.spec".format(tool))
@ -255,6 +194,15 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version)
executable = join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
# Remove _main suffix from mitmproxy executable
if executable.startswith("mitmproxy_main"):
shutil.move(
executable,
executable.replace("_main", "")
)
executable = executable.replace("_main", "")
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
@ -306,7 +254,7 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel,
if wheel:
files.append(wheel_name())
if bdist:
for bdist in BDISTS.keys():
for bdist in sorted(BDISTS.keys()):
files.append(archive_name(bdist))
for f in files:

2
release/setup.py
View File

@ -7,8 +7,6 @@ setup(
install_requires=[
"click>=6.2, <7.0",
"twine>=1.6.5, <1.9",
"virtualenv>=14.0.5, <15.2",
"wheel>=0.29.0, <0.30",
"pysftp==0.2.8",
],
entry_points={

0
release/specs/mitmproxy → release/specs/mitmproxy_main
View File

1
requirements.txt
View File

@ -1,3 +1,2 @@
https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5'
-e .[dev,examples,contentviews]
-e ./release

3
setup.py
View File

@ -84,6 +84,9 @@ setup(
"watchdog>=0.8.3, <0.9",
"brotlipy>=0.5.1, <0.7",
"sortedcontainers>=1.5.4, <1.6",
# transitive from cryptography, we just blacklist here.
# https://github.com/pypa/setuptools/issues/861
"setuptools>=11.3, !=29.0.0",
],
extras_require={
':sys_platform == "win32"': [

4
test/mitmproxy/addons/test_script.py
View File

@ -156,8 +156,8 @@ class TestScript:
sc.request(f)
assert tctx.master.event_log[0][0] == "error"
assert len(tctx.master.event_log[0][1].splitlines()) == 6
assert re.search('addonscripts/error.py", line \d+, in request', tctx.master.event_log[0][1])
assert re.search('addonscripts/error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1])
assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n")
def test_addon(self):

27
tox.ini
View File

@ -4,10 +4,11 @@ skipsdist = True
toxworkdir={env:TOX_WORK_DIR:.tox}
[testenv]
basepython = python3.5
deps =
{env:CI_DEPS:}
-rrequirements.txt
passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_*
passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_*
setenv = HOME = {envtmpdir}
commands =
mitmdump --sysinfo
@ -29,3 +30,27 @@ commands =
mitmproxy/proxy/protocol/ \
mitmproxy/log.py \
mitmproxy/tools/dump.py mitmproxy/tools/web
[testenv:wheel]
recreate = True
deps =
commands =
python setup.py -q bdist_wheel --dist-dir release/dist
pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl
# skip `mitmproxy --version` if SKIP_MITMPROXY is defined.
{env:SKIP_MITMPROXY:mitmproxy --version}
mitmdump --version
mitmweb --version
pathod --version
pathoc --version
[testenv:rtool]
deps =
-rrequirements.txt
-e./release
# The 3.2 release is broken 🎉
# the next commit after this updates the bootloaders, which then segfault! 🎉
# https://github.com/pyinstaller/pyinstaller/issues/2232
git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c
commands =
rtool {posargs}