Merge pull request #2977 from mitmproxy/updocs2

Revamp CI generation of snapshots and documentation
This commit is contained in:
Aldo Cortesi 2018-03-08 11:42:36 +13:00 committed by GitHub
commit 5617478359
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 324 additions and 429 deletions

View File

@ -9,22 +9,8 @@ environment:
CI_DEPS: codecov>=2.0.5
CI_COMMANDS: codecov
matrix:
- PYTHON: "C:\\Python35"
TOXENV: "py35"
# TODO: ENABLE WHEN AVAILABLE
# - PYTHON: "C:\\Python36"
# TOXENV: "py36"
SNAPSHOT_HOST:
secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
SNAPSHOT_PORT:
secure: TiJPtg60/edYTH8RnoBErg==
SNAPSHOT_USER:
secure: 6yBwmO5gv4vAwoFYII8qjQ==
SNAPSHOT_PASS:
secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
RTOOL_KEY:
secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
- PYTHON: "C:\\Python36"
TOXENV: "py36"
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@ -36,17 +22,10 @@ test_script:
- ps: |
$Env:VERSION = $(python -m mitmproxy.version)
$Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
tox -e rtool -- wheel
tox -e rtool -- bdist
tox -e cibuild -- build
- ps: |
if(
($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
(($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
) {
echo "Decrypt license..."
tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
$ibVersion = "17.12.0"
if($Env:RTOOL_KEY) {
$ibVersion = "18.2.0"
$ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe"
$ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe"
if (!(Test-Path $ibSetup)) {
@ -58,8 +37,8 @@ test_script:
}
echo "Install InstallBuilder..."
Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait
# Wait until executable exists - no idea why this is necessary.
while (!(Test-Path $ibCli)) { Start-Sleep 0.1 }
echo "Decrypt license..."
tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
echo "Run InstallBuilder..."
&$ibCli `
build `
@ -73,15 +52,7 @@ test_script:
}
deploy_script:
# we build binaries on every run, but we only upload them for master snapshots or tags.
ps: |
if(
($Env:TOXENV -match "py35") -and
(($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
) {
tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts
tox -e rtool -- upload-snapshot --bdist --wheel --installer
}
ps: tox -e cibuild -- upload
cache:
- C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml

View File

@ -21,12 +21,12 @@ matrix:
- os: osx
osx_image: xcode7.3
language: generic
env: TOXENV=py35 BDIST=1
env: TOXENV=py36 BDIST=1
- python: 3.5
env: TOXENV=py35 BDIST=1
env: TOXENV=py35
dist: precise
- python: 3.6
env: TOXENV=py36
env: TOXENV=py36 BDIST=1 WHEEL=1
- python: 3.6
env: TOXENV=individual_coverage
- language: node_js
@ -42,6 +42,17 @@ matrix:
yarn: true
directories:
- web/node_modules
- language: python
env:
- NAME=docs
install:
- wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
- sudo dpkg -i hugo*.deb
- pip install awscli
script:
- cd docs && ./ci
after_success:
- echo done
install:
- |
@ -50,12 +61,11 @@ install:
brew update || brew update
brew outdated pyenv || brew upgrade pyenv
eval "$(pyenv init -)"
env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3
pyenv global 3.5.3
pyenv shell 3.5.3
pip install -U pip setuptools wheel virtualenv
env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4
pyenv global 3.6.4
pyenv shell 3.6.4
fi
- pip install tox
- pip install tox virtualenv setuptools
script:
- tox -- --verbose --cov-report=term
@ -63,16 +73,14 @@ script:
if [[ $BDIST == "1" ]]
then
git fetch --unshallow --tags
tox -e rtool -- bdist
tox -e cibuild -- build
fi
after_success:
# we build binaries on every run, but we only upload them for master snapshots or tags.
- |
if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
if [[ $BDIST == "1" ]]
then
tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts
tox -e rtool -- upload-snapshot --bdist
tox -e cibuild -- upload
fi
notifications:

13
docs/ci Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
# This script gets run from CI to render and upload docs
./build
# Only upload if we have defined credentials - we only have these defined for
# trusted commits (i.e. not PRs).
if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
--paths "/master"
fi

View File

@ -12,6 +12,6 @@ SPATH="/archive/$1"
aws configure set preview.cloudfront true
aws --profile mitmproxy \
s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH
# aws --profile mitmproxy \
# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
# --paths "$SPATH"
aws --profile mitmproxy \
cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
--paths "$SPATH"

View File

@ -1,8 +0,0 @@
#!/bin/sh
aws configure set preview.cloudfront true
aws --profile mitmproxy \
s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
# aws --profile mitmproxy \
# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
# --paths "/master"

View File

@ -3,6 +3,6 @@
aws configure set preview.cloudfront true
aws --profile mitmproxy \
s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable
# aws --profile mitmproxy \
# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
# --paths "/master"
aws --profile mitmproxy \
cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
--paths "/stable"

263
release/ci.py Executable file
View File

@ -0,0 +1,263 @@
#!/usr/bin/env python3
import contextlib
import os
import platform
import sys
import shutil
import subprocess
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename
import cryptography.fernet
import click
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
VENV_BIN = "Scripts"
PYINSTALLER_ARGS = [
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
]
else:
VENV_BIN = "bin"
PYINSTALLER_ARGS = []
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
def Archive(name):
a = zipfile.ZipFile(name, "w")
a.add = a.write
return a
else:
def Archive(name):
return tarfile.open(name, "w:gz")
PLATFORM_TAG = {
"Darwin": "osx",
"Windows": "windows",
"Linux": "linux",
}.get(platform.system(), platform.system())
ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
VENV_DIR = join(BUILD_DIR, "venv")
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
BDISTS["mitmproxy"].remove("mitmproxy")
TOOLS = [
tool
for tools in sorted(BDISTS.values())
for tool in tools
]
TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
if TAG:
VERSION = TAG
UPLOAD_DIR = VERSION
elif BRANCH:
VERSION = BRANCH
UPLOAD_DIR = "branches/%s" % VERSION
else:
print("Could not establish build name - exiting." % BRANCH)
sys.exit(0)
print("BUILD VERSION=%s" % VERSION)
def archive_name(bdist: str) -> str:
if platform.system() == "Windows":
ext = "zip"
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=bdist,
version=VERSION,
platform=PLATFORM_TAG,
ext=ext
)
def wheel_name() -> str:
return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
def installer_name() -> str:
ext = {
"Windows": "exe",
"Darwin": "dmg",
"Linux": "run"
}[platform.system()]
return "mitmproxy-{version}-{platform}-installer.{ext}".format(
version=VERSION,
platform=PLATFORM_TAG,
ext=ext,
)
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
os.chdir(path)
yield
os.chdir(old_dir)
@click.group(chain=True)
def cli():
"""
mitmproxy build tool
"""
pass
@cli.command("info")
def info():
print("Version: %s" % VERSION)
@cli.command("build")
def build():
"""
Build a binary distribution
"""
if exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
os.makedirs(DIST_DIR, exist_ok=True)
if "WHEEL" in os.environ:
print("Building wheel...")
subprocess.check_call(
[
"python",
"setup.py", "-q", "bdist_wheel",
"--dist-dir", "release/dist",
]
)
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
# We can't have a folder and a file with the same name.
if tool == "mitmproxy":
tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
excludes = []
if tool != "mitmweb":
excludes.append("mitmproxy.tools.web")
if tool != "mitmproxy_main":
excludes.append("mitmproxy.tools.console")
subprocess.check_call(
[
"pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"--additional-hooks-dir", PYINSTALLER_HOOKS,
"--onefile",
"--console",
"--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
]
+ [x for e in excludes for x in ["--exclude-module", e]]
+ PYINSTALLER_ARGS
+ [tool]
)
# Delete the spec file - we're good without.
os.remove("{}.spec".format(tool))
# Test if it works at all O:-)
executable = join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
# Remove _main suffix from mitmproxy executable
if "_main" in executable:
shutil.move(
executable,
executable.replace("_main", "")
)
executable = executable.replace("_main", "")
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, basename(executable))
print("Packed {}.".format(archive_name(bdist)))
def is_pr():
if "TRAVIS_PULL_REQUEST" in os.environ:
if os.environ["TRAVIS_PULL_REQUEST"] == "false":
return False
return True
elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"):
return True
return False
@cli.command("upload")
def upload():
"""
Upload snapshot to snapshot server
"""
# This requires some explanation. The AWS access keys are only exposed to
# privileged builds - that is, they are not available to PRs from forks.
# However, they ARE exposed to PRs from a branch within the main repo. This
# check catches that corner case, and prevents an inadvertent upload.
if is_pr():
print("Refusing to upload a pull request")
return
if "AWS_ACCESS_KEY_ID" in os.environ:
subprocess.check_call(
[
"aws", "s3", "cp",
"--acl", "public-read",
DIST_DIR + "/",
"s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
"--recursive",
]
)
@cli.command("decrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.decrypt(infile.read()))
if __name__ == "__main__":
cli()

View File

@ -1 +0,0 @@
gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7

View File

@ -1,68 +0,0 @@
-----BEGIN RSA PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB
0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y
byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU
V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z
T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh
PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl
w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD
P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt
JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE
yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km
cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe
hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8
jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx
QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey
OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL
kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR
AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc
sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko
SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz
5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P
cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk
kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/
Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9
3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4
mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t
8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j
FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf
EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP
a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR
AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+
m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P
bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q
QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx
uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq
yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K
u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK
MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0
8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq
y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz
G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi
o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq
2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL
5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp
30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac
7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA
1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8
eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB
bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO
VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+
2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175
-----END RSA PRIVATE KEY-----
-----BEGIN PUBLIC KEY-----
MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y
qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe
24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX
FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ
0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI
KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt
0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+
tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ
TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ
q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J
6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD
5I1jQDXFbjAgAnaSyqyW368CAwEAAQ==
-----END PUBLIC KEY-----

View File

@ -1,79 +1,22 @@
#!/usr/bin/env python3
import contextlib
import fnmatch
import os
import sys
import platform
import re
import runpy
import shlex
import shutil
import subprocess
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename
from os.path import join, abspath, dirname
import click
import cryptography.fernet
import pysftp
import click
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
VENV_BIN = "Scripts"
PYINSTALLER_ARGS = [
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
]
else:
VENV_BIN = "bin"
PYINSTALLER_ARGS = []
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
def Archive(name):
a = zipfile.ZipFile(name, "w")
a.add = a.write
return a
else:
def Archive(name):
return tarfile.open(name, "w:gz")
PLATFORM_TAG = {
"Darwin": "osx",
"Windows": "windows",
"Linux": "linux",
}.get(platform.system(), platform.system())
ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
VENV_DIR = join(BUILD_DIR, "venv")
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
BDISTS["mitmproxy"].remove("mitmproxy")
TOOLS = [
tool
for tools in sorted(BDISTS.values())
for tool in tools
]
def git(args: str) -> str:
@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str:
return x["get_version"](dev, build, True)
def set_version(dev: bool) -> None:
"""
Update version information in mitmproxy's version.py to either include hardcoded information or not.
"""
version = get_version(dev, dev)
with open(VERSION_FILE, "r") as f:
content = f.read()
content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M)
with open(VERSION_FILE, "w") as f:
f.write(content)
def archive_name(bdist: str) -> str:
if platform.system() == "Windows":
ext = "zip"
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=bdist,
version=get_version(),
platform=PLATFORM_TAG,
ext=ext
)
def wheel_name() -> str:
return "mitmproxy-{version}-py3-none-any.whl".format(
version=get_version(True),
)
def installer_name() -> str:
ext = {
"Windows": "exe",
"Darwin": "dmg",
"Linux": "run"
}[platform.system()]
return "mitmproxy-{version}-{platform}-installer.{ext}".format(
version=get_version(),
platform=PLATFORM_TAG,
ext=ext,
)
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
@ -146,24 +51,6 @@ def cli():
pass
@cli.command("encrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def encrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.encrypt(infile.read()))
@cli.command("decrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.decrypt(infile.read()))
@cli.command("contributors")
def contributors():
"""
@ -176,97 +63,6 @@ def contributors():
f.write(contributors_data.encode())
@cli.command("wheel")
def make_wheel():
"""
Build a Python wheel
"""
set_version(True)
try:
subprocess.check_call([
"tox", "-e", "wheel",
], env={
**os.environ,
"VERSION": get_version(True),
})
finally:
set_version(False)
@cli.command("bdist")
def make_bdist():
"""
Build a binary distribution
"""
if exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
os.makedirs(DIST_DIR, exist_ok=True)
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
# We can't have a folder and a file with the same name.
if tool == "mitmproxy":
tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
excludes = []
if tool != "mitmweb":
excludes.append("mitmproxy.tools.web")
if tool != "mitmproxy_main":
excludes.append("mitmproxy.tools.console")
# Overwrite mitmproxy/version.py to include commit info
set_version(True)
try:
subprocess.check_call(
[
"pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"--additional-hooks-dir", PYINSTALLER_HOOKS,
"--onefile",
"--console",
"--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
]
+ [x for e in excludes for x in ["--exclude-module", e]]
+ PYINSTALLER_ARGS
+ [tool]
)
finally:
set_version(False)
# Delete the spec file - we're good without.
os.remove("{}.spec".format(tool))
# Test if it works at all O:-)
executable = join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
# Remove _main suffix from mitmproxy executable
if "_main" in executable:
shutil.move(
executable,
executable.replace("_main", "")
)
executable = executable.replace("_main", "")
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, basename(executable))
print("Packed {}.".format(archive_name(bdist)))
@cli.command("upload-release")
@click.option('--username', prompt=True)
@click.password_option(confirmation_prompt=False)
@ -305,77 +101,13 @@ def homebrew_pr():
])
@cli.command("upload-snapshot")
@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
@click.option("--wheel/--no-wheel", default=False)
@click.option("--bdist/--no-bdist", default=False)
@click.option("--installer/--no-installer", default=False)
def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
"""
Upload snapshot to snapshot server
"""
cnopts = pysftp.CnOpts(
knownhosts=join(RELEASE_DIR, 'known_hosts')
)
with pysftp.Connection(host=host,
port=port,
username=user,
private_key=private_key,
private_key_pass=private_key_password,
cnopts=cnopts) as sftp:
dir_name = "snapshots/v{}".format(get_version())
sftp.makedirs(dir_name)
with sftp.cd(dir_name):
files = []
if wheel:
files.append(wheel_name())
if bdist:
for bdist in sorted(BDISTS.keys()):
files.append(archive_name(bdist))
if installer:
files.append(installer_name())
for f in files:
local_path = join(DIST_DIR, f)
remote_filename = re.sub(
r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
get_version(True, True),
f
)
symlink_path = "../{}".format(f.replace(get_version(), "latest"))
# Upload new version
print("Uploading {} as {}...".format(f, remote_filename))
with click.progressbar(length=os.stat(local_path).st_size) as bar:
# We hide the file during upload
sftp.put(
local_path,
"." + remote_filename,
callback=lambda done, total: bar.update(done - bar.pos)
)
# Delete old versions
old_version = f.replace(get_version(), "*")
for f_old in sftp.listdir():
if fnmatch.fnmatch(f_old, old_version):
print("Removing {}...".format(f_old))
sftp.remove(f_old)
# Show new version
sftp.rename("." + remote_filename, remote_filename)
# update symlink for the latest release
if sftp.lexists(symlink_path):
print("Removing {}...".format(symlink_path))
sftp.remove(symlink_path)
if f != wheel_name():
# "latest" isn't a proper wheel version, so this could not be installed.
# https://github.com/mitmproxy/mitmproxy/issues/1065
sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
@cli.command("encrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def encrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.encrypt(infile.read()))
if __name__ == "__main__":

23
tox.ini
View File

@ -32,27 +32,12 @@ deps =
commands =
python test/individual_coverage.py
[testenv:wheel]
recreate = True
deps =
commands =
python setup.py -q bdist_wheel --dist-dir release/dist
pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl
# skip `mitmproxy --version` if SKIP_MITMPROXY is defined.
{env:SKIP_MITMPROXY:mitmproxy --version}
mitmdump --version
mitmweb --version
pathod --version
pathoc --version
[testenv:rtool]
passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY
[testenv:cibuild]
passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
deps =
-rrequirements.txt
pyinstaller==3.3.1
twine==1.9.1
pysftp==0.2.9
awscli
commands =
mitmdump --version
python ./release/rtool.py {posargs}
python ./release/ci.py {posargs}