Rooba
/
mitmproxy
mirror of https://github.com/mitmproxy/mitmproxy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve release workflow

This commit is contained in:
Thomas Kriechbaumer 2018-05-17 11:25:32 +02:00
parent 762aa287cc
commit 2bbfcfae92
4 changed files with 401 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
release/README.md
View File

@ -1,12 +1,12 @@
# Release Checklist
Make sure run all these steps on the correct branch you want to create a new
Make sure to run all these steps on the correct branch you want to create a new
release for! The command examples assume that you have a git remote called
`upstream` that points to the `mitmproxy/mitmproxy` repo.
- Verify that `mitmproxy/version.py` is correct
- Update CHANGELOG
- Update CONTRIBUTORS
- Update CONTRIBUTORS: `git shortlog -n -s > CONTRIBUTORS`
- Verify that all CI tests pass
- Create a major version branch - e.g. `v4.x`. Assuming you have a remote repo called `upstream` that points to the mitmproxy/mitmproxy repo::
- `git checkout -b v4.x upstream/master`
@ -20,16 +20,20 @@ release for! The command examples assume that you have a git remote called
- Wait for tag CI to complete
## GitHub Release
- Create release notice on Github [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already auto-created by the tag.
- We DO NOT upload release artifacts to GitHub anymore. Simply add the following snippet to the notice:
- Create release notice on Github
[here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already
auto-created by the tag.
- We DO NOT upload release artifacts to GitHub anymore. Simply add the
following snippet to the notice:
`You can find the latest release packages on our snapshot server: https://snapshots.mitmproxy.org/v<version number here>`
## PyPi
- Upload the whl file you downloaded in the prevous step
- `twine upload ./tmp/snap/mitmproxy-4.0.0-py3-none-any.whl`
- The created wheel is uploaded to PyPi automatically
- Please check https://pypi.python.org/pypi/mitmproxy about the latest version
## Homebrew
- The Homebrew maintainers are typically very fast and detect our new relese within a day.
- The Homebrew maintainers are typically very fast and detect our new relese
within a day.
- If you feel the need, you can run this from a macOS machine:
`brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>`
@ -52,9 +56,10 @@ release for! The command examples assume that you have a git remote called
- Check the build details page again
## Website
- Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml
- `./build && ./upload-test`
- If everything looks alright: `./upload-prod`
- Update version here:
https://github.com/mitmproxy/www/blob/master/src/config.toml
- Run `./build && ./upload-test`
- If everything looks alright, run `./upload-prod`
## Docs
- Make sure you've uploaded the previous version's docs to archive
@ -64,4 +69,5 @@ release for! The command examples assume that you have a git remote called
## Prepare for next release
- Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
- Last but not least, bump the version on master in
[https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.

114
release/ci.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python3
import glob
import re
import contextlib
import os
import platform
@ -73,17 +75,17 @@ TOOLS = [
TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
if TAG:
VERSION = TAG
VERSION = re.sub('^v', '', TAG)
UPLOAD_DIR = VERSION
elif BRANCH:
VERSION = BRANCH
VERSION = re.sub('^v', '', BRANCH)
UPLOAD_DIR = "branches/%s" % VERSION
else:
print("Could not establish build name - exiting." % BRANCH)
sys.exit(0)
print("BUILD VERSION=%s" % VERSION)
print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
def archive_name(bdist: str) -> str:
@ -99,23 +101,6 @@ def archive_name(bdist: str) -> str:
)
def wheel_name() -> str:
return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
def installer_name() -> str:
ext = {
"Windows": "exe",
"Darwin": "dmg",
"Linux": "run"
}[platform.system()]
return "mitmproxy-{version}-{platform}-installer.{ext}".format(
version=VERSION,
platform=PLATFORM_TAG,
ext=ext,
)
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
@ -134,7 +119,7 @@ def cli():
@cli.command("info")
def info():
print("Version: %s" % VERSION)
click.echo("Version: %s" % VERSION)
@cli.command("build")
@ -142,23 +127,41 @@ def build():
"""
Build a binary distribution
"""
os.makedirs(DIST_DIR, exist_ok=True)
if "WHEEL" in os.environ:
build_wheel()
else:
click.echo("Not building wheels.")
build_pyinstaller()
def build_wheel():
click.echo("Building wheel...")
subprocess.check_call([
"python",
"setup.py",
"-q",
"bdist_wheel",
"--dist-dir", DIST_DIR,
])
whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
click.echo("Found wheel package: {}".format(whl))
subprocess.check_call([
"tox",
"-e", "wheeltest",
"--",
whl
])
def build_pyinstaller():
if exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
os.makedirs(DIST_DIR, exist_ok=True)
if "WHEEL" in os.environ:
print("Building wheel...")
subprocess.check_call(
[
"python",
"setup.py", "-q", "bdist_wheel",
"--dist-dir", "release/dist",
]
)
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
@ -168,7 +171,7 @@ def build():
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
click.echo("Building %s binary..." % tool)
excludes = []
if tool != "mitmweb":
excludes.append("mitmproxy.tools.web")
@ -209,11 +212,11 @@ def build():
)
executable = executable.replace("_main", "")
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
click.echo("> %s --version" % executable)
click.echo(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, basename(executable))
print("Packed {}.".format(archive_name(bdist)))
click.echo("Packed {}.".format(archive_name(bdist)))
def is_pr():
@ -229,25 +232,40 @@ def is_pr():
@cli.command("upload")
def upload():
"""
Upload snapshot to snapshot server
Upload build artifacts to snapshot server and
upload wheel package to PyPi
"""
# This requires some explanation. The AWS access keys are only exposed to
# privileged builds - that is, they are not available to PRs from forks.
# However, they ARE exposed to PRs from a branch within the main repo. This
# check catches that corner case, and prevents an inadvertent upload.
if is_pr():
print("Refusing to upload a pull request")
click.echo("Refusing to upload a pull request")
return
if "AWS_ACCESS_KEY_ID" in os.environ:
subprocess.check_call(
[
"aws", "s3", "cp",
"--acl", "public-read",
DIST_DIR + "/",
"s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
"--recursive",
]
)
subprocess.check_call([
"aws", "s3", "cp",
"--acl", "public-read",
DIST_DIR + "/",
"s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
"--recursive",
])
upload_pypi = (
TAG and
"WHEEL" in os.environ and
"TWINE_USERNAME" in os.environ and
"TWINE_PASSWORD" in os.environ
)
if upload_pypi:
filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
click.echo("Uploading {} to PyPi...".format(filename))
subprocess.check_call([
"twine",
"upload",
join(DIST_DIR, filename)
])
@cli.command("decrypt")

328
release/rtool.py
View File

@ -1,22 +1,78 @@
#!/usr/bin/env python3
import contextlib
import fnmatch
import os
import sys
import platform
import re
import runpy
import shlex
import shutil
import subprocess
from os.path import join, abspath, dirname
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename
import cryptography.fernet
import click
import cryptography.fernet
import pysftp
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
VENV_BIN = "Scripts"
PYINSTALLER_ARGS = [
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
]
else:
VENV_BIN = "bin"
PYINSTALLER_ARGS = []
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
def Archive(name):
a = zipfile.ZipFile(name, "w")
a.add = a.write
return a
else:
def Archive(name):
return tarfile.open(name, "w:gz")
PLATFORM_TAG = {
"Darwin": "osx",
"Windows": "windows",
"Linux": "linux",
}.get(platform.system(), platform.system())
ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
VENV_DIR = join(BUILD_DIR, "venv")
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
BDISTS["mitmproxy"].remove("mitmproxy")
TOOLS = [
tool
for tools in sorted(BDISTS.values())
for tool in tools
]
def git(args: str) -> str:
@ -25,8 +81,50 @@ def git(args: str) -> str:
def get_version(dev: bool = False, build: bool = False) -> str:
x = runpy.run_path(VERSION_FILE)
return x["get_version"](dev, build, True)
version = runpy.run_path(VERSION_FILE)["VERSION"]
version = re.sub(r"\.dev.+?$", "", version) # replace dev suffix if present.
last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
commit = commit.lstrip("g")[:7]
tag_dist = int(tag_dist)
if tag_dist > 0 and dev:
dev_tag = ".dev{tag_dist:04}".format(tag_dist=tag_dist)
else:
dev_tag = ""
if tag_dist > 0 and build:
# The wheel build tag (we use the commit) must start with a digit, so we include "0x"
build_tag = "-0x{commit}".format(commit=commit)
else:
build_tag = ""
return version + dev_tag + build_tag
def set_version(dev: bool) -> None:
"""
Update version information in mitmproxy's version.py to either include the dev version or not.
"""
v = get_version(dev)
with open(VERSION_FILE) as f:
content = f.read()
content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(v), content)
with open(VERSION_FILE, "w") as f:
f.write(content)
def archive_name(bdist: str) -> str:
if platform.system() == "Windows":
ext = "zip"
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=bdist,
version=get_version(),
platform=PLATFORM_TAG,
ext=ext
)
def wheel_name() -> str:
@ -35,6 +133,19 @@ def wheel_name() -> str:
)
def installer_name() -> str:
ext = {
"Windows": "exe",
"Darwin": "dmg",
"Linux": "run"
}[platform.system()]
return "mitmproxy-{version}-{platform}-installer.{ext}".format(
version=get_version(),
platform=PLATFORM_TAG,
ext=ext,
)
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
@ -51,6 +162,24 @@ def cli():
pass
@cli.command("encrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def encrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.encrypt(infile.read()))
@cli.command("decrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.decrypt(infile.read()))
@cli.command("contributors")
def contributors():
"""
@ -63,31 +192,184 @@ def contributors():
f.write(contributors_data.encode())
@cli.command("homebrew-pr")
def homebrew_pr():
@cli.command("wheel")
def make_wheel():
"""
Create a new Homebrew PR
Build a Python wheel
"""
if platform.system() != "Darwin":
print("You need to run this on macOS to create a new Homebrew PR. Sorry.")
sys.exit(1)
set_version(True)
try:
subprocess.check_call([
"tox", "-e", "wheel",
], env={
**os.environ,
"VERSION": get_version(True),
})
finally:
set_version(False)
print("Creating a new PR with Homebrew...")
@cli.command("bdist")
def make_bdist():
"""
Build a binary distribution
"""
if exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
os.makedirs(DIST_DIR, exist_ok=True)
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
# We can't have a folder and a file with the same name.
if tool == "mitmproxy":
tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
excludes = []
if tool != "mitmweb":
excludes.append("mitmproxy.tools.web")
if tool != "mitmproxy_main":
excludes.append("mitmproxy.tools.console")
# Overwrite mitmproxy/version.py to include commit info
set_version(True)
try:
subprocess.check_call(
[
"pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"--additional-hooks-dir", PYINSTALLER_HOOKS,
"--onefile",
"--console",
"--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
]
+ [x for e in excludes for x in ["--exclude-module", e]]
+ PYINSTALLER_ARGS
+ [tool]
)
finally:
set_version(False)
# Delete the spec file - we're good without.
os.remove("{}.spec".format(tool))
# Test if it works at all O:-)
executable = join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
# Remove _main suffix from mitmproxy executable
if "_main" in executable:
shutil.move(
executable,
executable.replace("_main", "")
)
executable = executable.replace("_main", "")
print("> %s --version" % executable)
print(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, basename(executable))
print("Packed {}.".format(archive_name(bdist)))
@cli.command("upload-release")
@click.option('--username', prompt=True)
@click.password_option(confirmation_prompt=False)
@click.option('--repository', default="pypi")
def upload_release(username, password, repository):
"""
Upload wheels to PyPI
"""
filename = wheel_name()
print("Uploading {} to {}...".format(filename, repository))
subprocess.check_call([
"brew",
"bump-formula-pr",
"--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}.tar.gz".format(get_version()),
"mitmproxy",
"twine",
"upload",
"-u", username,
"-p", password,
"-r", repository,
join(DIST_DIR, filename)
])
@cli.command("encrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def encrypt(infile, outfile, key):
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.encrypt(infile.read()))
@cli.command("upload-snapshot")
@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
@click.option("--wheel/--no-wheel", default=False)
@click.option("--bdist/--no-bdist", default=False)
@click.option("--installer/--no-installer", default=False)
def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
"""
Upload snapshot to snapshot server
"""
with pysftp.Connection(host=host,
port=port,
username=user,
private_key=private_key,
private_key_pass=private_key_password) as sftp:
dir_name = "snapshots/v{}".format(get_version())
sftp.makedirs(dir_name)
with sftp.cd(dir_name):
files = []
if wheel:
files.append(wheel_name())
if bdist:
for bdist in sorted(BDISTS.keys()):
files.append(archive_name(bdist))
if installer:
files.append(installer_name())
for f in files:
local_path = join(DIST_DIR, f)
remote_filename = re.sub(
r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
get_version(True, True),
f
)
symlink_path = "../{}".format(f.replace(get_version(), "latest"))
# Upload new version
print("Uploading {} as {}...".format(f, remote_filename))
with click.progressbar(length=os.stat(local_path).st_size) as bar:
# We hide the file during upload
sftp.put(
local_path,
"." + remote_filename,
callback=lambda done, total: bar.update(done - bar.pos)
)
# Delete old versions
old_version = f.replace(get_version(), "*")
for f_old in sftp.listdir():
if fnmatch.fnmatch(f_old, old_version):
print("Removing {}...".format(f_old))
sftp.remove(f_old)
# Show new version
sftp.rename("." + remote_filename, remote_filename)
# update symlink for the latest release
if sftp.lexists(symlink_path):
print("Removing {}...".format(symlink_path))
sftp.remove(symlink_path)
if f != wheel_name():
# "latest" isn't a proper wheel version, so this could not be installed.
# https://github.com/mitmproxy/mitmproxy/issues/1065
sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
if __name__ == "__main__":

14
tox.ini
View File

@ -33,15 +33,27 @@ commands =
python ./test/individual_coverage.py
[testenv:cibuild]
passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
passenv = TRAVIS_* AWS_* APPVEYOR_* TWINE_* RTOOL_KEY WHEEL
deps =
-rrequirements.txt
pyinstaller==3.3.1
twine==1.11.0
awscli
commands =
mitmdump --version
python ./release/ci.py {posargs}
[testenv:wheeltest]
recreate = True
deps =
commands =
pip install {posargs}
mitmproxy --version
mitmdump --version
mitmweb --version
pathod --version
pathoc --version
[testenv:docs]
passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
deps =