Rooba
/
mitmproxy
mirror of https://github.com/mitmproxy/mitmproxy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make build script amazing

This commit is contained in:
Maximilian Hils 2015-07-22 02:43:45 +02:00
parent d2d2edc140
commit fe03a656a9
5 changed files with 170 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.mkd
View File

@ -57,7 +57,7 @@ $ git clone https://github.com/mitmproxy/mitmproxy.git
$ git clone https://github.com/mitmproxy/netlib.git
$ git clone https://github.com/mitmproxy/pathod.git
$ cd mitmproxy
$ ./dev
$ source ./dev
```
The *dev* script will create a virtualenv environment in a directory called

168
release/build.py
View File

@ -1,15 +1,13 @@
#!/usr/bin/env python
from os.path import dirname, realpath, join, exists, normpath
from os import chdir, mkdir, getcwd
from os import chdir
import os
import shutil
import subprocess
import tempfile
import glob
import re
from shlex import split
from contextlib import contextmanager
import click
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
@ -25,9 +23,9 @@ dist_dir = join(mitmproxy_dir, "dist")
test_venv_dir = join(root_dir, "venv.mitmproxy-release")
projects = ("netlib", "pathod", "mitmproxy")
tools = ["mitmweb", "mitmdump", "pathod", "pathoc"]
if os.name != "nt":
tools.append("mitmproxy")
tools = ["mitmproxy", "mitmdump", "mitmweb", "pathod", "pathoc"]
if os.name == "nt":
tools.remove("mitmproxy")
version_files = (join(root_dir, x) for x in (
"mitmproxy/libmproxy/version.py",
@ -35,6 +33,7 @@ version_files = (join(root_dir, x) for x in (
"netlib/netlib/version.py"
))
@click.group(chain=True)
def cli():
"""
@ -44,15 +43,21 @@ def cli():
@cli.command("contributors")
def update_contributors():
def contributors():
"""
Update CONTRIBUTORS.md
"""
print("Updating CONTRIBUTORS.md...")
contributors = subprocess.check_output(split("git shortlog -n -s"))
contributors_data = subprocess.check_output(split("git shortlog -n -s"))
with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f:
f.write(contributors)
f.write(contributors_data)
@cli.command("docs")
def render_docs():
def docs():
"""
Render the docs
"""
print("Rendering the docs...")
subprocess.check_call([
"cshape",
@ -61,11 +66,65 @@ def render_docs():
])
@cli.command("set-version")
@click.argument('version')
def set_version(version):
"""
Update version information
"""
print("Update versions...")
version = ", ".join(version.split("."))
for version_file in version_files:
print("Update %s..." % version_file)
with open(version_file, "rb") as f:
content = f.read()
new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content)
with open(version_file, "wb") as f:
f.write(new_content)
@cli.command("git")
@click.argument('args', nargs=-1, required=True)
def git(args):
"""
Run a git command on every project
"""
args = ["git"] + list(args)
for project in projects:
print("%s> %s..." % (project, " ".join(args)))
subprocess.check_call(
args,
cwd=join(root_dir, project)
)
@cli.command("sdist")
def sdist():
"""
Build a source distribution
"""
# Make sure that the regular python installation is not on the python path!
os.environ["PYTHONPATH"] = ""
print("Building release...")
if exists(dist_dir):
shutil.rmtree(dist_dir)
for project in projects:
print("Creating %s source distribution..." % project)
subprocess.check_call(
["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"],
cwd=join(root_dir, project)
)
@cli.command("test")
@click.pass_context
def test(ctx):
"""
Test the source distribution
"""
if not exists(dist_dir):
ctx.invoke(release)
ctx.invoke(sdist)
# Make sure that the regular python installation is not on the python path!
os.environ["PYTHONPATH"] = ""
@ -89,50 +148,7 @@ def test(ctx):
print(subprocess.check_output([tool, "--version"]))
print("Virtualenv available for further testing:")
print(normpath(join(test_venv_dir, venv_bin, "activate")))
@cli.command("release")
def release():
os.environ["PYTHONPATH"] = ""
print("Building release...")
if exists(dist_dir):
shutil.rmtree(dist_dir)
for project in projects:
print("Creating %s source distribution..." % project)
subprocess.check_call(
["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"],
cwd=join(root_dir, project)
)
@cli.command("set-version")
@click.argument('version')
def set_version(version):
version = ", ".join(version.split("."))
for version_file in version_files:
with open(version_file, "rb") as f:
content = f.read()
new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content)
with open(version_file, "wb") as f:
f.write(new_content)
@cli.command("add-tag")
@click.argument('version')
def git_tag(version):
for project in projects:
print("Tagging %s..." % project)
subprocess.check_call(
["git", "tag", version],
cwd=join(root_dir, project)
)
subprocess.check_call(
["git", "push", "--tags"],
cwd=join(root_dir, project)
)
print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate")))
@cli.command("upload")
@ -140,6 +156,9 @@ def git_tag(version):
@click.password_option(confirmation_prompt=False)
@click.option('--repository', default="pypi")
def upload_release(username, password, repository):
"""
Upload source distributions to PyPI
"""
print("Uploading distributions...")
subprocess.check_call([
"twine",
@ -151,5 +170,44 @@ def upload_release(username, password, repository):
])
"""
TODO: Fully automate build process.
This skeleton is missing OSX builds and updating mitmproxy.org.
@cli.command("wizard")
@click.option('--version', prompt=True)
@click.option('--username', prompt=True)
@click.password_option(confirmation_prompt=False)
@click.option('--repository', default="pypi")
@click.option('--test/--no-test', default=True)
@click.pass_context
def wizard(ctx, version, username, password, repository, test):
""
Interactive Release Wizard
""
for project in projects:
if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)):
raise RuntimeError("%s repository is not clean." % project)
if test:
ctx.invoke(sdist)
ctx.invoke(test)
click.confirm("Please test the release now. Is it ok?", abort=True)
ctx.invoke(set_version, version=version)
ctx.invoke(docs)
ctx.invoke(contributors)
ctx.invoke(git, args=["commit", "-a", "-m", "bump version"])
ctx.invoke(git, args=["tag", "v" + version])
ctx.invoke(git, args=["push", "--tags"])
ctx.invoke(sdist)
ctx.invoke(upload_release, username=username, password=password, repository=repository)
click.echo("All done!")
"""
if __name__ == "__main__":
cli()

42
release/release-checklist
View File

@ -1,42 +0,0 @@
- Check the version number:
mitmproxy/libmproxy/version.py
netlib/netlib/version.py
pathod/libpathod/version.py
- Ensure that the website style assets have been compiled for production, and
synced to the docs.
- Render the docs, update CONTRIBUTORS file:
./release/build.py docs contributors
- Run the test release, make sure the output is sensible
./release/build.py release
- Build the OSX binaries
- Follow instructions in osx-binaries
- Move to download dir:
mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
- Move all source distributions from mitmproxy's dist folder to the server:
mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download
- Tag with the version number, and do:
git push --tags
Tag and push v0.12 for all projects:
./release/build.py tag v0.12
- Upload to pypi:
./release/build.py upload
Be careful: Pypi requires you to bump the version number if you want to do any further changes.
- Now bump the version number to be ready for the next cycle:
TODO: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now?
mitmproxy/libmproxy/version.py
netlib/netlib/version.py
pathod/libpathod/version.py

55
release/release-checklist.md Normal file
View File

@ -0,0 +1,55 @@
# Release Checklist
## Test
- Create the source distributions, make sure the output is sensible:
`./release/build.py release`
All source distributions can be found in `./dist`.
- Test the source distributions:
`./release/build.py test`
This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it.
## Release
- Verify that repositories are in a clean state:
`./release/build.py git status`
- Update the version number in `version.py` for all projects:
`./release/build.py set-version 0.13`
- Ensure that the website style assets have been compiled for production, and synced to the docs.
- Render the docs, update CONTRIBUTORS file:
`./release/build.py docs contributors`
- Make version bump commit for all projects, tag and push it:
`./release/build.py git commit -am "bump version"`
`./release/build.py git tag v0.13`
`./release/build.py git push --tags`
- Recreate the source distributions with updated version information:
`./release/build.py sdist`
- Build the OSX binaries
- Follow instructions in osx-binaries
- Move to download dir:
`mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download`
- Move all source distributions from `./dist` to the server:
`mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download`
- Upload distributions in `./dist` to PyPI:
`./release/build.py upload`
You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`.
([more info](https://tom-christie.github.io/articles/pypi/))
- Now bump the version number to be ready for the next cycle:
**TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now?
We should probably just leave it as-is and only bump once we actually do the next release.
Also, we need a release policy. I propose the following:
- By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects.
- Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project.
This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)`

1
setup.py
View File

@ -39,6 +39,7 @@ dev_deps = {
"nose-cov>=1.6",
"coveralls>=0.4.1",
"click>=4.1",
"twine>=1.5.0",
"pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
"countershape"
}