From 8792e072108b46cda4d80a9d47d43e3c460e5c62 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 08:00:55 +1300 Subject: [PATCH 01/12] docs: enable upload to docs.mitmproxy.org/master --- .travis.yml | 61 +++++++++++++++++++++++++++------------------- docs/ci | 13 ++++++++++ docs/upload-master | 8 ------ 3 files changed, 49 insertions(+), 33 deletions(-) create mode 100755 docs/ci delete mode 100755 docs/upload-master diff --git a/.travis.yml b/.travis.yml index c22038fd8..9df9bd15b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,32 +16,43 @@ git: matrix: fast_finish: true include: - - python: 3.6 - env: TOXENV=lint - - os: osx - osx_image: xcode7.3 - language: generic - env: TOXENV=py35 BDIST=1 - - python: 3.5 - env: TOXENV=py35 BDIST=1 - dist: precise - - python: 3.6 - env: TOXENV=py36 - - python: 3.6 - env: TOXENV=individual_coverage - - language: node_js - node_js: "node" - before_install: - - curl -o- -L https://yarnpkg.com/install.sh | bash - - export PATH=$HOME/.yarn/bin:$PATH + # - python: 3.6 + # env: TOXENV=lint + # - os: osx + # osx_image: xcode7.3 + # language: generic + # env: TOXENV=py35 BDIST=1 + # - python: 3.5 + # env: TOXENV=py35 BDIST=1 + # dist: precise + # - python: 3.6 + # env: TOXENV=py36 + # - python: 3.6 + # env: TOXENV=individual_coverage + # - language: node_js + # node_js: "node" + # before_install: + # - curl -o- -L https://yarnpkg.com/install.sh | bash + # - export PATH=$HOME/.yarn/bin:$PATH + # install: + # - cd web && yarn + # - yarn global add codecov + # script: npm test && codecov + # cache: + # yarn: true + # directories: + # - web/node_modules + - language: python + env: + - NAME=docs install: - - cd web && yarn - - yarn global add codecov - script: npm test && codecov - cache: - yarn: true - directories: - - web/node_modules + - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + - sudo dpkg -i hugo*.deb + - pip install awscli + script: + - cd docs && ./ci + after_success: + - echo done install: - | diff --git a/docs/ci b/docs/ci new file mode 100755 index 000000000..416738ae0 --- /dev/null +++ b/docs/ci @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script gets run from CI to render and upload docs + +./build + +# Only upload if we have defined credentials - we only have these defined for +# trusted commits (i.e. not PRs). +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "updocs" ]]; then + aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master + # aws cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ + # --paths "/master" +fi diff --git a/docs/upload-master b/docs/upload-master deleted file mode 100755 index c8fcda807..000000000 --- a/docs/upload-master +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -aws configure set preview.cloudfront true -aws --profile mitmproxy \ - s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "/master" From fc23deb9f3c2fff469fa3911d665292de860938a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 10:25:38 +1300 Subject: [PATCH 02/12] release: revamped release upload process --- .travis.yml | 51 ++++---- docs/ci | 2 +- release/ci.py | 250 ++++++++++++++++++++++++++++++++++++++++ release/known_hosts.enc | 1 - release/rtool.pem | 68 ----------- release/rtool.py | 198 +------------------------------ tox.ini | 10 +- 7 files changed, 280 insertions(+), 300 deletions(-) create mode 100755 release/ci.py delete mode 100644 release/known_hosts.enc delete mode 100644 release/rtool.pem diff --git a/.travis.yml b/.travis.yml index 9df9bd15b..2d3c46268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,15 +18,15 @@ matrix: include: # - python: 3.6 # env: TOXENV=lint - # - os: osx - # osx_image: xcode7.3 - # language: generic - # env: TOXENV=py35 BDIST=1 + - os: osx + osx_image: xcode7.3 + language: generic + env: TOXENV=py36 BDIST=1 # - python: 3.5 - # env: TOXENV=py35 BDIST=1 + # env: TOXENV=py35 # dist: precise # - python: 3.6 - # env: TOXENV=py36 + # env: TOXENV=py36 BDIST=1 # - python: 3.6 # env: TOXENV=individual_coverage # - language: node_js @@ -42,17 +42,17 @@ matrix: # yarn: true # directories: # - web/node_modules - - language: python - env: - - NAME=docs - install: - - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb - - sudo dpkg -i hugo*.deb - - pip install awscli - script: - - cd docs && ./ci - after_success: - - echo done + # - language: python + # env: + # - NAME=docs + # install: + # - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + # - sudo dpkg -i hugo*.deb + # - pip install awscli + # script: + # - cd docs && ./ci + # after_success: + # - echo done install: - | @@ -61,12 +61,11 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3 - pyenv global 3.5.3 - pyenv shell 3.5.3 - pip install -U pip setuptools wheel virtualenv + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4 + pyenv global 3.6.4 + pyenv shell 3.6.4 fi - - pip install tox + - pip install tox virtualenv setuptools script: - tox -- --verbose --cov-report=term @@ -74,16 +73,14 @@ script: if [[ $BDIST == "1" ]] then git fetch --unshallow --tags - tox -e rtool -- bdist + tox -e cibuild -- build fi after_success: - # we build binaries on every run, but we only upload them for master snapshots or tags. - | - if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" ]] then - tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts - tox -e rtool -- upload-snapshot --bdist + tox -e cibuild -- upload fi notifications: diff --git a/docs/ci b/docs/ci index 416738ae0..94157116f 100755 --- a/docs/ci +++ b/docs/ci @@ -6,7 +6,7 @@ # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). -if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "updocs" ]]; then +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master # aws cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ # --paths "/master" diff --git a/release/ci.py b/release/ci.py new file mode 100755 index 000000000..5d9678de3 --- /dev/null +++ b/release/ci.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 + +import contextlib +import os +import platform +import re +import sys +import shutil +import subprocess +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename + +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] +else: + VENV_BIN = "bin" + PYINSTALLER_ARGS = [] + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") + +BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +# PyInstaller 3.2 does not bundle pydivert's Windivert binaries +PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + +VENV_DIR = join(BUILD_DIR, "venv") + +# Project Configuration +SNAPSHOT_BRANCHES = ["master", "updocs"] +VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] + +if os.environ.get("TRAVIS_TAG", None): + VERSION = os.environ["TRAVIS_TAG"] +elif os.environ.get("TRAVIS_BRANCH", None) in SNAPSHOT_BRANCHES: + VERSION = os.environ["TRAVIS_BRANCH"] +else: + print("Branch %s is not buildabranch - exiting." % os.environ.get("TRAVIS_BRANCH", None)) + sys.exit(0) + +print("BUILD VERSION=%s" % VERSION) + + +def set_version(dev: bool) -> None: + """ + Update version information in mitmproxy's version.py to either include hardcoded information or not. + """ + version = get_version(dev, dev) + with open(VERSION_FILE, "r") as f: + content = f.read() + content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) + with open(VERSION_FILE, "w") as f: + f.write(content) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=VERSION, + platform=PLATFORM_TAG, + ext=ext + ) + + +def wheel_name() -> str: + return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) + + +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=VERSION, + platform=PLATFORM_TAG, + ext=ext, + ) + + +@contextlib.contextmanager +def chdir(path: str): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + +@cli.command("info") +def info(): + print("Version: %s" % VERSION) + + +@cli.command("wheel") +def make_wheel(): + """ + Build a Python wheel + """ + set_version(True) + try: + subprocess.check_call([ + "tox", "-e", "wheel", + ], env={ + **os.environ, + "VERSION": VERSION, + }) + finally: + set_version(False) + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + os.makedirs(DIST_DIR, exist_ok=True) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + print("> %s --version" % executable) + print(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) + + +@cli.command("upload") +def upload(): + """ + Upload snapshot to snapshot server + """ + subprocess.check_call( + [ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % VERSION, + "--recursive", + ] + ) + + +if __name__ == "__main__": + cli() diff --git a/release/known_hosts.enc b/release/known_hosts.enc deleted file mode 100644 index 585ee678e..000000000 --- a/release/known_hosts.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7 \ No newline at end of file diff --git a/release/rtool.pem b/release/rtool.pem deleted file mode 100644 index 097dff1a9..000000000 --- a/release/rtool.pem +++ /dev/null @@ -1,68 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB - -0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y -byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU -V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z -T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh -PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl -w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD -P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt -JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE -yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km -cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe -hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8 -jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx -QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey -OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL -kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR -AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc -sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko -SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz -5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P -cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk -kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/ -Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9 -3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4 -mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t -8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j -FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf -EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP -a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR -AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+ -m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P -bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q -QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx -uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq -yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K -u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK -MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0 -8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq -y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz -G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi -o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq -2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL -5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp -30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac -7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA -1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8 -eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB -bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO -VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+ -2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175 ------END RSA PRIVATE KEY----- ------BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y -qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe -24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX -FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ -0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI -KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt -0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+ -tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ -TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ -q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J -6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD -5I1jQDXFbjAgAnaSyqyW368CAwEAAQ== ------END PUBLIC KEY----- \ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 25f37e61d..0eda7e6c3 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -8,14 +8,12 @@ import platform import re import runpy import shlex -import shutil import subprocess import tarfile import zipfile -from os.path import join, abspath, dirname, exists, basename +from os.path import join, abspath, dirname import click -import cryptography.fernet import pysftp # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -86,18 +84,6 @@ def get_version(dev: bool = False, build: bool = False) -> str: return x["get_version"](dev, build, True) -def set_version(dev: bool) -> None: - """ - Update version information in mitmproxy's version.py to either include hardcoded information or not. - """ - version = get_version(dev, dev) - with open(VERSION_FILE, "r") as f: - content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) - with open(VERSION_FILE, "w") as f: - f.write(content) - - def archive_name(bdist: str) -> str: if platform.system() == "Windows": ext = "zip" @@ -146,24 +132,6 @@ def cli(): pass -@cli.command("encrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def encrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.encrypt(infile.read())) - - -@cli.command("decrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.decrypt(infile.read())) - - @cli.command("contributors") def contributors(): """ @@ -176,97 +144,6 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("wheel") -def make_wheel(): - """ - Build a Python wheel - """ - set_version(True) - try: - subprocess.check_call([ - "tox", "-e", "wheel", - ], env={ - **os.environ, - "VERSION": get_version(True), - }) - finally: - set_version(False) - - -@cli.command("bdist") -def make_bdist(): - """ - Build a binary distribution - """ - if exists(PYINSTALLER_TEMP): - shutil.rmtree(PYINSTALLER_TEMP) - if exists(PYINSTALLER_DIST): - shutil.rmtree(PYINSTALLER_DIST) - - os.makedirs(DIST_DIR, exist_ok=True) - - for bdist, tools in sorted(BDISTS.items()): - with Archive(join(DIST_DIR, archive_name(bdist))) as archive: - for tool in tools: - # We can't have a folder and a file with the same name. - if tool == "mitmproxy": - tool = "mitmproxy_main" - # This is PyInstaller, so it messes up paths. - # We need to make sure that we are in the spec folder. - with chdir(PYINSTALLER_SPEC): - print("Building %s binary..." % tool) - excludes = [] - if tool != "mitmweb": - excludes.append("mitmproxy.tools.web") - if tool != "mitmproxy_main": - excludes.append("mitmproxy.tools.console") - - # Overwrite mitmproxy/version.py to include commit info - set_version(True) - try: - subprocess.check_call( - [ - "pyinstaller", - "--clean", - "--workpath", PYINSTALLER_TEMP, - "--distpath", PYINSTALLER_DIST, - "--additional-hooks-dir", PYINSTALLER_HOOKS, - "--onefile", - "--console", - "--icon", "icon.ico", - # This is PyInstaller, so setting a - # different log level obviously breaks it :-) - # "--log-level", "WARN", - ] - + [x for e in excludes for x in ["--exclude-module", e]] - + PYINSTALLER_ARGS - + [tool] - ) - finally: - set_version(False) - # Delete the spec file - we're good without. - os.remove("{}.spec".format(tool)) - - # Test if it works at all O:-) - executable = join(PYINSTALLER_DIST, tool) - if platform.system() == "Windows": - executable += ".exe" - - # Remove _main suffix from mitmproxy executable - if "_main" in executable: - shutil.move( - executable, - executable.replace("_main", "") - ) - executable = executable.replace("_main", "") - - print("> %s --version" % executable) - print(subprocess.check_output([executable, "--version"]).decode()) - - archive.add(executable, basename(executable)) - print("Packed {}.".format(archive_name(bdist))) - - @cli.command("upload-release") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @@ -305,78 +182,5 @@ def homebrew_pr(): ]) -@cli.command("upload-snapshot") -@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) -@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) -@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) -@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) -@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) -@click.option("--wheel/--no-wheel", default=False) -@click.option("--bdist/--no-bdist", default=False) -@click.option("--installer/--no-installer", default=False) -def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): - """ - Upload snapshot to snapshot server - """ - cnopts = pysftp.CnOpts( - knownhosts=join(RELEASE_DIR, 'known_hosts') - ) - with pysftp.Connection(host=host, - port=port, - username=user, - private_key=private_key, - private_key_pass=private_key_password, - cnopts=cnopts) as sftp: - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - if bdist: - for bdist in sorted(BDISTS.keys()): - files.append(archive_name(bdist)) - if installer: - files.append(installer_name()) - - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = re.sub( - r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), - get_version(True, True), - f - ) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - # We hide the file during upload - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Show new version - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - if f != wheel_name(): - # "latest" isn't a proper wheel version, so this could not be installed. - # https://github.com/mitmproxy/mitmproxy/issues/1065 - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) - - if __name__ == "__main__": cli() diff --git a/tox.ini b/tox.ini index e90455c1c..4b009851a 100644 --- a/tox.ini +++ b/tox.ini @@ -45,14 +45,12 @@ commands = pathod --version pathoc --version -[testenv:rtool] -passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY +[testenv:cibuild] +passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY deps = -rrequirements.txt pyinstaller==3.3.1 - twine==1.9.1 - pysftp==0.2.9 - + awscli commands = mitmdump --version - python ./release/rtool.py {posargs} + python ./release/ci.py {posargs} From 538a67dc3afe6272b045d37766b1d9f0a5d5044e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 11:50:47 +1300 Subject: [PATCH 03/12] release: cleanup and re-enable all our Travis tests Wheels are yet to come. --- .travis.yml | 66 ++++++++++++++++++++++++------------------------ release/rtool.py | 3 --- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d3c46268..23910601b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,43 +16,43 @@ git: matrix: fast_finish: true include: - # - python: 3.6 - # env: TOXENV=lint + - python: 3.6 + env: TOXENV=lint - os: osx osx_image: xcode7.3 language: generic env: TOXENV=py36 BDIST=1 - # - python: 3.5 - # env: TOXENV=py35 - # dist: precise - # - python: 3.6 - # env: TOXENV=py36 BDIST=1 - # - python: 3.6 - # env: TOXENV=individual_coverage - # - language: node_js - # node_js: "node" - # before_install: - # - curl -o- -L https://yarnpkg.com/install.sh | bash - # - export PATH=$HOME/.yarn/bin:$PATH - # install: - # - cd web && yarn - # - yarn global add codecov - # script: npm test && codecov - # cache: - # yarn: true - # directories: - # - web/node_modules - # - language: python - # env: - # - NAME=docs - # install: - # - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb - # - sudo dpkg -i hugo*.deb - # - pip install awscli - # script: - # - cd docs && ./ci - # after_success: - # - echo done + - python: 3.5 + env: TOXENV=py35 + dist: precise + - python: 3.6 + env: TOXENV=py36 BDIST=1 + - python: 3.6 + env: TOXENV=individual_coverage + - language: node_js + node_js: "node" + before_install: + - curl -o- -L https://yarnpkg.com/install.sh | bash + - export PATH=$HOME/.yarn/bin:$PATH + install: + - cd web && yarn + - yarn global add codecov + script: npm test && codecov + cache: + yarn: true + directories: + - web/node_modules + - language: python + env: + - NAME=docs + install: + - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + - sudo dpkg -i hugo*.deb + - pip install awscli + script: + - cd docs && ./ci + after_success: + - echo done install: - | diff --git a/release/rtool.py b/release/rtool.py index 0eda7e6c3..a3f499143 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 import contextlib -import fnmatch import os import sys import platform -import re import runpy import shlex import subprocess @@ -14,7 +12,6 @@ import zipfile from os.path import join, abspath, dirname import click -import pysftp # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ From 780ff05f12cda4665b37dd7023985f0d643da6bb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 12:39:32 +1300 Subject: [PATCH 04/12] release: satisfy lint --- release/ci.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/release/ci.py b/release/ci.py index 5d9678de3..666b921af 100755 --- a/release/ci.py +++ b/release/ci.py @@ -3,7 +3,6 @@ import contextlib import os import platform -import re import sys import shutil import subprocess @@ -82,18 +81,6 @@ else: print("BUILD VERSION=%s" % VERSION) -def set_version(dev: bool) -> None: - """ - Update version information in mitmproxy's version.py to either include hardcoded information or not. - """ - version = get_version(dev, dev) - with open(VERSION_FILE, "r") as f: - content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) - with open(VERSION_FILE, "w") as f: - f.write(content) - - def archive_name(bdist: str) -> str: if platform.system() == "Windows": ext = "zip" @@ -139,28 +126,12 @@ def cli(): """ pass + @cli.command("info") def info(): print("Version: %s" % VERSION) -@cli.command("wheel") -def make_wheel(): - """ - Build a Python wheel - """ - set_version(True) - try: - subprocess.check_call([ - "tox", "-e", "wheel", - ], env={ - **os.environ, - "VERSION": VERSION, - }) - finally: - set_version(False) - - @cli.command("build") def build(): """ From f8cce32562009427b99e30eecb323babeb371be3 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 14:05:26 +1300 Subject: [PATCH 05/12] release: shift appveyor to new process In the process also move to InstallBuilder 18.2 --- .appveyor.yml | 45 ++++++++------------------------------------- release/ci.py | 42 ++++++++++++++++++++++++++++-------------- release/rtool.py | 10 ++++++++++ tox.ini | 2 +- 4 files changed, 47 insertions(+), 52 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 6891f1b32..90364823d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,22 +9,8 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python35" - TOXENV: "py35" - # TODO: ENABLE WHEN AVAILABLE - # - PYTHON: "C:\\Python36" - # TOXENV: "py36" - - SNAPSHOT_HOST: - secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= - SNAPSHOT_PORT: - secure: TiJPtg60/edYTH8RnoBErg== - SNAPSHOT_USER: - secure: 6yBwmO5gv4vAwoFYII8qjQ== - SNAPSHOT_PASS: - secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV - RTOOL_KEY: - secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM + - PYTHON: "C:\\Python36" + TOXENV: "py36" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -36,17 +22,10 @@ test_script: - ps: | $Env:VERSION = $(python -m mitmproxy.version) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" - tox -e rtool -- wheel - tox -e rtool -- bdist - + tox -e cibuild -- build - ps: | - if( - ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - echo "Decrypt license..." - tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml - $ibVersion = "17.12.0" + if($Env:RTOOL_KEY) { + $ibVersion = "18.2.0" $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe" $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe" if (!(Test-Path $ibSetup)) { @@ -58,8 +37,8 @@ test_script: } echo "Install InstallBuilder..." Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait - # Wait until executable exists - no idea why this is necessary. - while (!(Test-Path $ibCli)) { Start-Sleep 0.1 } + echo "Decrypt license..." + tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml echo "Run InstallBuilder..." &$ibCli ` build ` @@ -73,15 +52,7 @@ test_script: } deploy_script: - # we build binaries on every run, but we only upload them for master snapshots or tags. - ps: | - if( - ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts - tox -e rtool -- upload-snapshot --bdist --wheel --installer - } + ps: tox -e cibuild -- upload cache: - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml diff --git a/release/ci.py b/release/ci.py index 666b921af..18209dfdd 100755 --- a/release/ci.py +++ b/release/ci.py @@ -10,6 +10,7 @@ import tarfile import zipfile from os.path import join, abspath, dirname, exists, basename +import cryptography.fernet import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -70,14 +71,17 @@ TOOLS = [ for tool in tools ] -if os.environ.get("TRAVIS_TAG", None): - VERSION = os.environ["TRAVIS_TAG"] -elif os.environ.get("TRAVIS_BRANCH", None) in SNAPSHOT_BRANCHES: - VERSION = os.environ["TRAVIS_BRANCH"] +TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) +BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) +if TAG: + VERSION = TAG +elif BRANCH in SNAPSHOT_BRANCHES: + VERSION = BRANCH else: - print("Branch %s is not buildabranch - exiting." % os.environ.get("TRAVIS_BRANCH", None)) + print("Branch %s is not build branch - exiting." % BRANCH) sys.exit(0) + print("BUILD VERSION=%s" % VERSION) @@ -206,15 +210,25 @@ def upload(): """ Upload snapshot to snapshot server """ - subprocess.check_call( - [ - "aws", "s3", "cp", - "--acl", "public-read", - DIST_DIR + "/", - "s3://snapshots.mitmproxy.org/%s/" % VERSION, - "--recursive", - ] - ) + if "AWS_ACCESS_KEY_ID" in os.environ: + subprocess.check_call( + [ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % VERSION, + "--recursive", + ] + ) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) if __name__ == "__main__": diff --git a/release/rtool.py b/release/rtool.py index a3f499143..2adc10962 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -11,6 +11,7 @@ import tarfile import zipfile from os.path import join, abspath, dirname +import cryptography.fernet import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -179,5 +180,14 @@ def homebrew_pr(): ]) +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) + + if __name__ == "__main__": cli() diff --git a/tox.ini b/tox.ini index 4b009851a..0033b8e8f 100644 --- a/tox.ini +++ b/tox.ini @@ -46,7 +46,7 @@ commands = pathoc --version [testenv:cibuild] -passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY +passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APPVEYOR_REPO_TAG_NAME APPVEYOR_REPO_TAG APPVEYOR_REPO_BRANCH RTOOL_KEY deps = -rrequirements.txt pyinstaller==3.3.1 From f72d91672e1ab7a374177d1063f90742041cb5f5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 21:22:33 +1300 Subject: [PATCH 06/12] release: build wheel --- .travis.yml | 2 +- release/ci.py | 10 ++++++++++ tox.ini | 15 +-------------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23910601b..66a4f61e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: env: TOXENV=py35 dist: precise - python: 3.6 - env: TOXENV=py36 BDIST=1 + env: TOXENV=py36 BDIST=1 WHEEL=1 - python: 3.6 env: TOXENV=individual_coverage - language: node_js diff --git a/release/ci.py b/release/ci.py index 18209dfdd..a7ecfae5a 100755 --- a/release/ci.py +++ b/release/ci.py @@ -148,6 +148,16 @@ def build(): os.makedirs(DIST_DIR, exist_ok=True) + if "WHEEL" in os.environ: + print("Building wheel...") + subprocess.check_call( + [ + "python", + "setup.py", "-q", "bdist_wheel", + "--dist-dir", "release/dist", + ] + ) + for bdist, tools in sorted(BDISTS.items()): with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: diff --git a/tox.ini b/tox.ini index 0033b8e8f..c9a575da9 100644 --- a/tox.ini +++ b/tox.ini @@ -32,21 +32,8 @@ deps = commands = python test/individual_coverage.py -[testenv:wheel] -recreate = True -deps = -commands = - python setup.py -q bdist_wheel --dist-dir release/dist - pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl - # skip `mitmproxy --version` if SKIP_MITMPROXY is defined. - {env:SKIP_MITMPROXY:mitmproxy --version} - mitmdump --version - mitmweb --version - pathod --version - pathoc --version - [testenv:cibuild] -passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APPVEYOR_REPO_TAG_NAME APPVEYOR_REPO_TAG APPVEYOR_REPO_BRANCH RTOOL_KEY +passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APPVEYOR_REPO_TAG_NAME APPVEYOR_REPO_TAG APPVEYOR_REPO_BRANCH RTOOL_KEY WHEEL deps = -rrequirements.txt pyinstaller==3.3.1 From 760882a3d1bf0ad6420bf10cdd24b83bfdf18a02 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 22:45:08 +1300 Subject: [PATCH 07/12] docs: cloudfront invalidation --- docs/ci | 4 ++-- docs/upload-archive | 6 +++--- docs/upload-stable | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/ci b/docs/ci index 94157116f..1584c5e18 100755 --- a/docs/ci +++ b/docs/ci @@ -8,6 +8,6 @@ # trusted commits (i.e. not PRs). if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master - # aws cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ - # --paths "/master" + aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/master" fi diff --git a/docs/upload-archive b/docs/upload-archive index 5622a2aea..86dd248e5 100755 --- a/docs/upload-archive +++ b/docs/upload-archive @@ -12,6 +12,6 @@ SPATH="/archive/$1" aws configure set preview.cloudfront true aws --profile mitmproxy \ s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "$SPATH" +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "$SPATH" diff --git a/docs/upload-stable b/docs/upload-stable index 7c658113f..c2c1267e7 100755 --- a/docs/upload-stable +++ b/docs/upload-stable @@ -3,6 +3,6 @@ aws configure set preview.cloudfront true aws --profile mitmproxy \ s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "/master" +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/stable" From be089e53cfef04656807e8e1a95c5ae5cd9fd58c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 7 Mar 2018 22:50:07 +1300 Subject: [PATCH 08/12] release: clean rtool --- release/rtool.py | 79 ------------------------------------------------ 1 file changed, 79 deletions(-) diff --git a/release/rtool.py b/release/rtool.py index 2adc10962..0009cf89e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -7,69 +7,16 @@ import platform import runpy import shlex import subprocess -import tarfile -import zipfile from os.path import join, abspath, dirname import cryptography.fernet import click -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if platform.system() == "Windows": - VENV_BIN = "Scripts" - PYINSTALLER_ARGS = [ - # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. - "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", - ] -else: - VENV_BIN = "bin" - PYINSTALLER_ARGS = [] - -# ZipFile and tarfile have slightly different APIs. Fix that. -if platform.system() == "Windows": - def Archive(name): - a = zipfile.ZipFile(name, "w") - a.add = a.write - return a -else: - def Archive(name): - return tarfile.open(name, "w:gz") - -PLATFORM_TAG = { - "Darwin": "osx", - "Windows": "windows", - "Linux": "linux", -}.get(platform.system(), platform.system()) ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") - -BUILD_DIR = join(RELEASE_DIR, "build") DIST_DIR = join(RELEASE_DIR, "dist") - -PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") -# PyInstaller 3.2 does not bundle pydivert's Windivert binaries -PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") -PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") -PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) - -VENV_DIR = join(BUILD_DIR, "venv") - -# Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") -BDISTS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] -} -if platform.system() == "Windows": - BDISTS["mitmproxy"].remove("mitmproxy") - -TOOLS = [ - tool - for tools in sorted(BDISTS.values()) - for tool in tools -] def git(args: str) -> str: @@ -82,38 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str: return x["get_version"](dev, build, True) -def archive_name(bdist: str) -> str: - if platform.system() == "Windows": - ext = "zip" - else: - ext = "tar.gz" - return "{project}-{version}-{platform}.{ext}".format( - project=bdist, - version=get_version(), - platform=PLATFORM_TAG, - ext=ext - ) - - def wheel_name() -> str: return "mitmproxy-{version}-py3-none-any.whl".format( version=get_version(True), ) -def installer_name() -> str: - ext = { - "Windows": "exe", - "Darwin": "dmg", - "Linux": "run" - }[platform.system()] - return "mitmproxy-{version}-{platform}-installer.{ext}".format( - version=get_version(), - platform=PLATFORM_TAG, - ext=ext, - ) - - @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() From 380e4bc14d2d37df5af3be53abf2e9b8be79efcb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 8 Mar 2018 07:55:52 +1300 Subject: [PATCH 09/12] release: don't upload assets for PRs from the main repo --- release/ci.py | 15 +++++++++++++++ tox.ini | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/release/ci.py b/release/ci.py index a7ecfae5a..53a2632fb 100755 --- a/release/ci.py +++ b/release/ci.py @@ -215,11 +215,26 @@ def build(): print("Packed {}.".format(archive_name(bdist))) +def is_pr(): + if os.environ.get("TRAVIS_PULL_REQUEST") != "false": + return True + elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"): + return True + return False + + @cli.command("upload") def upload(): """ Upload snapshot to snapshot server """ + # This requires some explanation. The AWS access keys are only exposed to + # privileged builds - that is, they are not available to PRs from forks. + # However, they ARE exposed to PRs from a branch within the main repo. This + # check catches that corner case, and prevents an inadvertent upload. + if is_pr(): + print("Refusing to upload a pull request") + return if "AWS_ACCESS_KEY_ID" in os.environ: subprocess.check_call( [ diff --git a/tox.ini b/tox.ini index c9a575da9..2d0fd0470 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ commands = python test/individual_coverage.py [testenv:cibuild] -passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY APPVEYOR_REPO_TAG_NAME APPVEYOR_REPO_TAG APPVEYOR_REPO_BRANCH RTOOL_KEY WHEEL +passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps = -rrequirements.txt pyinstaller==3.3.1 From 5cea2aacb233f2c3144bd8c42d64161ad3241f3b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 8 Mar 2018 08:09:01 +1300 Subject: [PATCH 10/12] release: build snapshots for all branches on the main repo --- release/ci.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/release/ci.py b/release/ci.py index 53a2632fb..43fde3dcb 100755 --- a/release/ci.py +++ b/release/ci.py @@ -56,7 +56,6 @@ PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) VENV_DIR = join(BUILD_DIR, "venv") # Project Configuration -SNAPSHOT_BRANCHES = ["master", "updocs"] VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") BDISTS = { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], @@ -75,10 +74,10 @@ TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) if TAG: VERSION = TAG -elif BRANCH in SNAPSHOT_BRANCHES: +elif BRANCH: VERSION = BRANCH else: - print("Branch %s is not build branch - exiting." % BRANCH) + print("Could not establish build name - exiting." % BRANCH) sys.exit(0) From 3a3299bc612218d5073f423a587f9d917868aeec Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 8 Mar 2018 08:18:26 +1300 Subject: [PATCH 11/12] release: put branches in /branches on the snapshots server This is a precautionary measure to prevent a clash in case we have a branch with the same name as a tag. Tags live in the top-level directory, because these are releases that users will most often access directly. --- release/ci.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/ci.py b/release/ci.py index 43fde3dcb..eaabaf3a0 100755 --- a/release/ci.py +++ b/release/ci.py @@ -74,8 +74,10 @@ TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) if TAG: VERSION = TAG + UPLOAD_DIR = VERSION elif BRANCH: VERSION = BRANCH + UPLOAD_DIR = "branches/%s" % VERSION else: print("Could not establish build name - exiting." % BRANCH) sys.exit(0) @@ -240,7 +242,7 @@ def upload(): "aws", "s3", "cp", "--acl", "public-read", DIST_DIR + "/", - "s3://snapshots.mitmproxy.org/%s/" % VERSION, + "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, "--recursive", ] ) From 0debcf5bf92b71e58ad723629e35b7f5a3393656 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 8 Mar 2018 09:52:05 +1300 Subject: [PATCH 12/12] release: fix PR detection for appveyor --- release/ci.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/ci.py b/release/ci.py index eaabaf3a0..94b1f13d5 100755 --- a/release/ci.py +++ b/release/ci.py @@ -217,7 +217,9 @@ def build(): def is_pr(): - if os.environ.get("TRAVIS_PULL_REQUEST") != "false": + if "TRAVIS_PULL_REQUEST" in os.environ: + if os.environ["TRAVIS_PULL_REQUEST"] == "false": + return False return True elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"): return True