diff --git a/.appveyor.yml b/.appveyor.yml index 6891f1b32..90364823d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,22 +9,8 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python35" - TOXENV: "py35" - # TODO: ENABLE WHEN AVAILABLE - # - PYTHON: "C:\\Python36" - # TOXENV: "py36" - - SNAPSHOT_HOST: - secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= - SNAPSHOT_PORT: - secure: TiJPtg60/edYTH8RnoBErg== - SNAPSHOT_USER: - secure: 6yBwmO5gv4vAwoFYII8qjQ== - SNAPSHOT_PASS: - secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV - RTOOL_KEY: - secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM + - PYTHON: "C:\\Python36" + TOXENV: "py36" install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -36,17 +22,10 @@ test_script: - ps: | $Env:VERSION = $(python -m mitmproxy.version) $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" - tox -e rtool -- wheel - tox -e rtool -- bdist - + tox -e cibuild -- build - ps: | - if( - ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - echo "Decrypt license..." - tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml - $ibVersion = "17.12.0" + if($Env:RTOOL_KEY) { + $ibVersion = "18.2.0" $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe" $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe" if (!(Test-Path $ibSetup)) { @@ -58,8 +37,8 @@ test_script: } echo "Install InstallBuilder..." Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait - # Wait until executable exists - no idea why this is necessary. - while (!(Test-Path $ibCli)) { Start-Sleep 0.1 } + echo "Decrypt license..." + tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml echo "Run InstallBuilder..." &$ibCli ` build ` @@ -73,15 +52,7 @@ test_script: } deploy_script: - # we build binaries on every run, but we only upload them for master snapshots or tags. - ps: | - if( - ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts - tox -e rtool -- upload-snapshot --bdist --wheel --installer - } + ps: tox -e cibuild -- upload cache: - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml diff --git a/.travis.yml b/.travis.yml index c22038fd8..66a4f61e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,12 +21,12 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 BDIST=1 + env: TOXENV=py36 BDIST=1 - python: 3.5 - env: TOXENV=py35 BDIST=1 + env: TOXENV=py35 dist: precise - python: 3.6 - env: TOXENV=py36 + env: TOXENV=py36 BDIST=1 WHEEL=1 - python: 3.6 env: TOXENV=individual_coverage - language: node_js @@ -42,6 +42,17 @@ matrix: yarn: true directories: - web/node_modules + - language: python + env: + - NAME=docs + install: + - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb + - sudo dpkg -i hugo*.deb + - pip install awscli + script: + - cd docs && ./ci + after_success: + - echo done install: - | @@ -50,12 +61,11 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3 - pyenv global 3.5.3 - pyenv shell 3.5.3 - pip install -U pip setuptools wheel virtualenv + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4 + pyenv global 3.6.4 + pyenv shell 3.6.4 fi - - pip install tox + - pip install tox virtualenv setuptools script: - tox -- --verbose --cov-report=term @@ -63,16 +73,14 @@ script: if [[ $BDIST == "1" ]] then git fetch --unshallow --tags - tox -e rtool -- bdist + tox -e cibuild -- build fi after_success: - # we build binaries on every run, but we only upload them for master snapshots or tags. - | - if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" ]] then - tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts - tox -e rtool -- upload-snapshot --bdist + tox -e cibuild -- upload fi notifications: diff --git a/docs/ci b/docs/ci new file mode 100755 index 000000000..1584c5e18 --- /dev/null +++ b/docs/ci @@ -0,0 +1,13 @@ +#!/bin/bash + +# This script gets run from CI to render and upload docs + +./build + +# Only upload if we have defined credentials - we only have these defined for +# trusted commits (i.e. not PRs). +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then + aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master + aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/master" +fi diff --git a/docs/upload-archive b/docs/upload-archive index 5622a2aea..86dd248e5 100755 --- a/docs/upload-archive +++ b/docs/upload-archive @@ -12,6 +12,6 @@ SPATH="/archive/$1" aws configure set preview.cloudfront true aws --profile mitmproxy \ s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "$SPATH" +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "$SPATH" diff --git a/docs/upload-master b/docs/upload-master deleted file mode 100755 index c8fcda807..000000000 --- a/docs/upload-master +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -aws configure set preview.cloudfront true -aws --profile mitmproxy \ - s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "/master" diff --git a/docs/upload-stable b/docs/upload-stable index 7c658113f..c2c1267e7 100755 --- a/docs/upload-stable +++ b/docs/upload-stable @@ -3,6 +3,6 @@ aws configure set preview.cloudfront true aws --profile mitmproxy \ s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable -# aws --profile mitmproxy \ -# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \ -# --paths "/master" +aws --profile mitmproxy \ + cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ + --paths "/stable" diff --git a/release/ci.py b/release/ci.py new file mode 100755 index 000000000..94b1f13d5 --- /dev/null +++ b/release/ci.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 + +import contextlib +import os +import platform +import sys +import shutil +import subprocess +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename + +import cryptography.fernet +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] +else: + VENV_BIN = "bin" + PYINSTALLER_ARGS = [] + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") + +BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +# PyInstaller 3.2 does not bundle pydivert's Windivert binaries +PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + +VENV_DIR = join(BUILD_DIR, "venv") + +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] + +TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) +BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) +if TAG: + VERSION = TAG + UPLOAD_DIR = VERSION +elif BRANCH: + VERSION = BRANCH + UPLOAD_DIR = "branches/%s" % VERSION +else: + print("Could not establish build name - exiting." % BRANCH) + sys.exit(0) + + +print("BUILD VERSION=%s" % VERSION) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=VERSION, + platform=PLATFORM_TAG, + ext=ext + ) + + +def wheel_name() -> str: + return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) + + +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=VERSION, + platform=PLATFORM_TAG, + ext=ext, + ) + + +@contextlib.contextmanager +def chdir(path: str): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("info") +def info(): + print("Version: %s" % VERSION) + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + os.makedirs(DIST_DIR, exist_ok=True) + + if "WHEEL" in os.environ: + print("Building wheel...") + subprocess.check_call( + [ + "python", + "setup.py", "-q", "bdist_wheel", + "--dist-dir", "release/dist", + ] + ) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + print("> %s --version" % executable) + print(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) + + +def is_pr(): + if "TRAVIS_PULL_REQUEST" in os.environ: + if os.environ["TRAVIS_PULL_REQUEST"] == "false": + return False + return True + elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"): + return True + return False + + +@cli.command("upload") +def upload(): + """ + Upload snapshot to snapshot server + """ + # This requires some explanation. The AWS access keys are only exposed to + # privileged builds - that is, they are not available to PRs from forks. + # However, they ARE exposed to PRs from a branch within the main repo. This + # check catches that corner case, and prevents an inadvertent upload. + if is_pr(): + print("Refusing to upload a pull request") + return + if "AWS_ACCESS_KEY_ID" in os.environ: + subprocess.check_call( + [ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, + "--recursive", + ] + ) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + +if __name__ == "__main__": + cli() diff --git a/release/known_hosts.enc b/release/known_hosts.enc deleted file mode 100644 index 585ee678e..000000000 --- a/release/known_hosts.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7 \ No newline at end of file diff --git a/release/rtool.pem b/release/rtool.pem deleted file mode 100644 index 097dff1a9..000000000 --- a/release/rtool.pem +++ /dev/null @@ -1,68 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB - -0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y -byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU -V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z -T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh -PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl -w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD -P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt -JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE -yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km -cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe -hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8 -jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx -QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey -OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL -kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR -AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc -sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko -SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz -5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P -cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk -kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/ -Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9 -3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4 -mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t -8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j -FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf -EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP -a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR -AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+ -m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P -bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q -QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx -uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq -yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K -u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK -MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0 -8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq -y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz -G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi -o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq -2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL -5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp -30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac -7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA -1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8 -eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB -bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO -VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+ -2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175 ------END RSA PRIVATE KEY----- ------BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y -qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe -24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX -FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ -0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI -KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt -0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+ -tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ -TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ -q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J -6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD -5I1jQDXFbjAgAnaSyqyW368CAwEAAQ== ------END PUBLIC KEY----- \ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 25f37e61d..0009cf89e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,79 +1,22 @@ #!/usr/bin/env python3 import contextlib -import fnmatch import os import sys import platform -import re import runpy import shlex -import shutil import subprocess -import tarfile -import zipfile -from os.path import join, abspath, dirname, exists, basename +from os.path import join, abspath, dirname -import click import cryptography.fernet -import pysftp +import click -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if platform.system() == "Windows": - VENV_BIN = "Scripts" - PYINSTALLER_ARGS = [ - # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. - "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", - ] -else: - VENV_BIN = "bin" - PYINSTALLER_ARGS = [] - -# ZipFile and tarfile have slightly different APIs. Fix that. -if platform.system() == "Windows": - def Archive(name): - a = zipfile.ZipFile(name, "w") - a.add = a.write - return a -else: - def Archive(name): - return tarfile.open(name, "w:gz") - -PLATFORM_TAG = { - "Darwin": "osx", - "Windows": "windows", - "Linux": "linux", -}.get(platform.system(), platform.system()) ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") - -BUILD_DIR = join(RELEASE_DIR, "build") DIST_DIR = join(RELEASE_DIR, "dist") - -PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") -# PyInstaller 3.2 does not bundle pydivert's Windivert binaries -PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") -PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") -PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) - -VENV_DIR = join(BUILD_DIR, "venv") - -# Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") -BDISTS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] -} -if platform.system() == "Windows": - BDISTS["mitmproxy"].remove("mitmproxy") - -TOOLS = [ - tool - for tools in sorted(BDISTS.values()) - for tool in tools -] def git(args: str) -> str: @@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str: return x["get_version"](dev, build, True) -def set_version(dev: bool) -> None: - """ - Update version information in mitmproxy's version.py to either include hardcoded information or not. - """ - version = get_version(dev, dev) - with open(VERSION_FILE, "r") as f: - content = f.read() - content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M) - with open(VERSION_FILE, "w") as f: - f.write(content) - - -def archive_name(bdist: str) -> str: - if platform.system() == "Windows": - ext = "zip" - else: - ext = "tar.gz" - return "{project}-{version}-{platform}.{ext}".format( - project=bdist, - version=get_version(), - platform=PLATFORM_TAG, - ext=ext - ) - - def wheel_name() -> str: return "mitmproxy-{version}-py3-none-any.whl".format( version=get_version(True), ) -def installer_name() -> str: - ext = { - "Windows": "exe", - "Darwin": "dmg", - "Linux": "run" - }[platform.system()] - return "mitmproxy-{version}-{platform}-installer.{ext}".format( - version=get_version(), - platform=PLATFORM_TAG, - ext=ext, - ) - - @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -146,24 +51,6 @@ def cli(): pass -@cli.command("encrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def encrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.encrypt(infile.read())) - - -@cli.command("decrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.decrypt(infile.read())) - - @cli.command("contributors") def contributors(): """ @@ -176,97 +63,6 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("wheel") -def make_wheel(): - """ - Build a Python wheel - """ - set_version(True) - try: - subprocess.check_call([ - "tox", "-e", "wheel", - ], env={ - **os.environ, - "VERSION": get_version(True), - }) - finally: - set_version(False) - - -@cli.command("bdist") -def make_bdist(): - """ - Build a binary distribution - """ - if exists(PYINSTALLER_TEMP): - shutil.rmtree(PYINSTALLER_TEMP) - if exists(PYINSTALLER_DIST): - shutil.rmtree(PYINSTALLER_DIST) - - os.makedirs(DIST_DIR, exist_ok=True) - - for bdist, tools in sorted(BDISTS.items()): - with Archive(join(DIST_DIR, archive_name(bdist))) as archive: - for tool in tools: - # We can't have a folder and a file with the same name. - if tool == "mitmproxy": - tool = "mitmproxy_main" - # This is PyInstaller, so it messes up paths. - # We need to make sure that we are in the spec folder. - with chdir(PYINSTALLER_SPEC): - print("Building %s binary..." % tool) - excludes = [] - if tool != "mitmweb": - excludes.append("mitmproxy.tools.web") - if tool != "mitmproxy_main": - excludes.append("mitmproxy.tools.console") - - # Overwrite mitmproxy/version.py to include commit info - set_version(True) - try: - subprocess.check_call( - [ - "pyinstaller", - "--clean", - "--workpath", PYINSTALLER_TEMP, - "--distpath", PYINSTALLER_DIST, - "--additional-hooks-dir", PYINSTALLER_HOOKS, - "--onefile", - "--console", - "--icon", "icon.ico", - # This is PyInstaller, so setting a - # different log level obviously breaks it :-) - # "--log-level", "WARN", - ] - + [x for e in excludes for x in ["--exclude-module", e]] - + PYINSTALLER_ARGS - + [tool] - ) - finally: - set_version(False) - # Delete the spec file - we're good without. - os.remove("{}.spec".format(tool)) - - # Test if it works at all O:-) - executable = join(PYINSTALLER_DIST, tool) - if platform.system() == "Windows": - executable += ".exe" - - # Remove _main suffix from mitmproxy executable - if "_main" in executable: - shutil.move( - executable, - executable.replace("_main", "") - ) - executable = executable.replace("_main", "") - - print("> %s --version" % executable) - print(subprocess.check_output([executable, "--version"]).decode()) - - archive.add(executable, basename(executable)) - print("Packed {}.".format(archive_name(bdist))) - - @cli.command("upload-release") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @@ -305,77 +101,13 @@ def homebrew_pr(): ]) -@cli.command("upload-snapshot") -@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) -@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) -@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) -@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) -@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) -@click.option("--wheel/--no-wheel", default=False) -@click.option("--bdist/--no-bdist", default=False) -@click.option("--installer/--no-installer", default=False) -def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): - """ - Upload snapshot to snapshot server - """ - cnopts = pysftp.CnOpts( - knownhosts=join(RELEASE_DIR, 'known_hosts') - ) - with pysftp.Connection(host=host, - port=port, - username=user, - private_key=private_key, - private_key_pass=private_key_password, - cnopts=cnopts) as sftp: - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - if bdist: - for bdist in sorted(BDISTS.keys()): - files.append(archive_name(bdist)) - if installer: - files.append(installer_name()) - - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = re.sub( - r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), - get_version(True, True), - f - ) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - # We hide the file during upload - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Show new version - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - if f != wheel_name(): - # "latest" isn't a proper wheel version, so this could not be installed. - # https://github.com/mitmproxy/mitmproxy/issues/1065 - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index e90455c1c..2d0fd0470 100644 --- a/tox.ini +++ b/tox.ini @@ -32,27 +32,12 @@ deps = commands = python test/individual_coverage.py -[testenv:wheel] -recreate = True -deps = -commands = - python setup.py -q bdist_wheel --dist-dir release/dist - pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl - # skip `mitmproxy --version` if SKIP_MITMPROXY is defined. - {env:SKIP_MITMPROXY:mitmproxy --version} - mitmdump --version - mitmweb --version - pathod --version - pathoc --version - -[testenv:rtool] -passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY +[testenv:cibuild] +passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps = -rrequirements.txt pyinstaller==3.3.1 - twine==1.9.1 - pysftp==0.2.9 - + awscli commands = mitmdump --version - python ./release/rtool.py {posargs} + python ./release/ci.py {posargs}