Merge pull request #2977 from mitmproxy/updocs2

Revamp CI generation of snapshots and documentation

.appveyor.yml

@@ -9,22 +9,8 @@ environment:
   CI_DEPS: codecov>=2.0.5
   CI_COMMANDS: codecov
   matrix:
-    - PYTHON: "C:\\Python35"
+    - PYTHON: "C:\\Python36"
-      TOXENV: "py35"
+      TOXENV: "py36"
-    # TODO: ENABLE WHEN AVAILABLE
-    # - PYTHON: "C:\\Python36"
-    #   TOXENV: "py36"
-
-  SNAPSHOT_HOST:
-    secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
-  SNAPSHOT_PORT:
-    secure: TiJPtg60/edYTH8RnoBErg==
-  SNAPSHOT_USER:
-    secure: 6yBwmO5gv4vAwoFYII8qjQ==
-  SNAPSHOT_PASS:
-    secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
-  RTOOL_KEY:
-    secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -36,17 +22,10 @@ test_script:
   - ps: |
       $Env:VERSION = $(python -m mitmproxy.version)
       $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
-      tox -e rtool -- wheel
+      tox -e cibuild -- build
-      tox -e rtool -- bdist
-
   - ps: |
-      if(
+      if($Env:RTOOL_KEY) {
-        ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
+        $ibVersion = "18.2.0"
-        (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-      ) {
-        echo "Decrypt license..."
-        tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
-        $ibVersion = "17.12.0"
         $ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe"
         $ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe"
         if (!(Test-Path $ibSetup)) {
@@ -58,8 +37,8 @@ test_script:
         }
         echo "Install InstallBuilder..."
         Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait
-        # Wait until executable exists - no idea why this is necessary.
+        echo "Decrypt license..."
-        while (!(Test-Path $ibCli)) { Start-Sleep 0.1 }
+        tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
         echo "Run InstallBuilder..."
         &$ibCli `
           build `
@@ -73,15 +52,7 @@ test_script:
       }
 
 deploy_script:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
+  ps: tox -e cibuild -- upload
-  ps: |
-    if(
-      ($Env:TOXENV -match "py35") -and
-      (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
-    ) {
-      tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts
-      tox -e rtool -- upload-snapshot --bdist --wheel --installer
-    }
 
 cache:
   - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml

.travis.yml

@@ -21,12 +21,12 @@ matrix:
     - os: osx
       osx_image: xcode7.3
       language: generic
-      env: TOXENV=py35 BDIST=1
+      env: TOXENV=py36 BDIST=1
     - python: 3.5
-      env: TOXENV=py35 BDIST=1
+      env: TOXENV=py35
       dist: precise
     - python: 3.6
-      env: TOXENV=py36
+      env: TOXENV=py36 BDIST=1 WHEEL=1
     - python: 3.6
       env: TOXENV=individual_coverage
     - language: node_js
@@ -42,6 +42,17 @@ matrix:
           yarn: true
           directories:
               - web/node_modules
+    - language: python
+      env:
+        - NAME=docs
+      install:
+        - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
+        - sudo dpkg -i hugo*.deb
+        - pip install awscli
+      script:
+        - cd docs && ./ci
+      after_success:
+        - echo done
 
 install:
   - |
@@ -50,12 +61,11 @@ install:
       brew update || brew update
       brew outdated pyenv || brew upgrade pyenv
       eval "$(pyenv init -)"
-      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3
+      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4
-      pyenv global 3.5.3
+      pyenv global 3.6.4
-      pyenv shell 3.5.3
+      pyenv shell 3.6.4
-      pip install -U pip setuptools wheel virtualenv
     fi
-  - pip install tox
+  - pip install tox virtualenv setuptools
 
 script:
   - tox -- --verbose --cov-report=term
@@ -63,16 +73,14 @@ script:
     if [[ $BDIST == "1" ]]
     then
         git fetch --unshallow --tags
-        tox -e rtool -- bdist
+        tox -e cibuild -- build
     fi
 
 after_success:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" ]]
     then
-        tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts
+        tox -e cibuild -- upload
-        tox -e rtool -- upload-snapshot --bdist
     fi
 
 notifications:

docs/ci

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This script gets run from CI to render and upload docs
+
+./build
+
+# Only upload if we have defined credentials - we only have these defined for
+# trusted commits (i.e. not PRs).
+if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
+    aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
+    aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
+        --paths "/master"
+fi

docs/upload-archive

@@ -12,6 +12,6 @@ SPATH="/archive/$1"
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \
     s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH
-# aws --profile mitmproxy \
+aws --profile mitmproxy \
-#     cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
+    cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-#     --paths "$SPATH"
+    --paths "$SPATH"

docs/upload-master

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-aws configure set preview.cloudfront true
-aws --profile mitmproxy \
-    s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
-# aws --profile mitmproxy \
-#     cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
-#     --paths "/master"

docs/upload-stable

@@ -3,6 +3,6 @@
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \
     s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable
-# aws --profile mitmproxy \
+aws --profile mitmproxy \
-#     cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
+    cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-#     --paths "/master"
+    --paths "/stable"

release/ci.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+
+import contextlib
+import os
+import platform
+import sys
+import shutil
+import subprocess
+import tarfile
+import zipfile
+from os.path import join, abspath, dirname, exists, basename
+
+import cryptography.fernet
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+    VENV_BIN = "Scripts"
+    PYINSTALLER_ARGS = [
+        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+    ]
+else:
+    VENV_BIN = "bin"
+    PYINSTALLER_ARGS = []
+
+# ZipFile and tarfile have slightly different APIs. Fix that.
+if platform.system() == "Windows":
+    def Archive(name):
+        a = zipfile.ZipFile(name, "w")
+        a.add = a.write
+        return a
+else:
+    def Archive(name):
+        return tarfile.open(name, "w:gz")
+
+PLATFORM_TAG = {
+    "Darwin": "osx",
+    "Windows": "windows",
+    "Linux": "linux",
+}.get(platform.system(), platform.system())
+
+ROOT_DIR = abspath(join(dirname(__file__), ".."))
+RELEASE_DIR = join(ROOT_DIR, "release")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
+DIST_DIR = join(RELEASE_DIR, "dist")
+
+PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+VENV_DIR = join(BUILD_DIR, "venv")
+
+# Project Configuration
+VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
+BDISTS = {
+    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+    "pathod": ["pathoc", "pathod"]
+}
+if platform.system() == "Windows":
+    BDISTS["mitmproxy"].remove("mitmproxy")
+
+TOOLS = [
+    tool
+    for tools in sorted(BDISTS.values())
+    for tool in tools
+]
+
+TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
+BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
+if TAG:
+    VERSION = TAG
+    UPLOAD_DIR = VERSION
+elif BRANCH:
+    VERSION = BRANCH
+    UPLOAD_DIR = "branches/%s" % VERSION
+else:
+    print("Could not establish build name - exiting." % BRANCH)
+    sys.exit(0)
+
+
+print("BUILD VERSION=%s" % VERSION)
+
+
+def archive_name(bdist: str) -> str:
+    if platform.system() == "Windows":
+        ext = "zip"
+    else:
+        ext = "tar.gz"
+    return "{project}-{version}-{platform}.{ext}".format(
+        project=bdist,
+        version=VERSION,
+        platform=PLATFORM_TAG,
+        ext=ext
+    )
+
+
+def wheel_name() -> str:
+    return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+
+
+def installer_name() -> str:
+    ext = {
+        "Windows": "exe",
+        "Darwin": "dmg",
+        "Linux": "run"
+    }[platform.system()]
+    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
+        version=VERSION,
+        platform=PLATFORM_TAG,
+        ext=ext,
+    )
+
+
+@contextlib.contextmanager
+def chdir(path: str):
+    old_dir = os.getcwd()
+    os.chdir(path)
+    yield
+    os.chdir(old_dir)
+
+
+@click.group(chain=True)
+def cli():
+    """
+    mitmproxy build tool
+    """
+    pass
+
+
+@cli.command("info")
+def info():
+    print("Version: %s" % VERSION)
+
+
+@cli.command("build")
+def build():
+    """
+    Build a binary distribution
+    """
+    if exists(PYINSTALLER_TEMP):
+        shutil.rmtree(PYINSTALLER_TEMP)
+    if exists(PYINSTALLER_DIST):
+        shutil.rmtree(PYINSTALLER_DIST)
+
+    os.makedirs(DIST_DIR, exist_ok=True)
+
+    if "WHEEL" in os.environ:
+        print("Building wheel...")
+        subprocess.check_call(
+            [
+                "python",
+                "setup.py", "-q", "bdist_wheel",
+                "--dist-dir", "release/dist",
+            ]
+        )
+
+    for bdist, tools in sorted(BDISTS.items()):
+        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+            for tool in tools:
+                # We can't have a folder and a file with the same name.
+                if tool == "mitmproxy":
+                    tool = "mitmproxy_main"
+                # This is PyInstaller, so it messes up paths.
+                # We need to make sure that we are in the spec folder.
+                with chdir(PYINSTALLER_SPEC):
+                    print("Building %s binary..." % tool)
+                    excludes = []
+                    if tool != "mitmweb":
+                        excludes.append("mitmproxy.tools.web")
+                    if tool != "mitmproxy_main":
+                        excludes.append("mitmproxy.tools.console")
+
+                    subprocess.check_call(
+                        [
+                            "pyinstaller",
+                            "--clean",
+                            "--workpath", PYINSTALLER_TEMP,
+                            "--distpath", PYINSTALLER_DIST,
+                            "--additional-hooks-dir", PYINSTALLER_HOOKS,
+                            "--onefile",
+                            "--console",
+                            "--icon", "icon.ico",
+                            # This is PyInstaller, so setting a
+                            # different log level obviously breaks it :-)
+                            # "--log-level", "WARN",
+                        ]
+                        + [x for e in excludes for x in ["--exclude-module", e]]
+                        + PYINSTALLER_ARGS
+                        + [tool]
+                    )
+                    # Delete the spec file - we're good without.
+                    os.remove("{}.spec".format(tool))
+
+                # Test if it works at all O:-)
+                executable = join(PYINSTALLER_DIST, tool)
+                if platform.system() == "Windows":
+                    executable += ".exe"
+
+                # Remove _main suffix from mitmproxy executable
+                if "_main" in executable:
+                    shutil.move(
+                        executable,
+                        executable.replace("_main", "")
+                    )
+                    executable = executable.replace("_main", "")
+
+                print("> %s --version" % executable)
+                print(subprocess.check_output([executable, "--version"]).decode())
+
+                archive.add(executable, basename(executable))
+        print("Packed {}.".format(archive_name(bdist)))
+
+
+def is_pr():
+    if "TRAVIS_PULL_REQUEST" in os.environ:
+        if os.environ["TRAVIS_PULL_REQUEST"] == "false":
+            return False
+        return True
+    elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"):
+        return True
+    return False
+
+
+@cli.command("upload")
+def upload():
+    """
+        Upload snapshot to snapshot server
+    """
+    # This requires some explanation. The AWS access keys are only exposed to
+    # privileged builds - that is, they are not available to PRs from forks.
+    # However, they ARE exposed to PRs from a branch within the main repo. This
+    # check catches that corner case, and prevents an inadvertent upload.
+    if is_pr():
+        print("Refusing to upload a pull request")
+        return
+    if "AWS_ACCESS_KEY_ID" in os.environ:
+        subprocess.check_call(
+            [
+                "aws", "s3", "cp",
+                "--acl", "public-read",
+                DIST_DIR + "/",
+                "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+                "--recursive",
+            ]
+        )
+
+
+@cli.command("decrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def decrypt(infile, outfile, key):
+    f = cryptography.fernet.Fernet(key.encode())
+    outfile.write(f.decrypt(infile.read()))
+
+
+if __name__ == "__main__":
+    cli()

release/known_hosts.enc

@@ -1 +0,0 @@
-gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7

release/rtool.pem

@@ -1,68 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB
-
-0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y
-byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU
-V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z
-T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh
-PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl
-w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD
-P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt
-JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE
-yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km
-cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe
-hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8
-jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx
-QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey
-OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL
-kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR
-AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc
-sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko
-SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz
-5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P
-cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk
-kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/
-Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9
-3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4
-mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t
-8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j
-FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf
-EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP
-a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR
-AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+
-m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P
-bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q
-QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx
-uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq
-yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K
-u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK
-MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0
-8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq
-y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz
-G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi
-o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq
-2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL
-5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp
-30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac
-7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA
-1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8
-eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB
-bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO
-VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+
-2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175
------END RSA PRIVATE KEY-----
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y
-qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe
-24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX
-FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ
-0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI
-KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt
-0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+
-tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ
-TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ
-q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J
-6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD
-5I1jQDXFbjAgAnaSyqyW368CAwEAAQ==
------END PUBLIC KEY-----

release/rtool.py

@@ -1,79 +1,22 @@
 #!/usr/bin/env python3
 
 import contextlib
-import fnmatch
 import os
 import sys
 import platform
-import re
 import runpy
 import shlex
-import shutil
 import subprocess
-import tarfile
+from os.path import join, abspath, dirname
-import zipfile
-from os.path import join, abspath, dirname, exists, basename
 
-import click
 import cryptography.fernet
-import pysftp
+import click
 
-# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
-# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
-if platform.system() == "Windows":
-    VENV_BIN = "Scripts"
-    PYINSTALLER_ARGS = [
-        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
-        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
-    ]
-else:
-    VENV_BIN = "bin"
-    PYINSTALLER_ARGS = []
-
-# ZipFile and tarfile have slightly different APIs. Fix that.
-if platform.system() == "Windows":
-    def Archive(name):
-        a = zipfile.ZipFile(name, "w")
-        a.add = a.write
-        return a
-else:
-    def Archive(name):
-        return tarfile.open(name, "w:gz")
-
-PLATFORM_TAG = {
-    "Darwin": "osx",
-    "Windows": "windows",
-    "Linux": "linux",
-}.get(platform.system(), platform.system())
 
 ROOT_DIR = abspath(join(dirname(__file__), ".."))
 RELEASE_DIR = join(ROOT_DIR, "release")
-
-BUILD_DIR = join(RELEASE_DIR, "build")
 DIST_DIR = join(RELEASE_DIR, "dist")
-
-PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
-# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
-PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
-PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
-PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
-
-VENV_DIR = join(BUILD_DIR, "venv")
-
-# Project Configuration
 VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-BDISTS = {
-    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
-    "pathod": ["pathoc", "pathod"]
-}
-if platform.system() == "Windows":
-    BDISTS["mitmproxy"].remove("mitmproxy")
-
-TOOLS = [
-    tool
-    for tools in sorted(BDISTS.values())
-    for tool in tools
-]
 
 
 def git(args: str) -> str:
@@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str:
     return x["get_version"](dev, build, True)
 
 
-def set_version(dev: bool) -> None:
-    """
-    Update version information in mitmproxy's version.py to either include hardcoded information or not.
-    """
-    version = get_version(dev, dev)
-    with open(VERSION_FILE, "r") as f:
-        content = f.read()
-    content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M)
-    with open(VERSION_FILE, "w") as f:
-        f.write(content)
-
-
-def archive_name(bdist: str) -> str:
-    if platform.system() == "Windows":
-        ext = "zip"
-    else:
-        ext = "tar.gz"
-    return "{project}-{version}-{platform}.{ext}".format(
-        project=bdist,
-        version=get_version(),
-        platform=PLATFORM_TAG,
-        ext=ext
-    )
-
-
 def wheel_name() -> str:
     return "mitmproxy-{version}-py3-none-any.whl".format(
         version=get_version(True),
     )
 
 
-def installer_name() -> str:
-    ext = {
-        "Windows": "exe",
-        "Darwin": "dmg",
-        "Linux": "run"
-    }[platform.system()]
-    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
-        version=get_version(),
-        platform=PLATFORM_TAG,
-        ext=ext,
-    )
-
-
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -146,24 +51,6 @@ def cli():
     pass
 
 
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.encrypt(infile.read()))
-
-
-@cli.command("decrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def decrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.decrypt(infile.read()))
-
-
 @cli.command("contributors")
 def contributors():
     """
@@ -176,97 +63,6 @@ def contributors():
             f.write(contributors_data.encode())
 
 
-@cli.command("wheel")
-def make_wheel():
-    """
-    Build a Python wheel
-    """
-    set_version(True)
-    try:
-        subprocess.check_call([
-            "tox", "-e", "wheel",
-        ], env={
-            **os.environ,
-            "VERSION": get_version(True),
-        })
-    finally:
-        set_version(False)
-
-
-@cli.command("bdist")
-def make_bdist():
-    """
-    Build a binary distribution
-    """
-    if exists(PYINSTALLER_TEMP):
-        shutil.rmtree(PYINSTALLER_TEMP)
-    if exists(PYINSTALLER_DIST):
-        shutil.rmtree(PYINSTALLER_DIST)
-
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    for bdist, tools in sorted(BDISTS.items()):
-        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
-            for tool in tools:
-                # We can't have a folder and a file with the same name.
-                if tool == "mitmproxy":
-                    tool = "mitmproxy_main"
-                # This is PyInstaller, so it messes up paths.
-                # We need to make sure that we are in the spec folder.
-                with chdir(PYINSTALLER_SPEC):
-                    print("Building %s binary..." % tool)
-                    excludes = []
-                    if tool != "mitmweb":
-                        excludes.append("mitmproxy.tools.web")
-                    if tool != "mitmproxy_main":
-                        excludes.append("mitmproxy.tools.console")
-
-                    # Overwrite mitmproxy/version.py to include commit info
-                    set_version(True)
-                    try:
-                        subprocess.check_call(
-                            [
-                                "pyinstaller",
-                                "--clean",
-                                "--workpath", PYINSTALLER_TEMP,
-                                "--distpath", PYINSTALLER_DIST,
-                                "--additional-hooks-dir", PYINSTALLER_HOOKS,
-                                "--onefile",
-                                "--console",
-                                "--icon", "icon.ico",
-                                # This is PyInstaller, so setting a
-                                # different log level obviously breaks it :-)
-                                # "--log-level", "WARN",
-                            ]
-                            + [x for e in excludes for x in ["--exclude-module", e]]
-                            + PYINSTALLER_ARGS
-                            + [tool]
-                        )
-                    finally:
-                        set_version(False)
-                    # Delete the spec file - we're good without.
-                    os.remove("{}.spec".format(tool))
-
-                # Test if it works at all O:-)
-                executable = join(PYINSTALLER_DIST, tool)
-                if platform.system() == "Windows":
-                    executable += ".exe"
-
-                # Remove _main suffix from mitmproxy executable
-                if "_main" in executable:
-                    shutil.move(
-                        executable,
-                        executable.replace("_main", "")
-                    )
-                    executable = executable.replace("_main", "")
-
-                print("> %s --version" % executable)
-                print(subprocess.check_output([executable, "--version"]).decode())
-
-                archive.add(executable, basename(executable))
-        print("Packed {}.".format(archive_name(bdist)))
-
-
 @cli.command("upload-release")
 @click.option('--username', prompt=True)
 @click.password_option(confirmation_prompt=False)
@@ -305,77 +101,13 @@ def homebrew_pr():
     ])
 
 
-@cli.command("upload-snapshot")
+@cli.command("encrypt")
-@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
+@click.argument('infile', type=click.File('rb'))
-@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
+@click.argument('outfile', type=click.File('wb'))
-@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
+@click.argument('key', envvar='RTOOL_KEY')
-@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
+def encrypt(infile, outfile, key):
-@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
+    f = cryptography.fernet.Fernet(key.encode())
-@click.option("--wheel/--no-wheel", default=False)
+    outfile.write(f.encrypt(infile.read()))
-@click.option("--bdist/--no-bdist", default=False)
-@click.option("--installer/--no-installer", default=False)
-def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
-    """
-    Upload snapshot to snapshot server
-    """
-    cnopts = pysftp.CnOpts(
-        knownhosts=join(RELEASE_DIR, 'known_hosts')
-    )
-    with pysftp.Connection(host=host,
-                           port=port,
-                           username=user,
-                           private_key=private_key,
-                           private_key_pass=private_key_password,
-                           cnopts=cnopts) as sftp:
-        dir_name = "snapshots/v{}".format(get_version())
-        sftp.makedirs(dir_name)
-        with sftp.cd(dir_name):
-            files = []
-            if wheel:
-                files.append(wheel_name())
-            if bdist:
-                for bdist in sorted(BDISTS.keys()):
-                    files.append(archive_name(bdist))
-            if installer:
-                files.append(installer_name())
-
-            for f in files:
-                local_path = join(DIST_DIR, f)
-                remote_filename = re.sub(
-                    r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
-                    get_version(True, True),
-                    f
-                )
-                symlink_path = "../{}".format(f.replace(get_version(), "latest"))
-
-                # Upload new version
-                print("Uploading {} as {}...".format(f, remote_filename))
-                with click.progressbar(length=os.stat(local_path).st_size) as bar:
-                    # We hide the file during upload
-                    sftp.put(
-                        local_path,
-                        "." + remote_filename,
-                        callback=lambda done, total: bar.update(done - bar.pos)
-                    )
-
-                # Delete old versions
-                old_version = f.replace(get_version(), "*")
-                for f_old in sftp.listdir():
-                    if fnmatch.fnmatch(f_old, old_version):
-                        print("Removing {}...".format(f_old))
-                        sftp.remove(f_old)
-
-                # Show new version
-                sftp.rename("." + remote_filename, remote_filename)
-
-                # update symlink for the latest release
-                if sftp.lexists(symlink_path):
-                    print("Removing {}...".format(symlink_path))
-                    sftp.remove(symlink_path)
-                if f != wheel_name():
-                    # "latest" isn't a proper wheel version, so this could not be installed.
-                    # https://github.com/mitmproxy/mitmproxy/issues/1065
-                    sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
 
 
 if __name__ == "__main__":

tox.ini

@@ -32,27 +32,12 @@ deps =
 commands =
   python test/individual_coverage.py
 
-[testenv:wheel]
+[testenv:cibuild]
-recreate = True
+passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
-deps =
-commands =
-  python setup.py -q bdist_wheel --dist-dir release/dist
-  pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl
-  # skip `mitmproxy --version` if SKIP_MITMPROXY is defined.
-  {env:SKIP_MITMPROXY:mitmproxy --version}
-  mitmdump --version
-  mitmweb --version
-  pathod --version
-  pathoc --version
-
-[testenv:rtool]
-passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY
 deps =
   -rrequirements.txt
   pyinstaller==3.3.1
-  twine==1.9.1
+  awscli
-  pysftp==0.2.9
-
 commands =
   mitmdump --version
-  python ./release/rtool.py {posargs}
+  python ./release/ci.py {posargs}