Add wheels for all CI (#5780)

Add wheel generation support for osx and linux.
2018-06-01 01:25:19 -04:00 · 2018-06-01 01:25:19 -04:00 · ce6d54e2f3
parent b19d755466
commit ce6d54e2f3
6 changed files with 175 additions and 50 deletions
--- a/.ci/appveyor-server-upload.sh
+++ b/.ci/appveyor-server-upload.sh
@ -0,0 +1,8 @@
+pacman -S --noconfirm git rsync
+if [ ! -d "/home/appveyor/.ssh" ]; then
+  mkdir "/home/appveyor/.ssh"
+fi
+echo -e "Host $1\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+cp "$(dirname "$0")/id_rsa" ~/.ssh/id_rsa
+echo "copying $3 from $(cygpath -u "$2") to root@$1:/web/downloads/$4"
+rsync -avh -e "ssh -p 2458" --include="*/" --include="$3" --exclude="*" "$(cygpath -u "$2")/" "root@$1:/web/downloads/$4"
--- a/.ci/build-wheels-linux.sh
+++ b/.ci/build-wheels-linux.sh
@ -0,0 +1,53 @@
+#!/bin/bash
+set -e -x
+
+yum -y install  autoconf automake cmake gcc gcc-c++ git make pkgconfig zlib-devel portmidi portmidi-devel gstreamer gstreamer-devel gstreamer-plugins-base gstreamer-plugins-base-devel gstreamer-plugins-good gstreamer-plugins-good-devel Xorg-x11-server-deve mesa-libEGL-devel mtdev-devel mesa-libEGL freetype freetype-devel
+mkdir ~/kivy_sources;
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/kivy_build/lib;
+
+cd ~/kivy_sources;
+git clone --depth 1 https://github.com/spurious/SDL-mirror.git
+cd SDL-mirror;
+./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_mixer/release/SDL2_mixer-2.0.2.tar.gz;
+tar xzf SDL2_mixer-2.0.2.tar.gz;
+cd SDL2_mixer-2.0.2;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_image/release/SDL2_image-2.0.3.tar.gz;
+tar xzf SDL2_image-2.0.3.tar.gz;
+cd SDL2_image-2.0.3;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_ttf/release/SDL2_ttf-2.0.14.tar.gz;
+tar xzf SDL2_ttf-2.0.14.tar.gz;
+cd SDL2_ttf-2.0.14;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd /io;
+for PYBIN in /opt/python/*3*/bin; do
+    "${PYBIN}/pip" install --upgrade setuptools pip;
+    "${PYBIN}/pip" install --upgrade cython nose pygments docutils;
+    USE_X11=1 USE_SDL2=1 USE_GSTREAMER=0 PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" "${PYBIN}/pip" wheel --no-deps . -w wheelhouse/;
+done
+
+for name in /io/wheelhouse/*.whl; do
+    echo "Fixing $name";
+    auditwheel repair $name -w /io/wheelhouse/;
+done
--- a/kivy/tools/appveyor/id_rsa.enc
+++ b/kivy/tools/appveyor/id_rsa.enc
--- a/.travis.yml
+++ b/.travis.yml
@ -1,3 +1,8 @@
+env:
+  global:
+    - KIVY_USE_SETUPTOOLS=1
+    - KIVY_SPLIT_EXAMPLES=1
+    - SERVER_IP=159.203.106.198
 matrix:
  fast_finish: true
  include:
@ -21,12 +26,20 @@ matrix:
      env: RUN=docs
      os: linux
      dist: trusty
+    - language: python
+      sudo: required
+      env: RUN=wheels DOCKER_IMAGE=markrwilliams/manylinux2:x86_64
+      services:
+        - docker
    - language: generic
      env: RUN=unit PY=2
      os: osx
    - language: generic
      env: RUN=unit PY=3
      os: osx
+    - language: generic
+      os: osx
+      env: PYVERS="3.5.4 3.6.5" PY=3 RUN=wheels

 before_install:
  # https://github.com/travis-ci/travis-ci/issues/6307
@ -38,7 +51,7 @@ before_install:
    fi

 install:
-  - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "wheels" ]; then
      if [ "${RUN}" != "pep8" ]; then
        yes | sudo add-apt-repository ppa:zoogie/sdl2-snapshots;
        yes | sudo add-apt-repository ppa:gstreamer-developers/ppa;
@ -77,19 +90,21 @@ install:
      sudo installer -package gstreamer-1.0-1.10.2-x86_64.pkg -target /;
      sudo installer -package gstreamer-1.0-devel-1.10.2-x86_64.pkg -target /;
      curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py;
-      if [ "${PY}" == "3" ]; then
-         curl -O -L https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg;
-         sudo installer -package python-3.5.2-macosx10.6.pkg -target /;
-         python3 get-pip.py --user;
-         python3 -m pip install --upgrade --user cython pillow nose mock docutils;
-      else
-         python get-pip.py --user;
-         python -m pip install --upgrade --user cython pillow nose mock docutils;
+      if [ "${RUN}" != "wheels" ]; then
+        if [ "${PY}" == "3" ]; then
+           curl -O -L https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg;
+           sudo installer -package python-3.5.2-macosx10.6.pkg -target /;
+           python3 get-pip.py --user;
+           python3 -m pip install --upgrade --user cython pillow nose mock docutils;
+        else
+           python get-pip.py --user;
+           python -m pip install --upgrade --user cython pillow nose mock docutils;
+        fi;
      fi;
    fi;

 before_script:
-  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "pep8" ]; then
+  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "pep8" ] && [ "${RUN}" != "wheels" ]; then
      export DISPLAY=:99.0;
      /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1280x720x24 -ac +extension GLX;
      export PYTHONPATH=$PYTHONPATH:$(pwd);
@ -117,15 +132,76 @@ script:
      if [ "${RUN}" == "docs" ]; then
        make html;
      fi;
+      if [ "${RUN}" == "wheels" ] && ([ "${TRAVIS_EVENT_TYPE}" == "cron" ] || [ "${TRAVIS_TAG}" != "" ] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel]" ]] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel linux]" ]]); then
+        mkdir wheelhouse;
+        wheel_date=$(python -c "from datetime import datetime; print(datetime.utcnow().strftime('%Y%m%d'))");
+        git_tag=$(git rev-parse --short HEAD);
+        wheel_name="dev0.$wheel_date.$git_tag";
+
+        chmod +x .ci/build-wheels-linux.sh;
+        docker run --rm -v `pwd`:/io $DOCKER_IMAGE "/io/.ci/build-wheels-linux.sh";
+        for name in wheelhouse/*manylinux*.whl; do
+          new_name="${name/dev0/$wheel_name}";
+          cp $name $new_name;
+        done;
+        ls wheelhouse/;
+
+        openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
+        chmod 600 ~/.ssh/id_rsa;
+        echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+        rsync -avh -e "ssh -p 2458" --include="*/" --include="*manylinux*.whl" --exclude="*" "wheelhouse/" "root@$SERVER_IP:/web/downloads/ci/linux/kivy/";
+      fi;
    fi;
  - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
      set -ex;
-      if [ "${PY}" == "3" ]; then
-        python3 setup.py build_ext --inplace;
-        python3 -m nose.core kivy/tests;
-      else
-        make;
-        make test;
+      if [ "${RUN}" != "wheels" ]; then
+        if [ "${PY}" == "3" ]; then
+          python3 setup.py build_ext --inplace;
+          python3 -m nose.core kivy/tests;
+        else
+          make;
+          make test;
+        fi;
+      elif [ "${TRAVIS_EVENT_TYPE}" == "cron" ] || [ "${TRAVIS_TAG}" != "" ] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel]" ]] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel osx]" ]]; then
+        mkdir ../wheelhouse;
+
+        for pyver in $PYVERS; do
+          git reset --hard;
+          git clean -d -x -f;
+          pyver_short=${pyver:0:3};
+
+          curl -O -L https://www.python.org/ftp/python/$pyver/python-$pyver-macosx10.6.pkg;
+          sudo installer -package python-$pyver-macosx10.6.pkg -target /;
+          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py;
+          python$pyver_short get-pip.py --user;
+
+          python$pyver_short -m pip install --upgrade --user pip;
+          python$pyver_short -m pip install --upgrade --user cython nose wheel pillow mock docutils;
+          python$pyver_short -m pip install --upgrade delocate;
+
+          USE_SDL2=1 USE_GSTREAMER=1 python$pyver_short setup.py build_ext --inplace;
+          USE_SDL2=1 USE_GSTREAMER=1 python$pyver_short setup.py bdist_wheel;
+          # python$pyver_short -m nose.core kivy/tests;
+
+          /Library/Frameworks/Python.framework/Versions/$pyver_short/bin/delocate-wheel dist/*.whl;
+          /Library/Frameworks/Python.framework/Versions/$pyver_short/bin/delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl;
+          cp dist/*.whl ../wheelhouse/;
+        done;
+
+        wheel_date=$(python -c "from datetime import datetime; print(datetime.utcnow().strftime('%Y%m%d'))");
+        git_tag=$(git rev-parse --short HEAD);
+        wheel_name="dev0.$wheel_date.$git_tag";
+
+        for name in ../wheelhouse/*.whl; do
+          new_name="${name/dev0/$wheel_name}";
+          cp $name $new_name;
+        done;
+        ls ../wheelhouse/;
+
+        openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
+        chmod 600 ~/.ssh/id_rsa;
+        echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+        rsync -avh -e "ssh -p 2458" --include="*/" --include="*.whl" --exclude="*" "../wheelhouse/" "root@$SERVER_IP:/web/downloads/ci/osx/kivy/";
      fi;
    fi;

@ -134,8 +210,8 @@ after_success:
  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" == "docs" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then
      openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
      chmod 600 ~/.ssh/id_rsa;
-      echo -e "Host 159.203.106.198\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
-      rsync --delete --force -r -e "ssh -p 2457" ./doc/build/html/ root@159.203.106.198:/web/docs/;
+      echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+      rsync --delete --force -r -e "ssh -p 2457" ./doc/build/html/ root@$SERVER_IP:/web/docs/;
    fi;

 notifications:
--- a/appveyor.yml
+++ b/appveyor.yml
@ -6,6 +6,7 @@ environment:
  DO_WHEEL: True
  KEY_WITH_NO_TEETH:
    secure: 7cS7xjpCL/VH5jIIGSf13camkiu1enMh5hO0UsBgmRlBXyKk3t/7HB79ofyJKgDb
+  SERVER_IP: 159.203.106.198
  USE_SDL2: 1
  USE_GSTREAMER: 1
  KIVY_USE_SETUPTOOLS: 1
@ -13,18 +14,12 @@ environment:
  MSYSTEM: MINGW64
  CHERE_INVOKING: 1
  matrix:
-  # - PYVER: 27
-  #   BITTNESS: 86
-  #   COMPILER: mingw
-  # - PYVER: 27
-  #   BITTNESS: 64
-  #   COMPILER: mingw
-  # - PYVER: 34
-  #   BITTNESS: 86
-  #   COMPILER: mingw
-  # - PYVER: 34
-  #   BITTNESS: 64
-  #   COMPILER: mingw
+  - PYVER: 27
+    BITTNESS: 86
+    COMPILER: mingw
+  - PYVER: 27
+    BITTNESS: 64
+    COMPILER: mingw
  - PYVER: 35
    BITTNESS: 86
    COMPILER: msvc
@ -74,12 +69,6 @@ build_script:
    Check-Error


-    C:\Python27\Scripts\pip.exe install pydrive
-
-    Check-Error
-
-
-
    if ($env:BITTNESS -eq "64") {
      $PYTHON_ROOT = "C:\Python$env:PYVER-x64"
      $WHELL_BITNESS = "win_amd64"
@ -130,13 +119,13 @@ build_script:

    if ($env:APPVEYOR_REPO_TAG -eq "true"){
      $WHEEL_NAME = "dev0-"
-    } elseif ($env:APPVEYOR_SCHEDULED_BUILD -eq "True" -or $env:APPVEYOR_FORCED_BUILD -eq "True" -or $env:APPVEYOR_RE_BUILD -eq "True"){
+    } elseif ($env:APPVEYOR_SCHEDULED_BUILD -eq "True" -or $env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel]") -or $env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel win]")){
      $WHEEL_NAME = "dev0.$WHEEL_DATE`.$GIT_TAG-"
    } else {
      $DO_WHEEL = "False"
    }

-    if ($env:APPVEYOR_REPO_BRANCH -ne "master") {
+    if ($env:APPVEYOR_REPO_BRANCH -ne "master" -and -Not ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel win]")) -and -Not ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel]"))) {
      $DO_WHEEL = "False"
    }

@ -159,7 +148,9 @@ build_script:

    Check-Error

-    pip install kivy.deps.angle
+    if ($env:COMPILER -eq "msvc") {
+      pip install kivy.deps.angle
+    }

    Copy-Item "$PYTHON_ROOT\Lib\site-packages\kivy\deps\*" -destination "$env:APPVEYOR_BUILD_FOLDER\kivy\deps" -recurse

@ -174,7 +165,7 @@ build_script:
      python setup.py bdist_wheel -d "$env:WHEEL_DIR"
      Check-Error

-      if ($env:BITTNESS -eq "64" -and $env:PYVER -eq "27") {
+      if ($env:BITTNESS -eq "64" -and $env:PYVER -eq "35") {
        python setup.py bdist_wheel -d "$env:WHEEL_DIR" --build_examples --universal
        Check-Error
      }
@ -194,10 +185,10 @@ build_script:
        Check-Error
      }

-      secure-file\tools\secure-file -decrypt C:\projects\kivy\kivy\tools\appveyor\id_rsa.enc -secret "$env:KEY_WITH_NO_TEETH"
+      secure-file\tools\secure-file -decrypt C:\projects\kivy\.ci\id_rsa.enc -secret "$env:KEY_WITH_NO_TEETH"
      Check-Error

-      C:\msys64\usr\bin\bash --login C:\projects\kivy\kivy\tools\appveyor\kivy-upload.sh
+      C:\msys64\usr\bin\bash --login -c "/c/projects/kivy/.ci/appveyor-server-upload.sh $env:SERVER_IP '$env:WHEEL_DIR' Kivy* appveyor/kivy/"
      Check-Error

    }
@ -214,7 +205,11 @@ test_script:


    if ($env:DO_TEST -eq "True"){
-      $env:KIVY_GL_BACKEND = "angle_sdl2"
+      if ($env:COMPILER -eq "msvc") {
+        $env:KIVY_GL_BACKEND = "angle_sdl2"
+      } else {
+        $env:KIVY_GL_BACKEND = "mock"
+      }
      python -m nose.core kivy/tests
      Check-Error
    }
--- a/kivy/tools/appveyor/kivy-upload.sh
+++ b/kivy/tools/appveyor/kivy-upload.sh
@ -1,7 +0,0 @@
-pacman -S --noconfirm git rsync
-if [ ! -d "/home/appveyor/.ssh" ]; then
-  mkdir "/home/appveyor/.ssh"
-fi
-echo -e "Host 159.203.106.198\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-cp $(cygpath -u "C:\projects\kivy\kivy\tools\appveyor\id_rsa") ~/.ssh/id_rsa
-rsync -avh -e "ssh -p 2458" "/c/kivy_wheels/" root@159.203.106.198:/web/downloads/appveyor/kivy