Add wheels for all CI (#5780)

Add wheel generation support for osx and linux.

.ci/appveyor-server-upload.sh

@@ -0,0 +1,8 @@
+pacman -S --noconfirm git rsync
+if [ ! -d "/home/appveyor/.ssh" ]; then
+  mkdir "/home/appveyor/.ssh"
+fi
+echo -e "Host $1\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
+cp "$(dirname "$0")/id_rsa" ~/.ssh/id_rsa
+echo "copying $3 from $(cygpath -u "$2") to root@$1:/web/downloads/$4"
+rsync -avh -e "ssh -p 2458" --include="*/" --include="$3" --exclude="*" "$(cygpath -u "$2")/" "root@$1:/web/downloads/$4"

.ci/build-wheels-linux.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e -x
+
+yum -y install  autoconf automake cmake gcc gcc-c++ git make pkgconfig zlib-devel portmidi portmidi-devel gstreamer gstreamer-devel gstreamer-plugins-base gstreamer-plugins-base-devel gstreamer-plugins-good gstreamer-plugins-good-devel Xorg-x11-server-deve mesa-libEGL-devel mtdev-devel mesa-libEGL freetype freetype-devel
+mkdir ~/kivy_sources;
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/kivy_build/lib;
+
+cd ~/kivy_sources;
+git clone --depth 1 https://github.com/spurious/SDL-mirror.git
+cd SDL-mirror;
+./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_mixer/release/SDL2_mixer-2.0.2.tar.gz;
+tar xzf SDL2_mixer-2.0.2.tar.gz;
+cd SDL2_mixer-2.0.2;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_image/release/SDL2_image-2.0.3.tar.gz;
+tar xzf SDL2_image-2.0.3.tar.gz;
+cd SDL2_image-2.0.3;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd ~/kivy_sources;
+wget http://www.libsdl.org/projects/SDL_ttf/release/SDL2_ttf-2.0.14.tar.gz;
+tar xzf SDL2_ttf-2.0.14.tar.gz;
+cd SDL2_ttf-2.0.14;
+PATH="$HOME/kivy_build/bin:$PATH" PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" ./configure --prefix="$HOME/kivy_build" --bindir="$HOME/kivy_build/bin";
+PATH="$HOME/kivy_build/bin:$PATH" make;
+make install;
+make distclean;
+
+cd /io;
+for PYBIN in /opt/python/*3*/bin; do
+    "${PYBIN}/pip" install --upgrade setuptools pip;
+    "${PYBIN}/pip" install --upgrade cython nose pygments docutils;
+    USE_X11=1 USE_SDL2=1 USE_GSTREAMER=0 PKG_CONFIG_PATH="$HOME/kivy_build/lib/pkgconfig" "${PYBIN}/pip" wheel --no-deps . -w wheelhouse/;
+done
+
+for name in /io/wheelhouse/*.whl; do
+    echo "Fixing $name";
+    auditwheel repair $name -w /io/wheelhouse/;
+done

.travis.yml

@@ -1,3 +1,8 @@
+env:
+  global:
+    - KIVY_USE_SETUPTOOLS=1
+    - KIVY_SPLIT_EXAMPLES=1
+    - SERVER_IP=159.203.106.198
 matrix:
   fast_finish: true
   include:
@@ -21,12 +26,20 @@ matrix:
       env: RUN=docs
       os: linux
       dist: trusty
+    - language: python
+      sudo: required
+      env: RUN=wheels DOCKER_IMAGE=markrwilliams/manylinux2:x86_64
+      services:
+        - docker
     - language: generic
       env: RUN=unit PY=2
       os: osx
     - language: generic
       env: RUN=unit PY=3
       os: osx
+    - language: generic
+      os: osx
+      env: PYVERS="3.5.4 3.6.5" PY=3 RUN=wheels
 
 before_install:
   # https://github.com/travis-ci/travis-ci/issues/6307
@@ -38,7 +51,7 @@ before_install:
     fi
 
 install:
-  - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "wheels" ]; then
       if [ "${RUN}" != "pep8" ]; then
         yes | sudo add-apt-repository ppa:zoogie/sdl2-snapshots;
         yes | sudo add-apt-repository ppa:gstreamer-developers/ppa;
@@ -77,6 +90,7 @@ install:
       sudo installer -package gstreamer-1.0-1.10.2-x86_64.pkg -target /;
       sudo installer -package gstreamer-1.0-devel-1.10.2-x86_64.pkg -target /;
       curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py;
+      if [ "${RUN}" != "wheels" ]; then
         if [ "${PY}" == "3" ]; then
            curl -O -L https://www.python.org/ftp/python/3.5.2/python-3.5.2-macosx10.6.pkg;
            sudo installer -package python-3.5.2-macosx10.6.pkg -target /;
@@ -87,9 +101,10 @@ install:
            python -m pip install --upgrade --user cython pillow nose mock docutils;
         fi;
       fi;
+    fi;
 
 before_script:
-  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "pep8" ]; then
+  - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" != "pep8" ] && [ "${RUN}" != "wheels" ]; then
       export DISPLAY=:99.0;
       /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1280x720x24 -ac +extension GLX;
       export PYTHONPATH=$PYTHONPATH:$(pwd);
@@ -117,9 +132,29 @@ script:
       if [ "${RUN}" == "docs" ]; then
         make html;
       fi;
+      if [ "${RUN}" == "wheels" ] && ([ "${TRAVIS_EVENT_TYPE}" == "cron" ] || [ "${TRAVIS_TAG}" != "" ] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel]" ]] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel linux]" ]]); then
+        mkdir wheelhouse;
+        wheel_date=$(python -c "from datetime import datetime; print(datetime.utcnow().strftime('%Y%m%d'))");
+        git_tag=$(git rev-parse --short HEAD);
+        wheel_name="dev0.$wheel_date.$git_tag";
+
+        chmod +x .ci/build-wheels-linux.sh;
+        docker run --rm -v `pwd`:/io $DOCKER_IMAGE "/io/.ci/build-wheels-linux.sh";
+        for name in wheelhouse/*manylinux*.whl; do
+          new_name="${name/dev0/$wheel_name}";
+          cp $name $new_name;
+        done;
+        ls wheelhouse/;
+
+        openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
+        chmod 600 ~/.ssh/id_rsa;
+        echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+        rsync -avh -e "ssh -p 2458" --include="*/" --include="*manylinux*.whl" --exclude="*" "wheelhouse/" "root@$SERVER_IP:/web/downloads/ci/linux/kivy/";
+      fi;
     fi;
   - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
       set -ex;
+      if [ "${RUN}" != "wheels" ]; then
         if [ "${PY}" == "3" ]; then
           python3 setup.py build_ext --inplace;
           python3 -m nose.core kivy/tests;
@@ -127,6 +162,47 @@ script:
           make;
           make test;
         fi;
+      elif [ "${TRAVIS_EVENT_TYPE}" == "cron" ] || [ "${TRAVIS_TAG}" != "" ] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel]" ]] || [[ "$TRAVIS_COMMIT_MESSAGE" =~ "[build wheel osx]" ]]; then
+        mkdir ../wheelhouse;
+
+        for pyver in $PYVERS; do
+          git reset --hard;
+          git clean -d -x -f;
+          pyver_short=${pyver:0:3};
+
+          curl -O -L https://www.python.org/ftp/python/$pyver/python-$pyver-macosx10.6.pkg;
+          sudo installer -package python-$pyver-macosx10.6.pkg -target /;
+          curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py;
+          python$pyver_short get-pip.py --user;
+
+          python$pyver_short -m pip install --upgrade --user pip;
+          python$pyver_short -m pip install --upgrade --user cython nose wheel pillow mock docutils;
+          python$pyver_short -m pip install --upgrade delocate;
+
+          USE_SDL2=1 USE_GSTREAMER=1 python$pyver_short setup.py build_ext --inplace;
+          USE_SDL2=1 USE_GSTREAMER=1 python$pyver_short setup.py bdist_wheel;
+          # python$pyver_short -m nose.core kivy/tests;
+
+          /Library/Frameworks/Python.framework/Versions/$pyver_short/bin/delocate-wheel dist/*.whl;
+          /Library/Frameworks/Python.framework/Versions/$pyver_short/bin/delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl;
+          cp dist/*.whl ../wheelhouse/;
+        done;
+
+        wheel_date=$(python -c "from datetime import datetime; print(datetime.utcnow().strftime('%Y%m%d'))");
+        git_tag=$(git rev-parse --short HEAD);
+        wheel_name="dev0.$wheel_date.$git_tag";
+
+        for name in ../wheelhouse/*.whl; do
+          new_name="${name/dev0/$wheel_name}";
+          cp $name $new_name;
+        done;
+        ls ../wheelhouse/;
+
+        openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
+        chmod 600 ~/.ssh/id_rsa;
+        echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+        rsync -avh -e "ssh -p 2458" --include="*/" --include="*.whl" --exclude="*" "../wheelhouse/" "root@$SERVER_IP:/web/downloads/ci/osx/kivy/";
+      fi;
     fi;
 
 
@@ -134,8 +210,8 @@ after_success:
   - if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "${RUN}" == "docs" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ] && [ "${TRAVIS_BRANCH}" == "master" ]; then
       openssl aes-256-cbc -K $encrypted_675f1a0c317c_key -iv $encrypted_675f1a0c317c_iv -in ./kivy/tools/travis/id_rsa.enc -out ~/.ssh/id_rsa -d;
       chmod 600 ~/.ssh/id_rsa;
-      echo -e "Host 159.203.106.198\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
+      echo -e "Host $SERVER_IP\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config;
-      rsync --delete --force -r -e "ssh -p 2457" ./doc/build/html/ root@159.203.106.198:/web/docs/;
+      rsync --delete --force -r -e "ssh -p 2457" ./doc/build/html/ root@$SERVER_IP:/web/docs/;
     fi;
 
 notifications:

appveyor.yml

@@ -6,6 +6,7 @@ environment:
   DO_WHEEL: True
   KEY_WITH_NO_TEETH:
     secure: 7cS7xjpCL/VH5jIIGSf13camkiu1enMh5hO0UsBgmRlBXyKk3t/7HB79ofyJKgDb
+  SERVER_IP: 159.203.106.198
   USE_SDL2: 1
   USE_GSTREAMER: 1
   KIVY_USE_SETUPTOOLS: 1
@@ -13,18 +14,12 @@ environment:
   MSYSTEM: MINGW64
   CHERE_INVOKING: 1
   matrix:
-  # - PYVER: 27
+  - PYVER: 27
-  #   BITTNESS: 86
+    BITTNESS: 86
-  #   COMPILER: mingw
+    COMPILER: mingw
-  # - PYVER: 27
+  - PYVER: 27
-  #   BITTNESS: 64
+    BITTNESS: 64
-  #   COMPILER: mingw
+    COMPILER: mingw
-  # - PYVER: 34
-  #   BITTNESS: 86
-  #   COMPILER: mingw
-  # - PYVER: 34
-  #   BITTNESS: 64
-  #   COMPILER: mingw
   - PYVER: 35
     BITTNESS: 86
     COMPILER: msvc
@@ -74,12 +69,6 @@ build_script:
     Check-Error
 
 
-    C:\Python27\Scripts\pip.exe install pydrive
-
-    Check-Error
-
-
-
     if ($env:BITTNESS -eq "64") {
       $PYTHON_ROOT = "C:\Python$env:PYVER-x64"
       $WHELL_BITNESS = "win_amd64"
@@ -130,13 +119,13 @@ build_script:
 
     if ($env:APPVEYOR_REPO_TAG -eq "true"){
       $WHEEL_NAME = "dev0-"
-    } elseif ($env:APPVEYOR_SCHEDULED_BUILD -eq "True" -or $env:APPVEYOR_FORCED_BUILD -eq "True" -or $env:APPVEYOR_RE_BUILD -eq "True"){
+    } elseif ($env:APPVEYOR_SCHEDULED_BUILD -eq "True" -or $env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel]") -or $env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel win]")){
       $WHEEL_NAME = "dev0.$WHEEL_DATE`.$GIT_TAG-"
     } else {
       $DO_WHEEL = "False"
     }
 
-    if ($env:APPVEYOR_REPO_BRANCH -ne "master") {
+    if ($env:APPVEYOR_REPO_BRANCH -ne "master" -and -Not ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel win]")) -and -Not ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[build wheel]"))) {
       $DO_WHEEL = "False"
     }
 
@@ -159,7 +148,9 @@ build_script:
 
     Check-Error
 
+    if ($env:COMPILER -eq "msvc") {
       pip install kivy.deps.angle
+    }
 
     Copy-Item "$PYTHON_ROOT\Lib\site-packages\kivy\deps\*" -destination "$env:APPVEYOR_BUILD_FOLDER\kivy\deps" -recurse
 
@@ -174,7 +165,7 @@ build_script:
       python setup.py bdist_wheel -d "$env:WHEEL_DIR"
       Check-Error
 
-      if ($env:BITTNESS -eq "64" -and $env:PYVER -eq "27") {
+      if ($env:BITTNESS -eq "64" -and $env:PYVER -eq "35") {
         python setup.py bdist_wheel -d "$env:WHEEL_DIR" --build_examples --universal
         Check-Error
       }
@@ -194,10 +185,10 @@ build_script:
         Check-Error
       }
 
-      secure-file\tools\secure-file -decrypt C:\projects\kivy\kivy\tools\appveyor\id_rsa.enc -secret "$env:KEY_WITH_NO_TEETH"
+      secure-file\tools\secure-file -decrypt C:\projects\kivy\.ci\id_rsa.enc -secret "$env:KEY_WITH_NO_TEETH"
       Check-Error
 
-      C:\msys64\usr\bin\bash --login C:\projects\kivy\kivy\tools\appveyor\kivy-upload.sh
+      C:\msys64\usr\bin\bash --login -c "/c/projects/kivy/.ci/appveyor-server-upload.sh $env:SERVER_IP '$env:WHEEL_DIR' Kivy* appveyor/kivy/"
       Check-Error
 
     }
@@ -214,7 +205,11 @@ test_script:
 
 
     if ($env:DO_TEST -eq "True"){
+      if ($env:COMPILER -eq "msvc") {
         $env:KIVY_GL_BACKEND = "angle_sdl2"
+      } else {
+        $env:KIVY_GL_BACKEND = "mock"
+      }
       python -m nose.core kivy/tests
       Check-Error
     }

kivy/tools/appveyor/kivy-upload.sh

@@ -1,7 +0,0 @@
-pacman -S --noconfirm git rsync
-if [ ! -d "/home/appveyor/.ssh" ]; then
-  mkdir "/home/appveyor/.ssh"
-fi
-echo -e "Host 159.203.106.198\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config
-cp $(cygpath -u "C:\projects\kivy\kivy\tools\appveyor\id_rsa") ~/.ssh/id_rsa
-rsync -avh -e "ssh -p 2458" "/c/kivy_wheels/" root@159.203.106.198:/web/downloads/appveyor/kivy