From 01049a8f4536f42a5129e62863053c1a274e8665 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Mon, 1 May 2023 15:40:02 +0200 Subject: [PATCH] Perform RPi builds on `balenalib/raspberrypi3-*` images and skip `DISPMANX` API usage if can't be used (#8223) --- .ci/Dockerfile.armv7l | 38 ++++++++++------- .ci/ubuntu_ci.sh | 7 +--- .github/workflows/rpi_wheels.yml | 6 +-- setup.py | 71 ++++++++++++++++++++++++++++---- 4 files changed, 92 insertions(+), 30 deletions(-) diff --git a/.ci/Dockerfile.armv7l b/.ci/Dockerfile.armv7l index 35ba2ad99..6bc3b5acc 100644 --- a/.ci/Dockerfile.armv7l +++ b/.ci/Dockerfile.armv7l @@ -1,6 +1,8 @@ -ARG image=balenalib/armv7hf-debian:buster +ARG image=balenalib/raspberrypi3-debian-python:3.7-buster FROM $image +ENV KIVY_CROSS_PLATFORM=rpi + COPY . /kivy WORKDIR /kivy @@ -10,10 +12,18 @@ RUN [ "cross-build-start" ] RUN /bin/bash -c 'source .ci/ubuntu_ci.sh && \ export PIP_EXTRA_INDEX_URL="https://www.piwheels.org/simple" && \ install_ubuntu_build_deps && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install xorg wget libxrender-dev && \ - install_python && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install xorg wget libxrender-dev lsb-release libraspberrypi-dev raspberrypi-kernel-headers && \ install_kivy_test_run_pip_deps' +# If we're on Debian buster, we need to install cmake from backports as the cmake version +# in buster is too old to build sdl2 +RUN /bin/bash -c 'if [ "$(lsb_release -cs)" = "buster" ]; then \ + echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list; \ + apt-get update; \ + apt-get -y install -t buster-backports cmake; \ + fi' + + # Install patchelf (needed during delocation) RUN /bin/bash -c 'wget https://github.com/NixOS/patchelf/releases/download/0.17.2/patchelf-0.17.2-armv7l.tar.gz' RUN /bin/bash -c 'tar -xvf patchelf-0.17.2-armv7l.tar.gz -C /' @@ -22,22 +32,22 @@ RUN /bin/bash -c 'rm patchelf-0.17.2-armv7l.tar.gz' # Install auditwheel for delocation RUN /bin/bash -c 'pip install auditwheel' -# Download the Raspberry Pi firmware, so Raspberry Pi 1-3 can use the 'egl_rpi' window provider. -ARG KIVY_CROSS_PLATFORM="" -ARG KIVY_CROSS_SYSROOT="" -RUN if [ "$KIVY_CROSS_PLATFORM" = "rpi" ]; then \ - apt-get -y install git; \ - git clone --depth=1 https://github.com/raspberrypi/firmware "$KIVY_CROSS_SYSROOT"; \ - ln -s "$KIVY_CROSS_SYSROOT"/opt/vc "$KIVY_CROSS_SYSROOT"/usr; \ - fi # Build the dependencies (sdl) RUN ./tools/build_linux_dependencies.sh # Build the wheel. -RUN KIVY_SPLIT_EXAMPLES=1 USE_X11=1 USE_SDL2=1 USE_PANGOFT2=0 USE_GSTREAMER=0 KIVY_SDL_GL_ALPHA_SIZE=0 KIVY_DEPS_ROOT=$(pwd)/kivy-dependencies KIVY_CROSS_PLATFORM="$KIVY_CROSS_PLATFORM" KIVY_CROSS_SYSROOT="$KIVY_CROSS_SYSROOT" python3 -m pip -v wheel --extra-index-url https://www.piwheels.org/simple . -w /kivy-wheel +RUN KIVY_SPLIT_EXAMPLES=1 USE_X11=1 USE_SDL2=1 USE_PANGOFT2=0 USE_GSTREAMER=0 KIVY_SDL_GL_ALPHA_SIZE=0 KIVY_DEPS_ROOT=$(pwd)/kivy-dependencies KIVY_CROSS_PLATFORM="$KIVY_CROSS_PLATFORM" python3 -m pip -v wheel --extra-index-url https://www.piwheels.org/simple . -w /kivy-wheel -# Delocate the wheel -RUN /bin/bash -c 'auditwheel repair /kivy-wheel/Kivy-*.whl -w /kivy-delocated-wheel --plat manylinux_2_31_armv7l --no-update-tags --exclude libbrcmGLESv2.so --exclude libbcm_host.so --exclude libbrcmEGL.so' +# Delocate the wheel. +# if we're on buster, platform is manylinux_2_28_armv7l, if bullseye, manylinux_2_31_armv7l, otherwise let's just try manylinux2014_armv7l +RUN /bin/bash -c 'if [ "$(lsb_release -cs)" = "buster" ]; then \ + MANYLINUX_PLATFORM=manylinux_2_28_armv7l; \ + elif [ "$(lsb_release -cs)" = "bullseye" ]; then \ + MANYLINUX_PLATFORM=manylinux_2_31_armv7l; \ + else \ + MANYLINUX_PLATFORM=manylinux2014_armv7l; \ + fi; \ + auditwheel repair /kivy-wheel/Kivy-*.whl -w /kivy-delocated-wheel --plat "$MANYLINUX_PLATFORM" --no-update-tags --exclude libbrcmGLESv2.so --exclude libbcm_host.so --exclude libbrcmEGL.so' RUN [ "cross-build-end" ] diff --git a/.ci/ubuntu_ci.sh b/.ci/ubuntu_ci.sh index 37371e18b..cda910967 100644 --- a/.ci/ubuntu_ci.sh +++ b/.ci/ubuntu_ci.sh @@ -23,9 +23,6 @@ generate_sdist() { python3 -m pip uninstall cython -y } -install_python() { - sudo apt-get -y install python3 python3-dev python3-setuptools -} install_kivy_test_run_pip_deps() { curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py @@ -179,11 +176,11 @@ install_ubuntu_build_deps() { libegl1-mesa-dev libdbus-1-dev libibus-1.0-dev libudev-dev fcitx-libs-dev } -generate_armv7l_wheels() { +generate_rpi_wheels() { image=$1 mkdir dist - docker build -f .ci/Dockerfile.armv7l -t kivy/kivy-armv7l --build-arg image="$image" --build-arg KIVY_CROSS_PLATFORM="$2" --build-arg KIVY_CROSS_SYSROOT="$3" . + docker build -f .ci/Dockerfile.armv7l -t kivy/kivy-armv7l --build-arg image="$image" . docker cp "$(docker create kivy/kivy-armv7l)":/kivy-delocated-wheel . cp kivy-delocated-wheel/Kivy-* dist/ diff --git a/.github/workflows/rpi_wheels.yml b/.github/workflows/rpi_wheels.yml index 4d045abef..e440533f0 100644 --- a/.github/workflows/rpi_wheels.yml +++ b/.github/workflows/rpi_wheels.yml @@ -9,8 +9,6 @@ on: env: SERVER_IP: '159.203.106.198' - KIVY_CROSS_PLATFORM: 'rpi' - KIVY_CROSS_SYSROOT: '/rpi-firmware' jobs: raspberrypi: @@ -18,7 +16,7 @@ jobs: if: github.event_name == 'schedule' || (github.event_name == 'create' && github.event.ref_type == 'tag') || contains(github.event.head_commit.message, '[build wheel]') || contains(github.event.head_commit.message, '[build wheel armv7l]') || contains(github.event.pull_request.title, '[build wheel]') || contains(github.event.pull_request.title, '[build wheel armv7l]') strategy: matrix: - docker_images: ['balenalib/armv7hf-python:3.7-bullseye', 'balenalib/armv7hf-python:3.9-bullseye'] + docker_images: ['balenalib/raspberrypi3-debian-python:3.7-buster', 'balenalib/raspberrypi3-debian-python:3.9-bullseye'] steps: - uses: actions/checkout@v3 - name: Generate version metadata @@ -28,7 +26,7 @@ jobs: - name: Make ${{ matrix.docker_images }} wheel run: | source .ci/ubuntu_ci.sh - generate_armv7l_wheels ${{ matrix.docker_images }} "$KIVY_CROSS_PLATFORM" "$KIVY_CROSS_SYSROOT" + generate_rpi_wheels ${{ matrix.docker_images }} - name: Rename wheels if: github.event.ref_type != 'tag' run: | diff --git a/setup.py b/setup.py index f8db499ef..35e34afdd 100644 --- a/setup.py +++ b/setup.py @@ -19,8 +19,10 @@ from time import sleep from pathlib import Path import logging import sysconfig +import textwrap +import tempfile -from setuptools import Extension, find_packages, setup +from setuptools import Distribution, Extension, find_packages, setup from setuptools.command.build_ext import build_ext if sys.version_info[0] == 2: @@ -103,6 +105,43 @@ def get_isolated_env_paths(): return includes, libs +def check_c_source_compiles(code, include_dirs=None): + """Check if C code compiles. + This function can be used to check if a specific feature is available on + the current platform, and therefore enable or disable some modules. + """ + + def get_compiler(): + """Get the compiler instance used by setuptools. + This is a bit hacky, but seems the only way to get the compiler instance + used by setuptools, without using private APIs or the deprecated + distutils module. (See: https://github.com/pypa/setuptools/issues/2806) + """ + fake_dist_build_ext = Distribution().get_command_obj("build_ext") + fake_dist_build_ext.finalize_options() + # register an extension to ensure a compiler is created + fake_dist_build_ext.extensions = [Extension("ignored", ["ignored.c"])] + # disable building fake extensions + fake_dist_build_ext.build_extensions = lambda: None + # run to populate self.compiler + fake_dist_build_ext.run() + return fake_dist_build_ext.compiler + + # Create a temporary file which contains the code + with tempfile.TemporaryDirectory() as tmpdir: + temp_file = os.path.join(tmpdir, "test.c") + with open(temp_file, "w", encoding="utf-8") as tf: + tf.write(code) + try: + get_compiler().compile( + [temp_file], extra_postargs=[], include_dirs=include_dirs + ) + except Exception as ex: + print(ex) + return False + return True + + # ----------------------------------------------------------------------------- # Determine on which platform we are @@ -163,7 +202,7 @@ if KIVY_DEPS_ROOT is None and platform in ('linux', 'darwin'): # Detect options # c_options = OrderedDict() -c_options['use_rpi'] = platform == 'rpi' +c_options['use_rpi_vidcore_lite'] = platform == 'rpi' c_options['use_egl'] = False c_options['use_opengl_es2'] = None c_options['use_opengl_mock'] = environ.get('READTHEDOCS', None) == 'True' @@ -954,11 +993,29 @@ if c_options['use_avfoundation']: else: print('AVFoundation cannot be used, OSX >= 10.7 is required') -if c_options['use_rpi']: - sources['lib/vidcore_lite/egl.pyx'] = merge( - base_flags, gl_flags) - sources['lib/vidcore_lite/bcm.pyx'] = merge( - base_flags, gl_flags) +if c_options['use_rpi_vidcore_lite']: + + # DISPMANX is only available on old versions of Raspbian (Buster). + # For this reason, we need to be sure that EGL_DISPMANX_* is available + # before compiling the vidcore_lite module, even if we're on a RPi. + HAVE_DISPMANX = check_c_source_compiles( + textwrap.dedent( + """ + #include + #include + int main(int argc, char **argv) { + EGL_DISPMANX_WINDOW_T window; + bcm_host_init(); + } + """ + ), + include_dirs=gl_flags["include_dirs"], + ) + if HAVE_DISPMANX: + sources['lib/vidcore_lite/egl.pyx'] = merge( + base_flags, gl_flags) + sources['lib/vidcore_lite/bcm.pyx'] = merge( + base_flags, gl_flags) if c_options['use_x11']: libs = ['Xrender', 'X11']