From 3dc439945b9f0ddd96d88c304b4bcbdcb81bf0cc Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Fri, 17 Mar 2017 14:55:39 +0100 Subject: [PATCH 1/6] Build: restructure dependency builds for Linux This reduces the size of files to be cached between builds by the CI system. The scripts are flexible enough so they can also be used by developers locally. --- .gitignore | 5 ++ .travis.yml | 4 +- 3rdParty/README.md | 12 +++ 3rdParty/buildLinuxDependencies.sh | 123 +++++++++++++++++++++++++++++ 3rdParty/buildWxLinux.sh | 71 +++++++++++++++++ build/getWxWidgets.sh | 26 ------ 6 files changed, 213 insertions(+), 28 deletions(-) create mode 100644 3rdParty/README.md create mode 100755 3rdParty/buildLinuxDependencies.sh create mode 100644 3rdParty/buildWxLinux.sh delete mode 100755 build/getWxWidgets.sh diff --git a/.gitignore b/.gitignore index 6955ad75bf..ef53118a33 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,11 @@ drupal/sites/default/project ## translation system locale/*/*.flag +## dependency builds and CI cache directories +3rdParty/linux/ +3rdParty/mac/ +3rdParty/buildCache/ + .libs/ svn_version.h diff --git a/.travis.yml b/.travis.yml index e50ef70d55..597fc8b4c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ sudo: false cache: apt: true directories: - - build/wxWidgets-3.0.2 + - 3rdParty/buildCache addons: coverity_scan: @@ -80,5 +80,5 @@ script: - if [[ "${BOINC_TYPE}" == "server" ]]; then ( ./configure --disable-client --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "client" ]]; then ( ./configure --disable-server --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "apps" ]]; then ( ./configure --enable-apps --disable-server --disable-client --disable-manager && make ) fi -- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./build/getWxWidgets.sh && ./configure --disable-server --disable-client --with-wxdir=./build/wxWidgets-3.0.2/buildgtk && make ) fi - if [[ "${BOINC_TYPE}" == "libs-mingw" ]]; then ( cd lib && CC=/usr/bin/x86_64-w64-mingw32-gcc; CXX=/usr/bin/x86_64-w64-mingw32-g++; make -f Makefile.mingw ) fi +- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./3rdParty/buildLinuxDependencies.sh && ./configure --disable-server --disable-client --with-wxdir=./3rdParty/buildCache/linux && make ) fi diff --git a/3rdParty/README.md b/3rdParty/README.md new file mode 100644 index 0000000000..5182e86841 --- /dev/null +++ b/3rdParty/README.md @@ -0,0 +1,12 @@ +# Purpose of 3rdParty + +This directory is used to download and build dependencies for BOINC components. +Archives should be downloaded into a platform specific subdirectory (e.g linux, mac, ...) +to allow cross compilation of several platforms on a single host. + +Build products of dependencies should be installed into a corresponding subdirectory of +3rdParty/buildCache to allow CI systems to cache this data between BOINC builds. + +See the existing [buildLinuxDependencies.sh](buildLinuxDependencies.sh) and +[buildMacDependencies.sh](buildMacDependencies.sh) scripts for how this is done +on Linux and Mac currently. diff --git a/3rdParty/buildLinuxDependencies.sh b/3rdParty/buildLinuxDependencies.sh new file mode 100755 index 0000000000..b71bdcd43f --- /dev/null +++ b/3rdParty/buildLinuxDependencies.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# This file is part of BOINC. +# http://boinc.berkeley.edu +# Copyright (C) 2017 University of California +# +# BOINC is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# +# BOINC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with BOINC. If not, see . +# + +## support script to build a specific wxWidgets version for linux +## This script checks if a cached version is available and builts one if not. +## The build is done in 3rdParty/linux and usable files are installed to 3rdParty/buildCache/linux +## in order to keep the cache as small as possible +## The 3rdParty/buildCache directory can be cached by CI systems for all subsequent builds + +# checks if a given path is canonical (absolute and does not contain relative links) +# from http://unix.stackexchange.com/a/256437 +isPathCanonical() { + case "x$1" in + (x*/..|x*/../*|x../*|x*/.|x*/./*|x./*) + rc=1 + ;; + (x/*) + rc=0 + ;; + (*) + rc=1 + ;; + esac + return $rc +} + +# check working directory because the script needs to be called like: ./3rdParty/buildLinuxDependencies.sh +if [ ! -d "3rdParty" ]; then + echo "start this script in the source root directory" + exit 1 +fi + +ROOTDIR=$(pwd) +cache_dir="" +doclean="" +wxoption="" +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --cache_dir) + cache_dir="$2" + shift + ;; + --clean) + doclean="yes" + ;; + --debug) + wxoption="--debug" + ;; + *) + echo "unrecognized option $key" + ;; + esac + shift # past argument or value +done + +if [ "x$cache_dir" != "x" ]; then + if isPathCanonical "$cache_dir" && [ "$cache_dir" != "/" ]; then + PREFIX="$cache_dir" + else + echo "cache_dir must be an absolute path without ./ or ../ in it" + exit 1 + fi +else + PREFIX="$(pwd)/3rdParty/buildCache/linux" +fi + +download_and_build() { + cd "${ROOTDIR}/3rdParty/linux" || exit 1 + DIRNAME="${1}" + FILENAME="${2}" + DLURL="${3}" + BUILDSCRIPT="${4}" + FLAGFILE="${PREFIX}/${DIRNAME}_done" + if [ -e "${FLAGFILE}" ]; then + echo "${DIRNAME} seems already to be present in ${PREFIX}" + return 0 + fi + if [ ! -d ${DIRNAME} ]; then + if [ ! -e ${FILENAME} ]; then + wget ${DLURL} + fi + tar -xf ${FILENAME} + fi + cd ${DIRNAME} || exit 1 + source ${BUILDSCRIPT} --prefix ${PREFIX} + if [ $? -ne 0 ]; then exit 1; fi + cd ../.. || exit 1 + touch ${FLAGFILE} +} + +mkdir -p 3rdParty/linux +mkdir -p ${PREFIX} +cd "${ROOTDIR}/3rdParty/linux" || exit 1 + +if [ "${doclean}" = "yes" ]; then + echo "cleaning cache" + rm -rf ${PREFIX} + mkdir -p ${PREFIX} +fi + +#download_and_build $DIRNAME $FILENAME $DOWNLOADURL $BUILDSCRIPT +download_and_build "wxWidgets-3.0.2" "wxWidgets-3.0.2.tar.bz2" "https://sourceforge.net/projects/wxwindows/files/3.0.2/wxWidgets-3.0.2.tar.bz2" "${ROOTDIR}/3rdParty/buildWxLinux.sh ${wxoption}" + +# change back to root directory +cd ${ROOTDIR} || exit 1 diff --git a/3rdParty/buildWxLinux.sh b/3rdParty/buildWxLinux.sh new file mode 100644 index 0000000000..68cabe5d36 --- /dev/null +++ b/3rdParty/buildWxLinux.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# This file is part of BOINC. +# http://boinc.berkeley.edu +# Copyright (C) 2017 University of California +# +# BOINC is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# +# BOINC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with BOINC. If not, see . + +# Script to build a wxWidgets GTK version for BOINC + +# Usage: +# cd [path]/wxWidgets-3.0.2/ +# source path_to_this_script [--clean] [--debug] [--prefix PATH] +# +# the --clean argument will force a full rebuild. +# the --debug argument will build the debug version of the library +# if --prefix is given as absolute path the library is installed into there + + +doclean="" +configuration="Release" +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -clean|--clean) + doclean="yes" + ;; + -debug|--debug) + configuration="Debug" + ;; + -prefix|--prefix) + lprefix="$2" + shift + ;; + esac + shift # past argument or value +done + +debug_flag="--disable-debug_flag" +if [ $configuration = "Debug" ]; then + debug_flag="--enable-debug" +fi + +if [ "${doclean}" = "yes" ]; then + make clean +fi + +mkdir buildgtk +if [ $? -ne 0 ]; then exit 1; fi +cd buildgtk || return 1 + +../configure --prefix="${lprefix}" --with-gtk --disable-shared --enable-webview --disable-gtktest --disable-sdltest ${debug_flag} +if [ $? -ne 0 ]; then cd ..; return 1; fi +make 1>/dev/null # the wxWidgets build is very noisy so tune it down to warnings and errors only +if [ $? -ne 0 ]; then cd ..; return 1; fi +make install +if [ $? -ne 0 ]; then cd ..; return 1; fi + +cd .. +return 0 diff --git a/build/getWxWidgets.sh b/build/getWxWidgets.sh deleted file mode 100755 index 273a199b2c..0000000000 --- a/build/getWxWidgets.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -x - -## Travis-CI support script to built a specific wxWidgets version -## This script checks if a cached version is available and builts one if not. -## The directory is cached by Travis-CI for all subsequent runs of the VM - -# change to correct working directory because the script is called like: ./build/getWxWidgets.sh -cd ./build - -# if the versioning changes try to delete the old directory and change .travis.yml to cache the new directory -if [ ! -e "wxWidgets-3.0.2/build_complete" ]; then - wget https://sourceforge.net/projects/wxwindows/files/3.0.2/wxWidgets-3.0.2.tar.bz2 - tar -xf wxWidgets-3.0.2.tar.bz2 - cd wxWidgets-3.0.2 - mkdir buildgtk - cd buildgtk - ../configure --with-gtk --disable-shared --enable-webview --disable-gtktest --disable-sdltest - make - cd .. - touch build_complete - cd .. -fi - -# change back to root directory -cd .. From abd19d2671fa33959d3c25679fb53e5d8105f8a4 Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Mon, 20 Mar 2017 11:46:47 +0100 Subject: [PATCH 2/6] Build: patch linux wxWidgets to work with GCC6 From: https://github.com/wxWidgets/wxWidgets/commit/73e9e18ea09ffffcaac50237def0d9728a213c02 --- 3rdParty/buildWxLinux.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/3rdParty/buildWxLinux.sh b/3rdParty/buildWxLinux.sh index 68cabe5d36..15cc31a69a 100644 --- a/3rdParty/buildWxLinux.sh +++ b/3rdParty/buildWxLinux.sh @@ -27,6 +27,38 @@ # the --debug argument will build the debug version of the library # if --prefix is given as absolute path the library is installed into there +# Patch wxwidgets to work with GCC6 +# from https://github.com/wxWidgets/wxWidgets/commit/73e9e18ea09ffffcaac50237def0d9728a213c02 +if [ ! -f src/stc/scintilla/src/Editor.cxx.orig ]; then + cat >> /tmp/Editor.cxx.patch << ENDOFFILE +--- Editor.cxx 2014-10-06 23:33:44.000000000 +0200 ++++ Editor_patched.cxx 2017-03-20 10:24:14.776685161 +0100 +@@ -11,6 +11,7 @@ + #include + #include + ++#include + #include + #include + #include +@@ -5841,9 +5842,9 @@ + } + + static bool Close(Point pt1, Point pt2) { +- if (abs(pt1.x - pt2.x) > 3) ++ if (std::abs(pt1.x - pt2.x) > 3) + return false; +- if (abs(pt1.y - pt2.y) > 3) ++ if (std::abs(pt1.y - pt2.y) > 3) + return false; + return true; + } +ENDOFFILE + patch -blfu src/stc/scintilla/src/Editor.cxx /tmp/Editor.cxx.patch + rm -f /tmp/Editor.cxx.patch +else + echo "src/stc/scintilla/src/Editor.cxx already patched" +fi doclean="" configuration="Release" From a59f6bd76b2f65ef200f59f9a60df40d67f40fbb Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Mon, 20 Mar 2017 11:49:31 +0100 Subject: [PATCH 3/6] Build: fix wxWidgets Linux build to work without prefix Didn't work with an empty prefix. Also fixes behavior when --clean is specified since we use a special build directory. --- 3rdParty/buildWxLinux.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/3rdParty/buildWxLinux.sh b/3rdParty/buildWxLinux.sh index 15cc31a69a..02197074b8 100644 --- a/3rdParty/buildWxLinux.sh +++ b/3rdParty/buildWxLinux.sh @@ -62,6 +62,8 @@ fi doclean="" configuration="Release" +lprefix="" +cmdline_prefix="" while [[ $# -gt 0 ]]; do key="$1" case $key in @@ -73,6 +75,7 @@ while [[ $# -gt 0 ]]; do ;; -prefix|--prefix) lprefix="$2" + cmdline_prefix="--prefix=${lprefix}" shift ;; esac @@ -84,20 +87,20 @@ if [ $configuration = "Debug" ]; then debug_flag="--enable-debug" fi -if [ "${doclean}" = "yes" ]; then - make clean +if [ -d buildgtk ] && [ "${doclean}" = "yes" ]; then + rm -rf buildgtk fi - -mkdir buildgtk -if [ $? -ne 0 ]; then exit 1; fi +mkdir -p buildgtk cd buildgtk || return 1 -../configure --prefix="${lprefix}" --with-gtk --disable-shared --enable-webview --disable-gtktest --disable-sdltest ${debug_flag} +../configure "${cmdline_prefix}" --with-gtk --disable-shared --enable-webview --disable-gtktest --disable-sdltest ${debug_flag} if [ $? -ne 0 ]; then cd ..; return 1; fi make 1>/dev/null # the wxWidgets build is very noisy so tune it down to warnings and errors only if [ $? -ne 0 ]; then cd ..; return 1; fi -make install -if [ $? -ne 0 ]; then cd ..; return 1; fi +if [ "x${lprefix}" != "x" ]; then + make install + if [ $? -ne 0 ]; then cd ..; return 1; fi +fi cd .. return 0 From 84431d41a20de64e08d5fad1dc94cd2c8add48f8 Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Mon, 20 Mar 2017 11:53:12 +0100 Subject: [PATCH 4/6] Travis: use correct option for configure When using an installed version of wxWidgets one has to use --with-wx-prefix instead of --with-wxdir which is expecting the source directory. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 597fc8b4c2..bc141e2403 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,4 +81,4 @@ script: - if [[ "${BOINC_TYPE}" == "client" ]]; then ( ./configure --disable-server --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "apps" ]]; then ( ./configure --enable-apps --disable-server --disable-client --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "libs-mingw" ]]; then ( cd lib && CC=/usr/bin/x86_64-w64-mingw32-gcc; CXX=/usr/bin/x86_64-w64-mingw32-g++; make -f Makefile.mingw ) fi -- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./3rdParty/buildLinuxDependencies.sh && ./configure --disable-server --disable-client --with-wxdir=./3rdParty/buildCache/linux && make ) fi +- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./3rdParty/buildLinuxDependencies.sh && ./configure --disable-server --disable-client --with-wx-prefix=./3rdParty/buildCache/linux && make ) fi From abd808be6684cc211ac6046b6eb5f65adf104687 Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Mon, 20 Mar 2017 12:35:17 +0100 Subject: [PATCH 5/6] Travis: prefixes should always be absolute Configure assumes that prefixes are absolute without checking. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bc141e2403..b25e5ba918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,4 +81,4 @@ script: - if [[ "${BOINC_TYPE}" == "client" ]]; then ( ./configure --disable-server --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "apps" ]]; then ( ./configure --enable-apps --disable-server --disable-client --disable-manager && make ) fi - if [[ "${BOINC_TYPE}" == "libs-mingw" ]]; then ( cd lib && CC=/usr/bin/x86_64-w64-mingw32-gcc; CXX=/usr/bin/x86_64-w64-mingw32-g++; make -f Makefile.mingw ) fi -- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./3rdParty/buildLinuxDependencies.sh && ./configure --disable-server --disable-client --with-wx-prefix=./3rdParty/buildCache/linux && make ) fi +- if [[ "${BOINC_TYPE}" == "manager" ]]; then ( ./3rdParty/buildLinuxDependencies.sh && ./configure --disable-server --disable-client --with-wx-prefix=${TRAVIS_BUILD_DIR}/3rdParty/buildCache/linux && make ) fi From a964b72f129c2a3fb6596d2fec1d0e795aae58d5 Mon Sep 17 00:00:00 2001 From: Christian Beer Date: Tue, 11 Apr 2017 13:07:44 +0200 Subject: [PATCH 6/6] Build Linux: enable dependency debug build Automatically cleans buildCache when switching between Release and Debug builds. Mostly useful for local debugging. --- 3rdParty/buildLinuxDependencies.sh | 14 ++++++++++++++ 3rdParty/buildWxLinux.sh | 9 ++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/3rdParty/buildLinuxDependencies.sh b/3rdParty/buildLinuxDependencies.sh index b71bdcd43f..ae51b62ac0 100755 --- a/3rdParty/buildLinuxDependencies.sh +++ b/3rdParty/buildLinuxDependencies.sh @@ -51,6 +51,7 @@ ROOTDIR=$(pwd) cache_dir="" doclean="" wxoption="" +build_config="Release" while [[ $# -gt 0 ]]; do key="$1" case $key in @@ -63,6 +64,7 @@ while [[ $# -gt 0 ]]; do ;; --debug) wxoption="--debug" + build_config="Debug" ;; *) echo "unrecognized option $key" @@ -82,6 +84,17 @@ else PREFIX="$(pwd)/3rdParty/buildCache/linux" fi +if [ -f "${PREFIX}/build-config" ]; then + cur_config=$(<${PREFIX}/build-config) + if [ "${cur_config}" != "${build_config}" ]; then + doclean="yes" + wxoption="${wxoption} --clean" + fi +else + doclean="yes" + wxoption="${wxoption} --clean" +fi + download_and_build() { cd "${ROOTDIR}/3rdParty/linux" || exit 1 DIRNAME="${1}" @@ -114,6 +127,7 @@ if [ "${doclean}" = "yes" ]; then echo "cleaning cache" rm -rf ${PREFIX} mkdir -p ${PREFIX} + echo ${build_config} >${PREFIX}/build-config fi #download_and_build $DIRNAME $FILENAME $DOWNLOADURL $BUILDSCRIPT diff --git a/3rdParty/buildWxLinux.sh b/3rdParty/buildWxLinux.sh index 02197074b8..bfe2cf6959 100644 --- a/3rdParty/buildWxLinux.sh +++ b/3rdParty/buildWxLinux.sh @@ -61,7 +61,7 @@ else fi doclean="" -configuration="Release" +debug_flag="--disable-debug_flag" lprefix="" cmdline_prefix="" while [[ $# -gt 0 ]]; do @@ -71,7 +71,7 @@ while [[ $# -gt 0 ]]; do doclean="yes" ;; -debug|--debug) - configuration="Debug" + debug_flag="--enable-debug" ;; -prefix|--prefix) lprefix="$2" @@ -82,11 +82,6 @@ while [[ $# -gt 0 ]]; do shift # past argument or value done -debug_flag="--disable-debug_flag" -if [ $configuration = "Debug" ]; then - debug_flag="--enable-debug" -fi - if [ -d buildgtk ] && [ "${doclean}" = "yes" ]; then rm -rf buildgtk fi