diff --git a/deploy/cleanup_bintray.sh b/deploy/cleanup_bintray.sh new file mode 100755 index 0000000000..482a358e20 --- /dev/null +++ b/deploy/cleanup_bintray.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# This file is part of BOINC. +# http://boinc.berkeley.edu +# Copyright (C) 2018 University of California +# +# BOINC is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# +# BOINC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with BOINC. If not, see . +# + +## Clean up old files and versions on Bintray (https://bintray.com/docs/api/) +## Parameters: +## --weekly_threshold textual representation of the time to keep the weekly builds (default: 6 months, used as argument to date -d) +## --tmp_dir directory where API responses are stored (default: /tmp) + +# be carefull with set -x in this script because of ${BINTRAY_API_KEY} which needs to stay secret +set -e # Exit on errors +trap 'exit 1' ERR + +# check working directory because the script needs to be called like: ./deploy/cleanup_bintray.sh +if [ ! -d "deploy" ]; then + echo "start this script in the source root directory" + exit 1 +fi + +ROOTDIR=$(pwd) +TMPDIR="/tmp" +weekly_threshold="6 months" +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --tmp_dir) + TMPDIR="$2" + shift + ;; + --weekly_threshold) + weekly_threshold="$2" + shift + ;; + *) + echo "unrecognized option $key" + ;; + esac + shift # past argument or value +done + +command -v jq >/dev/null 2>&1 || { echo >&2 "jq (command-line json parser) is needed but not installed. Aborting."; exit 1; } + +# this needs to be set in the environment before starting the script +if [ "${BINTRAY_API_KEY}" == "" ] ; then + echo "BINTRAY_API_KEY is missing; doing nothing" + exit 0 +fi + +CI_RUN="${TRAVIS:-false}" +if [[ $CI_RUN == "true" ]]; then + echo "This script cannot be used in a CI environment" + exit 0 +fi + +BINTRAY_API=https://api.bintray.com +BINTRAY_USER="${BINTRAY_USER:-ChristianBeer}" +BINTRAY_API_KEY="$BINTRAY_API_KEY" # env +BINTRAY_REPO="${BINTRAY_REPO:-boinc-ci}" +BINTRAY_REPO_OWNER="${BINTRAY_REPO_OWNER:-boinc}" # owner and user not always the same + +GITHUB_API=https://api.github.com +GITHUB_USER="${GITHUB_USER:-BOINC}" +# github has a rate limit of 60 requests per hour for non authenticated requests +# see https://developer.github.com/v3/#authentication on how to authenticate +#GITHUB_OAUTH="-H \"Authorization: token OAUTH-TOKEN\"" +GITHUB_OAUTH="" +GITHUB_REPO="${GITHUB_REPO:-boinc}" + +BINTRAY_CURL="curl -s -S -u${BINTRAY_USER}:${BINTRAY_API_KEY} -H Accept:application/json -w \n" +GITHUB_CURL="curl -s -S ${GITHUB_OAUTH} -H Accept:application/vnd.github.v3+json -w \n" + +# get quotas from bintray +${BINTRAY_CURL} "${BINTRAY_API}/orgs/${BINTRAY_REPO_OWNER}" -o "${TMPDIR}/organisation.txt" + +quota_used=$(jq '.quota_used_bytes' < "${TMPDIR}/organisation.txt") +free_storage=$(jq '.free_storage' < "${TMPDIR}/organisation.txt") +free_storage_quota_limit=$(jq '.free_storage_quota_limit' < "${TMPDIR}/organisation.txt") +storage_percent=$(echo "scale=2; $quota_used / $free_storage_quota_limit *100" | bc) +echo "Storage quota used: ${storage_percent}%" + +last_month_download=$(jq '.last_month_free_downloads' < "${TMPDIR}/organisation.txt") +monthly_download_quota=$(jq '.monthly_free_downloads_quota_limit' < "${TMPDIR}/organisation.txt") +download_percent=$(echo "scale=2; $last_month_download / $monthly_download_quota *100" | bc) +echo "Download quota used: ${download_percent}%" + +rm -f "${TMPDIR}/organisation.txt" + +BINTRAY_PACKAGE="pull-requests" +echo "Cleaning package: $BINTRAY_PACKAGE" + +${BINTRAY_CURL} "${BINTRAY_API}/packages/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}" -o "${TMPDIR}/package_pr.txt" + +pr_ids=$(jq -r .versions[]? < "${TMPDIR}/package_pr.txt" | cut -d_ -f1 | cut -b 3- | sort -u) +for pr in $pr_ids +do + echo $pr + state=$(${GITHUB_CURL} "${GITHUB_API}/repos/${GITHUB_USER}/${GITHUB_REPO}/pulls/${pr}" | jq -r .state) + if [ "$state" = "closed" ]; then + echo " is closed" + data="[{\"PR\": [\"${pr}\"]}]" + ${BINTRAY_CURL} -H Content-Type:application/json -X POST -d "${data}" "${BINTRAY_API}/search/attributes/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions" -o "${TMPDIR}/versions.txt" + versions=$(jq -r .[].name < "${TMPDIR}/versions.txt") + for version in $versions + do + echo "Deleting ${version}" + ${BINTRAY_CURL} -X DELETE "${BINTRAY_API}/packages/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions/${version}" + done + fi +done + +rm -f "${TMPDIR}/package_pr.txt" "${TMPDIR}/versions.txt" +echo "Finished cleaning package: $BINTRAY_PACKAGE" + +BINTRAY_PACKAGE="weekly" +echo "Cleaning package: $BINTRAY_PACKAGE" + +time_threshold=$(date -u -d "now-${weekly_threshold}" +%s) +data="[{\"create_time_utc\": \"[,${time_threshold}]\"}]" + +${BINTRAY_CURL} -H Content-Type:application/json -X POST -d "${data}" "${BINTRAY_API}/search/attributes/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions?attribute_values=1" -o "${TMPDIR}/versions.txt" +versions=$(jq -r .[].name? < "${TMPDIR}/versions.txt") +for version in $versions +do + echo "Deleting ${version}" + ${BINTRAY_CURL} -X DELETE "${BINTRAY_API}/packages/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions/${version}" +done + +rm -f "${TMPDIR}/versions.txt" +echo "Finished cleaning package: $BINTRAY_PACKAGE" diff --git a/deploy/deploy_to_bintray.sh b/deploy/deploy_to_bintray.sh index 343e562f76..b0b21bad56 100755 --- a/deploy/deploy_to_bintray.sh +++ b/deploy/deploy_to_bintray.sh @@ -114,9 +114,9 @@ ${CURL} -H Content-Type:application/json -X POST -d "${data}" "${API}/packages/$ echo "Adding attributes to version ${VERSION}" # this can be used to cleanup old versions pr="${TRAVIS_PULL_REQUEST:-0}" -created=$(date -Is) +created=$(date -u +%s) data="[{\"name\": \"PR\", \"values\": [\"${pr}\"], \"type\": \"string\"}, - {\"name\": \"create_time\", \"values\": [\"${created}\"], \"type\": \"date\"}]" + {\"name\": \"create_time_utc\", \"values\": [${created}], \"type\": \"number\"}]" echo $data ${CURL} -H Content-Type:application/json -X POST -d "${data}" "${API}/packages/${BINTRAY_REPO_OWNER}/${BINTRAY_REPO}/${PKG_NAME}/versions/${VERSION}/attributes"