From 6d7199af087263f0cad07053b0c40c93dfa93ed7 Mon Sep 17 00:00:00 2001 From: Bruce Allen Date: Wed, 7 Dec 2005 04:19:17 +0000 Subject: [PATCH] Added a script that maintains a set of sanatized scheduler logs for users, helpers, and admins to view on the web. A second script removes these log files if more than one week old. The logs are 'sanatized' by removing IP addresses and authenticators. These scripts should be controlled by using the standard start/stop scripts. To do this, make entries in config.xml. The files both contain comments at the top detailing how to do this. svn path=/trunk/boinc/; revision=9049 --- checkin_notes | 16 +++++++++++++ tools/Makefile.am | 2 +- tools/cleanlogs.sh | 18 ++++++++++++++ tools/makelog.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100755 tools/cleanlogs.sh create mode 100755 tools/makelog.sh diff --git a/checkin_notes b/checkin_notes index 52c140f145..8fcd3fc845 100755 --- a/checkin_notes +++ b/checkin_notes @@ -14262,3 +14262,19 @@ Charlie 6 Dec 2005 clientgui/ BOINCGUIApp.cpp + +Bruce 7 Dec 2005 + - Added a script that maintains a set of sanatized scheduler logs + for users, helpers, and admins to view on the web. + - A second script removes these log files if more than one week + old. + - The logs are 'sanatized' by removing IP addresses and authenticators. + - These scripts should be controlled by using the standard start/stop + scripts. To do this, make entries in config.xml. The files both + contain comments at the top detailing how to do this. + + tools/ + makelog.sh + cleanlogs.sh + Makefile.am + diff --git a/tools/Makefile.am b/tools/Makefile.am index 1d08926bc7..979578560e 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -4,7 +4,7 @@ include $(top_srcdir)/Makefile.incl bin_PROGRAMS = create_work sign_executable dir_hier_path dir_hier_move -EXTRA_DIST = make_project add xadd update_versions dbcheck_files_exist upgrade +EXTRA_DIST = make_project add xadd update_versions dbcheck_files_exist upgrade makelog.sh cleanlogs.sh # TODO: use libboinc for these: diff --git a/tools/cleanlogs.sh b/tools/cleanlogs.sh new file mode 100755 index 0000000000..eacfd7db96 --- /dev/null +++ b/tools/cleanlogs.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# This script ensures that only one week of the scheduler logs are +# archived for users to study. + +# To run this script as part of the standard BOINC backend services +# (once every 24 hours) just include this: +# +# cleanlogs.sh +# cleanlogs.sh.log +# 24 hr +# +# in your project's config.xml file + +cd ../html/user/sched_logs/ || exit 1 +find . -type d -name "20*" -mtime 7 | xargs rm -rf || exit 2 +echo "`date '+%Y-%m-%d %H:%M:'`" "cleaned scheduler logs" +exit 0 diff --git a/tools/makelog.sh b/tools/makelog.sh new file mode 100755 index 0000000000..95928b502a --- /dev/null +++ b/tools/makelog.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# This daemon (endless loop) constructs user-readable scheduler +# logs once per minute. It runs with a 3-minute latency. + +# To run this script as part of the standard BOINC backend +# components, just include: +# +# makelog.sh +# +# in your config.xml file + +export lastmin=dontmatch + +while [ true ] ; do + +# we extract the logs from 3 minutes ago, because the cgi +# process can run for a couple of minutes. + export currmin=`date --date="-3 minute" '+%Y-%m-%d %H:%M'` + +# create correct filename, directory name, path, etc. + export oneword=`echo $currmin | sed 's/ /_/'` + export filename=${oneword}.txt + export dirname=`echo $filename | cut -b 1-13` + export dirpath=../html/user/sched_logs/${dirname} + if [ ! -d $dirpath ] ; then + mkdir $dirpath + fi + export filepath=${dirpath}/${filename} + +# if the minute has changed, or the one-minute log file does not exist, +# then create it. One must consider both cases otherwise log rotation +# can screw things up. + if [ "${currmin}" != "${lastmin}" -o ! -f "${filepath}" ] ; then + +# put some text into the start of the 'human readable' log file + echo "Note that all times in this file are in UTC. To compare with your BOINC logs" > $filepath + echo "you will need to convert your local time into UTC. To make comparison easier you" >> $filepath + echo "may also want to consider using a high-precision time protocol such as NTP to set your" >> $filepath + echo "computers clock. This will allow comparisons of the time stamps to fractions of" >> $filepath + echo "a second." >> $filepath + echo " " >> $filepath + echo "Note also that these files are created with three-minute latency." >> $filepath + echo " " >> $filepath + echo " " >> $filepath + +# now grep for all log entries from 3 minutes ago. Use sed to hide any sensitive info +# such as authenticator and IP address. Must + grep --no-filename "${currmin}" ../log_*/cgi.log ../log_*/cgi.log.0 | sed 's/authenticator .*//g; s/\[auth [^]]*\]//g; s/from [0-9.]*//g; s/auth [0-9a-f]*\,//g; s/\[IP [0-9.]*\]//g; s/IP [0-9.]*\,//g' >> $filepath + export lastmin=$currmin + else + +# if the minute has not changed and log file exists, sleep a bit and +# try again + sleep 15; + fi +done + +# this is an infinite loop, so we should never get here!!