You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spamassassin.apache.org by da...@apache.org on 2017/05/29 16:28:46 UTC
svn commit: r1796665 - in /spamassassin/trunk/build/mkupdates:
do-stable-update-with-scores run_nightly run_part2
Author: davej
Date: Mon May 29 16:28:46 2017
New Revision: 1796665
URL: http://svn.apache.org/viewvc?rev=1796665&view=rev
Log:
Merged run_part2 into the run_nightly.
Removed:
spamassassin/trunk/build/mkupdates/run_part2
Modified:
spamassassin/trunk/build/mkupdates/do-stable-update-with-scores
spamassassin/trunk/build/mkupdates/run_nightly
Modified: spamassassin/trunk/build/mkupdates/do-stable-update-with-scores
URL: http://svn.apache.org/viewvc/spamassassin/trunk/build/mkupdates/do-stable-update-with-scores?rev=1796665&r1=1796664&r2=1796665&view=diff
==============================================================================
--- spamassassin/trunk/build/mkupdates/do-stable-update-with-scores (original)
+++ spamassassin/trunk/build/mkupdates/do-stable-update-with-scores Mon May 29 16:28:46 2017
@@ -22,7 +22,7 @@ if [ $RV -ne 0 ] ; then
else
echo "Running mkupdate-with-scores..."
echo ""
- /usr/local/spamassassin/automc/svn/build/mkupdates/mkupdate-with-scores
+ /usr/local/spamassassin/automc/svn/trunk/build/mkupdates/mkupdate-with-scores
RV=$?
if [ $RV -ne 0 ] ; then
echo "Exit Status $RV is not zero for mkupdate-with-scores"
Modified: spamassassin/trunk/build/mkupdates/run_nightly
URL: http://svn.apache.org/viewvc/spamassassin/trunk/build/mkupdates/run_nightly?rev=1796665&r1=1796664&r2=1796665&view=diff
==============================================================================
--- spamassassin/trunk/build/mkupdates/run_nightly (original)
+++ spamassassin/trunk/build/mkupdates/run_nightly Mon May 29 16:28:46 2017
@@ -1,78 +1,226 @@
#!/bin/bash
-#
-# driver for cron.
-# 30 8 * * * automc bash /usr/local/spamassassin/svn/trunk/build/mkupdates/run_nightly > /var/www/bbmass.spamassassin.org/mkupdates/mkupdates.txt 2>&1
-set -x
+PERL=/usr/bin/perl
LOGDIR="/var/www/bbmass.spamassassin.org/mkupdates"
+REPDIR="/var/www/ruleqa.spamassassin.org/reports"
+UPDDIR="/var/www/bbmass.spamassassin.org/updates"
-cd /usr/local/spamassassin/svn/trunk/
+VERSIONS="3.3.3"
+
+cd /usr/local/spamassassin/automc/svn/trunk
. /etc/profile
-#Switched to system perl on spamassassin-vm box
-PERL=/usr/bin/perl
-#PERL=/local/perl586/bin/perl
-export PERL
-# Will this script be updated? If so, re-run it after updating.
-FLAG=NO
-if [ "`svn status -uq build/mkupdates/run_nightly | head -1 | cut -b8`" = "*" ]; then
- FLAG=YES
-fi
-
-svn cleanup # paranoia
-svn update || exit $?
-
-# Do the restart if we need to ...
-if [ "$FLAG" = "YES" ]; then
- #THIS IS NOT WORKING CURRENTLY 6-15-2014
- exec $0 "$@"
-
- # It shouldn't get here, but just in case ...
- exit 2
-fi
-
-$PERL ./build/mkupdates/listpromotable > rules/active.list.new || exit $?
-mv rules/active.list.new rules/active.list
-
-(
-echo "
-
-To commit this and proceed with update tarball creation:
-
- ssh sa-vm1.apache.org
- sudo -u automc /usr/local/spamassassin/svn/trunk/build/mkupdates/run_part2
-
-"
-
-svn diff
-) > $HOME/LATEST
-
-cat $HOME/LATEST
-# mailx -s "[admin] planned commit for active.list" \
-# dev@Spamassassin.apache.org < /home/updatesd/LATEST
-
-# cycle the logfiles; keep 6 (3 days worth I think)
-X=6
-[[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && rm -f $LOGDIR/mkupdates_${X}.txt
-while [[ $X -gt 0 ]]; do
- ((X--))
- Y=$((X+1))
- [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && mv -f $LOGDIR/mkupdates_${X}.txt $LOGDIR/mkupdates_${Y}.txt
-done
-mv -f $LOGDIR/mkupdates.txt $LOGDIR/mkupdates_${Y}.txt
+# ---------------------------------------------------------------------------
+
+promote_active_rules() {
+ pwd
+
+ $PERL build/mkupdates/listpromotable > rules/active.list.new || exit $?
+ mv rules/active.list.new rules/active.list
+
+ svn diff rules > $REPDIR/LATEST
+
+ cat $REPDIR/LATEST
+
+ echo "Committing promotions in rules/active.list..."
+ svn commit -m 'promotions validated' rules/active.list
+
+ # create a list of "bad" rules in the current sandboxes, updated daily
+ $PERL masses/rule-qa/list-bad-rules > $REPDIR/badrules.txt 2>&1
+
+ #On Wednesday's, we send out a bad sandbox rules report to the list
+ if [[ `date +%w` = 3 ]] ; then
+ (
+ echo "From: noreply@sa-vm1.apache.org (Rules Report Cron)"
+ echo "Subject: [auto] bad sandbox rules report"
+ echo
+ cat $REPDIR/badrules.txt
+ ) | /usr/sbin/sendmail -oi dev@spamassassin.apache.org
+ fi
+}
+
+# ---------------------------------------------------------------------------
+
+make_tarball_for_version() {
+ version="$1"
+
+ tmpdir=$HOME/tmp/stage/$version
+ rm -rf $tmpdir; mkdir -p $tmpdir || exit $?
+
+ # extract the new rules files.
+
+ # use "make install" logic, since we want rules as close as possible
+ # to what's installed
+ # TODO: this *would* be performed in a checkout of the desired
+ # version's branch. right now we're only using 1 version though
+ make clean
+ $PERL Makefile.PL PREFIX=$tmpdir < /dev/null || exit $?
+ make || exit $?
+
+ # remove new features, unsupported in existing code in the field
+ # (TODO: need a better way to exclude files that require new features
+ # like this; judicious use of "ifplugin" helps)
+ # rm rules/60_somerandomfeature.cf
+
+ # ensure the basic lint/rule-sanity test suite passes for this ruleset
+ # before we build an update from it. useful particularly to catch
+ # "tflags nopublish" leakage (bug 6297)
+ make test \
+ TEST_FILES="t/basic_lint.t t/basic_lint_without_sandbox.t t/basic_meta.t" \
+ || exit $?
+
+ # remove the rules files for rules we won't be shipping
+ rm rules/70_sandbox.cf rules/70_inactive.cf
+
+ # double check we still lint without those 2 files
+ ./spamassassin --lint || exit $?
+
+ rulesdir=`pwd`/rules
+
+ (
+ cd $rulesdir
+
+ # Use this to include plugin .pm files:
+ # tar cvf - *.cf *.pm || exit $?
+
+ # or this, to ban code from the updates:
+ tar cvf - *.cf || exit $?
-# create a list of "bad" rules in the current sandboxes, updated daily
-$PERL masses/rule-qa/list-bad-rules \
- > /var/www/ruleqa.spamassassin.org/reports/badrules.txt 2>&1
+ ) | gzip -9 > $tmpdir/update.tgz || exit $?
-#On Wednesday's, we send out a bad sandbox rules report to the list
-if [ `date +%w` = 3 ] ; then
+ # ensure non-empty
+ [ -s $tmpdir/update.tgz ] || exit 3
+
+ linttmp=$tmpdir/lintdir
+ rm -rf $linttmp
+ mkdir $linttmp
+ (
+ cd $linttmp
+ # check validity of tarball; also extract
+ gunzip -cd < $tmpdir/update.tgz | tar xf - || exit $?
+ )
+
+ sitetmp=$tmpdir/sitetmp
+ rm -rf $sitetmp
+ mkdir $sitetmp
+ cp rules/*.pre $sitetmp
+
+ # now, ensure the ruleset (entirely as distributed) lints, also.
+ # use "-p /dev/null" so any user_prefs data is ignored.
+ ./spamassassin -x --configpath=$linttmp --siteconfigpath=$sitetmp \
+ -p /dev/null --lint \
+ || exit $?
+
+ # sign and get sums
+ gpg --batch --homedir $HOME/key \
+ -bas $tmpdir/update.tgz || exit $?
+
+ $PERL build/sha1sum.pl $tmpdir/update.tgz > $tmpdir/update.tgz.sha1 || exit $?
+
+
+ # get SVN revision number.
+ # note: use 'Last Changed Rev' instead of 'Revision'. Because we share
+ # an SVN repository with other projects, this means that the same
+ # rev of *our* codebase may appear under multiple rev#s, as other projects
+ # check their changes in.
+
+ tagstamp=`date "+%Y%m%d%H%M%S"`
+ tagurl=https://svn.apache.org/repos/asf/spamassassin/tags/sa-update_${version}_${tagstamp}
+
+ # this svn copy is critical, to ensure each version's tarball has a different
+ # rev#. if you remove it, we need to prefix the version# to the svnrev# in
+ # the filenames instead so each version doesn't clobber others.
+ svn up
+ svn copy -m 'promotions validated' . $tagurl < /dev/null
+
+ # for svn 1.3:
+ # (svn info --non-interactive $tagurl || svn info $tagurl ) < /dev/null \
+ # > $tmpdir/svn 2>&1 || exit $?
+ # for crappy zone svn, 1.2:
(
- echo "From: noreply@sa-vm1.apache.org (Rules Report Cron)"
- echo "Subject: [auto] bad sandbox rules report"
- echo
- cat /var/www/ruleqa.spamassassin.org/reports/badrules.txt
- ) | /usr/sbin/sendmail -oi dev@spamassassin.apache.org
-fi
+ rm -rf tmpcheckout
+ svn co $tagurl tmpcheckout && svn info tmpcheckout
+ rm -rf tmpcheckout
+ ) < /dev/null > $tmpdir/svn 2>&1 || exit $?
+
+ svnrev=`(grep 'Last Changed Rev: ' $tmpdir/svn || exit 1) | \
+ sed -e 's/^.*: //'`
+
+ if [ "$svnrev" == "" ] ; then
+ echo "missing SVN revision"
+ cat $tmpdir/svn
+ exit 5
+ fi
+
+ if [ "$svnrev" -lt 1 ] ; then
+ echo "bad SVN revision: $svnrev"
+ cat $tmpdir/svn
+ exit 5
+ fi
+
+ mv $tmpdir/update.tgz $UPDDIR/$svnrev.tar.gz || exit $?
+ mv $tmpdir/update.tgz.sha1 $UPDDIR/$svnrev.tar.gz.sha1 || exit $?
+ mv $tmpdir/update.tgz.asc $UPDDIR/$svnrev.tar.gz.asc || exit $?
+
+ chmod 644 $UPDDIR/$svnrev.*
+
+
+ # next, create/update the new DNS record....
+
+ # turn "3.2.0" into "0.2.3"
+ rvers=`echo "$version" | rev`
+
+ /usr/local/bin/updateDNS.sh ${rvers}.updates TXT ${svnrev}
+
+ # clean up 4-day-old (and older) update tarballs. This seems as
+ # good a place as any to do this!
+ # note: for manual updates, the file permissions should be 0444 so let's clean
+ # out only 0644 (automatic) updates. a bit of a kluge, but ...
+ find $UPDDIR -mtime +4 -perm 0644 -type f -name '*.tar.*' -delete
+
+}
+
+# ---------------------------------------------------------------------------
+
+cycle_logfiles () {
+ # cycle the logfiles; keep 6 (3 days worth I think)
+ if [[ -e "$LOGDIR/mkupdates.txt" ]]; then
+ X=6
+ [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && rm -f $LOGDIR/mkupdates_${X}.txt
+ while [[ $X -gt 0 ]]; do
+ ((X--))
+ Y=$((X+1))
+ [[ -e "$LOGDIR/mkupdates_${X}.txt" ]] && mv -f $LOGDIR/mkupdates_${X}.txt $LOGDIR/mkupdates_${Y}.txt
+ done
+ mv -f $LOGDIR/mkupdates.txt $LOGDIR/mkupdates_${Y}.txt
+ fi
+}
+
+# ---------------------------------------------------------------------------
+
+[[ -d $UPDDIR ]] || echo "Updates dir '$UPDDIR' not found." 1>&2
+[[ -d $UPDDIR ]] || exit 6
+
+set -x
+
+promote_active_rules
+
+for VER in $VERSIONS; do
+ make_tarball_for_version $VER
+done
+
+set +x
+
+ls -l $UPDDIR/GPG.KEY
+ls -l $UPDDIR/MIRRORED.BY
+ls -ltr $UPDDIR/*.tar.* | tail -20
+
+cycle_logfiles
+
+rm -rf ruleqa.cache.*
+
+exit
+
+