You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by ne...@apache.org on 2011/04/18 16:24:58 UTC
svn commit: r1094588 - in /subversion/trunk/tools/dev/benchmarks/suite1:
benchmark.py cronjob crontab.entry run
Author: neels
Date: Mon Apr 18 14:24:58 2011
New Revision: 1094588
URL: http://svn.apache.org/viewvc?rev=1094588&view=rev
Log:
Merge my changes with gstein's. Greg has added and tweaked "my"
benchmarks/suite1 (I like!), but I was oblivious for a few days and have
stubbornly continued to work on my private copy. This is changing now.
* tools/dev/benchmarks/suite1/benchmark.py
(TOTAL_RUN): Add to constify the 'TOTAL RUN' name.
(Timings.min_max_avg): move function out of Timings.compare_to() (code dup).
(Timings.summary): use Timings.min_max_avg().
(Timings.compare_to): use Timings.min_max_avg() and the TOTAL_RUN constant,
and slightly tweak output.
(run): Move the random seeding one loop level inwards to ensure N identical
runs being averaged. Use the TOTAL_RUN constant.
* tools/dev/benchmarks/suite1/run:
Hardcode my bin paths conditionally to my username (so I don't have to
keep editing when I update the VM). Also leave gstein's paths in there.
Don't print every single run, just the min-max-avg listings.
Don't print the output of 'combine', it is duplicated by 'compare'.
* tools/dev/benchmarks/suite1/cronjob,
* tools/dev/benchmarks/suite1/crontab.entry:
New files, as run on our ASF VM.
Added:
subversion/trunk/tools/dev/benchmarks/suite1/cronjob (with props)
subversion/trunk/tools/dev/benchmarks/suite1/crontab.entry
Modified:
subversion/trunk/tools/dev/benchmarks/suite1/benchmark.py
subversion/trunk/tools/dev/benchmarks/suite1/run
Modified: subversion/trunk/tools/dev/benchmarks/suite1/benchmark.py
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dev/benchmarks/suite1/benchmark.py?rev=1094588&r1=1094587&r2=1094588&view=diff
==============================================================================
--- subversion/trunk/tools/dev/benchmarks/suite1/benchmark.py (original)
+++ subversion/trunk/tools/dev/benchmarks/suite1/benchmark.py Mon Apr 18 14:24:58 2011
@@ -42,6 +42,7 @@ import optparse
DEFAULT_TIMINGS_PATH = './benchmark_py_last_run.py-pickle'
+TOTAL_RUN = 'TOTAL RUN'
timings = None
@@ -111,6 +112,12 @@ class Timings:
self.timings[name] = times
times.append(seconds)
+ def min_max_avg(self, name):
+ ttimings = self.timings.get(name)
+ return ( min(ttimings),
+ max(ttimings),
+ reduce(lambda x,y: x + y, ttimings) / len(ttimings) )
+
def summary(self):
s = []
if self.name:
@@ -123,16 +130,28 @@ class Timings:
timings = self.timings.get(name)
if not name or not timings: continue
- s.append('%5d %7.3f %7.3f %7.3f %s' % (
+ tmin, tmax, tavg = self.min_max_avg(name)
+
+ s.append('%5d %7.2f %7.2f %7.2f %s' % (
len(timings),
- min(timings),
- max(timings),
- reduce(lambda x,y: x + y, timings) / len(timings),
+ tmin,
+ tmax,
+ tavg,
name))
+
return '\n'.join(s)
def compare_to(self, other):
+ def do_div(a, b):
+ if b:
+ return float(a) / float(b)
+ else:
+ return 0.0
+
+ def do_diff(a, b):
+ return float(a) - float(b)
+
selfname = self.name
if not selfname:
selfname = 'unnamed'
@@ -140,56 +159,53 @@ class Timings:
if not othername:
othername = 'the other'
- s = ['COMPARE %s to %s'%(othername, selfname),
- ' 1.23|+0.45 means factor=1.23, difference in seconds = 0.45',
- ' factor < 1 or difference < 0 means \'%s\' is faster than \'%s\''
- % (self.name, othername)]
+ selftotal = self.min_max_avg(TOTAL_RUN)[2]
+ othertotal = other.min_max_avg(TOTAL_RUN)[2]
- s.append(' min max avg operation')
+ s = ['COMPARE %s to %s' % (othername, selfname)]
+
+ if TOTAL_RUN in self.timings and TOTAL_RUN in other.timings:
+ s.append(' %s times: %5.1f seconds avg for %s' % (TOTAL_RUN,
+ othertotal, othername))
+ s.append(' %s %5.1f seconds avg for %s' % (' ' * len(TOTAL_RUN),
+ selftotal, selfname))
- def do_div(a, b):
- if b:
- return float(a) / float(b)
- else:
- return 0.0
- def do_diff(a, b):
- return float(a) - float(b)
-
- def min_max_avg(ttimings):
- return ( min(ttimings),
- max(ttimings),
- reduce(lambda x,y: x + y, ttimings) / len(ttimings) )
+ s.append(' min max avg operation')
names = sorted(self.timings.keys())
for name in names:
- timings = self.timings.get(name)
- other_timings = other.timings.get(name)
- if not other_timings:
+ if not name in other.timings:
continue
- min_me, max_me, avg_me = min_max_avg(timings)
- min_other, max_other, avg_other = min_max_avg(other_timings)
+ min_me, max_me, avg_me = self.min_max_avg(name)
+ min_other, max_other, avg_other = other.min_max_avg(name)
s.append('%-16s %-16s %-16s %s' % (
- '%7.3f|%+7.4f' % (
+ '%7.2f|%+7.3f' % (
do_div(min_me, min_other),
do_diff(min_me, min_other)
),
- '%7.3f|%+7.4f' % (
+ '%7.2f|%+7.3f' % (
do_div(max_me, max_other),
do_diff(max_me, max_other)
),
- '%7.3f|%+7.4f' % (
+ '%7.2f|%+7.3f' % (
do_div(avg_me, avg_other),
do_diff(avg_me, avg_other)
),
name))
+
+ s.extend([
+ '("1.23|+0.45" means factor=1.23, difference in seconds = 0.45',
+ 'factor < 1 or difference < 0 means \'%s\' is faster than \'%s\')'
+ % (self.name, othername)])
+
return '\n'.join(s)
@@ -364,12 +380,12 @@ def propadd_tree(in_dir, fraction):
def run(levels, spread, N):
- # ensure identical modifications for every run of this script
- random.seed(0)
-
for i in range(N):
-
base = tempfile.mkdtemp()
+
+ # ensure identical modifications for every run
+ random.seed(0)
+
try:
repos = j(base, 'repos')
wc = j(base, 'wc')
@@ -466,7 +482,7 @@ def run(levels, spread, N):
print '\nDone with svn benchmark in', (stopped - started)
### timings comes from the global namespace; it should be passed
- timings.submit_timing('TOTAL RUN',
+ timings.submit_timing(TOTAL_RUN,
timedelta_to_seconds(stopped - started))
# rename ps to prop mod
Added: subversion/trunk/tools/dev/benchmarks/suite1/cronjob
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dev/benchmarks/suite1/cronjob?rev=1094588&view=auto
==============================================================================
--- subversion/trunk/tools/dev/benchmarks/suite1/cronjob (added)
+++ subversion/trunk/tools/dev/benchmarks/suite1/cronjob Mon Apr 18 14:24:58 2011
@@ -0,0 +1,78 @@
+#!/bin/bash
+# This is the cronjob as run on our ASF box aka svn-qavm.
+# It uses neels' mad bash script magic called 'pat' to update and
+# build the latest trunk, invokes a benchmark and sends as mail.
+
+#EMAILS=your@email.addresses
+EMAILS=""
+
+if [ "$USER" = "neels" ]; then
+ # I don't want to keep editing files after every update. ~Neels
+ EMAILS=dev@subversion.apache.org
+fi
+
+
+echo
+echo "--------------------------------------------------------------------"
+date
+echo
+
+results="$(tempfile)"
+
+
+# first update trunk to HEAD and rebuild.
+# update/build is logged to the cronjob log (via stdout)
+
+cd /home/neels/pat/trunk
+/home/neels/bin/pat update
+
+if [ "$?" != "0" ]; then
+ subject="Failed to update to HEAD."
+ echo "$subject" > "$results"
+ echo "$subject"
+else
+
+ rev="$(svn info /home/neels/pat/trunk/src | grep Revision)"
+ if [ -z "$rev" ]; then
+ subject="Working copy problem."
+ echo "$subject" > "$results"
+ echo "$subject"
+ else
+
+ /home/neels/bin/pat remake
+ if [ "$?" != "0" ]; then
+ subject="Failed to build $rev."
+ echo "$subject" > "$results"
+ echo "$subject"
+ else
+
+
+ # updating and building succeeded!
+ # run the benchmark:
+
+ compiled="$(/home/neels/pat/trunk/prefix/bin/svn --version | grep "compiled")"
+ subject="$rev$compiled"
+
+ cd /home/neels/svnbench/
+
+ # make more or less sure that runs don't leak into each other via
+ # I/O caching.
+ sync
+
+ # basically, just run it. But also, I want to
+ # - append output to stdout, for cronjob logging.
+ # - send output as mail, but only this run's output less update&build
+ "$(which time)" -p ./run 2>&1 | tee "$results"
+
+ fi
+ fi
+fi
+
+if [ -n "$EMAILS" ]; then
+ cat "$results" | mail -s "[svnbench] $subject" $EMAILS
+else
+ echo "No email addresses configured."
+fi
+
+rm "$results"
+
Propchange: subversion/trunk/tools/dev/benchmarks/suite1/cronjob
------------------------------------------------------------------------------
svn:executable = *
Added: subversion/trunk/tools/dev/benchmarks/suite1/crontab.entry
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dev/benchmarks/suite1/crontab.entry?rev=1094588&view=auto
==============================================================================
--- subversion/trunk/tools/dev/benchmarks/suite1/crontab.entry (added)
+++ subversion/trunk/tools/dev/benchmarks/suite1/crontab.entry Mon Apr 18 14:24:58 2011
@@ -0,0 +1,3 @@
+# m h dom mon dow command
+21 0 * * Mon /home/neels/svnbench/cronjob >>/home/neels/cronjob.log 2>&1
+
Modified: subversion/trunk/tools/dev/benchmarks/suite1/run
URL: http://svn.apache.org/viewvc/subversion/trunk/tools/dev/benchmarks/suite1/run?rev=1094588&r1=1094587&r2=1094588&view=diff
==============================================================================
--- subversion/trunk/tools/dev/benchmarks/suite1/run (original)
+++ subversion/trunk/tools/dev/benchmarks/suite1/run Mon Apr 18 14:24:58 2011
@@ -17,8 +17,14 @@
# specific language governing permissions and limitations
# under the License.
-SVN_1_6="$HOME/src/svn-1.6.x/subversion/svn/svn"
-SVN_trunk="$HOME/src/svn/subversion/svn/svn"
+# Where are the svn binaries you want to benchmark?
+if [ "$USER" = "neels" ]; then
+ SVN_1_6="$HOME/pat/stable/prefix/bin/svn"
+ SVN_trunk="$HOME/pat/trunk/prefix/bin/svn"
+else
+ SVN_1_6="$HOME/src/svn-1.6.x/subversion/svn/svn"
+ SVN_trunk="$HOME/src/svn/subversion/svn/svn"
+fi
benchmark="$PWD/benchmark.py"
@@ -34,10 +40,12 @@ batch(){
spread="$2"
N="$3"
pre="${levels}x${spread}_"
- "$benchmark" "--svn=${SVN_1_6}" run ${pre}1.6 $levels $spread $N
- "$benchmark" "--svn=${SVN_trunk}" run ${pre}trunk $levels $spread $N
- echo
+ echo
+ echo "---------------------------------------------------------------------"
+ echo
echo "Results for dir levels: $levels spread: $spread"
+ "$benchmark" "--svn=${SVN_1_6}" run ${pre}1.6 $levels $spread $N >/dev/null
+ "$benchmark" "--svn=${SVN_trunk}" run ${pre}trunk $levels $spread $N > /dev/null
"$benchmark" compare ${pre}1.6 ${pre}trunk
}
@@ -72,14 +80,9 @@ batch $cl $cs $N
echo ""
echo =========================================================================
echo ""
-echo "calculating total of 1.6..."
-"$benchmark" combine total_1.6 *x*_1.6
-
-echo ""
-echo "calculating total of trunk..."
-"$benchmark" combine total_trunk *x*_trunk
+"$benchmark" combine total_1.6 *x*_1.6 >/dev/null
+"$benchmark" combine total_trunk *x*_trunk >/dev/null
-echo ""
echo "comparing averaged totals..."
"$benchmark" compare total_1.6 total_trunk