You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by bh...@apache.org on 2014/01/06 20:09:57 UTC
[14/20] git commit: ACCUMULO-1944 Fix coverage for functional tests
ACCUMULO-1944 Fix coverage for functional tests
The -C flag for test/system/auto/run.py did not work. This changeset
allows it to work again if a functional Cobertura installation is
placed under $ACCUMULO_HOME/lib/test/cobertura.
The code for producing instrumented Accumulo code was inactive and
out of date. It was reworked so that, if -C is passed, the Accumulo
JARs are instrumented and placed into a location ahead of their
standard location in the test classpath. (If -C is not passed, any
instrumented JARs are removed.) The classpath is also dynamically
adjusted to include whatever Cobertura JAR is available; its name
includes a version number (as of 2.0.x).
The command-line scripts shipped with Cobertura 2.0.x are out of
date and do not work out of the box. Pull request #102 was submitted
to cobertura/cobertura on Github to fix the problem; in lieu of that,
the scripts must be manually updated to use a correct classpath.
Tested with Cobertura 2.0.3 / 2.0.4-SNAPSHOT with needed script
updates.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/950f144c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/950f144c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/950f144c
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 950f144c8e3517053af704189f0fdfd769487dfd
Parents: 9520ccb
Author: Bill Havanki <bh...@cloudera.com>
Authored: Mon Dec 2 15:33:56 2013 -0500
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon Jan 6 14:06:17 2014 -0500
----------------------------------------------------------------------
test/system/auto/TestUtils.py | 20 +++++++++++--
test/system/auto/run.py | 57 ++++++++++++++++++++++++++------------
2 files changed, 58 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/950f144c/test/system/auto/TestUtils.py
----------------------------------------------------------------------
diff --git a/test/system/auto/TestUtils.py b/test/system/auto/TestUtils.py
old mode 100755
new mode 100644
index 68d2a9c..5368f4e
--- a/test/system/auto/TestUtils.py
+++ b/test/system/auto/TestUtils.py
@@ -24,6 +24,7 @@ import socket
import signal
import select
import random
+import re
import shutil
import sleep
@@ -49,6 +50,13 @@ else:
SITE = "test-" + ID
SITE_PATH = os.path.join(ACCUMULO_CONF_DIR, SITE)
+COBERTURA_HOME = os.path.join(ACCUMULO_HOME, 'lib', 'test', 'cobertura')
+def findCoberturaJar():
+ jars = [f for f in os.listdir(COBERTURA_HOME) if re.search(r'cobertura.*\.jar', f)]
+ if len(jars) >= 1:
+ return jars[0]
+ return None
+
LOG_PROPERTIES= os.path.join(ACCUMULO_CONF_DIR, 'log4j.properties')
LOG_GENERIC = os.path.join(ACCUMULO_CONF_DIR, 'generic_logger.xml')
LOG_MONITOR = os.path.join(ACCUMULO_CONF_DIR, 'monitor_logger.xml')
@@ -63,6 +71,8 @@ $ACCUMULO_HOME/server/target/classes/,
$ACCUMULO_HOME/lib/accumulo-fate.jar,
$ACCUMULO_HOME/examples/simple/target/classes,
$ACCUMULO_HOME/lib/accumulo-examples-simple.jar,
+ $ACCUMULO_HOME/instrumented/[^.].*.jar,
+ $ACCUMULO_HOME/lib/test/cobertura/cobertura.jar,
$ACCUMULO_HOME/lib/[^.].*.jar,
$ZOOKEEPER_HOME/zookeeper[^.].*.jar,
$HADOOP_CONF_DIR,
@@ -256,8 +266,14 @@ class TestUtilsMixin:
self.pkill(host, 'accumulo.config.file', signal)
def create_config_file(self, settings):
+ cobertura_jar = findCoberturaJar()
+ if cobertura_jar:
+ settings_classpath = General_CLASSPATH.replace('cobertura.jar', os.path.basename(cobertura_jar))
+ else:
+ settings_classpath = General_CLASSPATH
+
fp = open(SITE_PATH, 'w')
- fp.write('<configuration>\n')
+ fp.write('<configuration>\n')
settings = self.settings.copy()
settings.update({ 'instance.zookeeper.host': ZOOKEEPERS,
'instance.dfs.dir': ACCUMULO_DIR,
@@ -265,7 +281,7 @@ class TestUtilsMixin:
'master.port.client': 41000 + FUZZ,
'monitor.port.client': 50099,
'gc.port.client': 45000 + FUZZ,
- 'general.classpaths' :General_CLASSPATH,
+ 'general.classpaths': settings_classpath,
'instance.secret': 'secret',
})
for a, v in settings.items():
http://git-wip-us.apache.org/repos/asf/accumulo/blob/950f144c/test/system/auto/run.py
----------------------------------------------------------------------
diff --git a/test/system/auto/run.py b/test/system/auto/run.py
index 3ff3465..2bda355 100755
--- a/test/system/auto/run.py
+++ b/test/system/auto/run.py
@@ -22,12 +22,12 @@ import logging
import unittest
import glob
import re
+import shutil
import sys
import socket
from subprocess import Popen, PIPE
-from TestUtils import ACCUMULO_HOME, ACCUMULO_DIR
-COBERTURA_HOME = os.path.join(ACCUMULO_HOME, 'lib', 'test', 'cobertura')
+from TestUtils import ACCUMULO_HOME, ACCUMULO_DIR, COBERTURA_HOME, findCoberturaJar
import sleep
log = logging.getLogger('test.auto')
@@ -109,26 +109,39 @@ def removeCoverageFromPreviousRun():
except OSError:
pass
+def classpath(dir):
+ return ':'.join([f for f in os.listdir(dir) if f.endswith('.jar')])
+
def instrumentAccumuloJar(jar):
- instrumented = jar[:-4] + "-instrumented" + ".jar"
- try:
- os.unlink(instrumented)
- except OSError:
- pass
- os.link(jar, instrumented)
+ basedir = os.path.join(ACCUMULO_HOME, 'lib')
+ instpath = os.path.join(ACCUMULO_HOME, 'instrumented')
+ if not os.access(instpath, os.F_OK):
+ os.mkdir(instpath)
+ auxcp = classpath(basedir)
+ jarname = os.path.basename(jar)
+ destjar = os.path.join(instpath, jarname)
+ if os.access(destjar, os.F_OK):
+ print "%s already instrumented" % jarname
+ return destjar
cmd = os.path.join(COBERTURA_HOME, "cobertura-instrument.sh")
- run(['sh', '-c', '%s --includeClasses "accumulo.*" %s' % (
- cmd, instrumented)])
+ print '- Instrumenting %s' % jarname
+ run(['sh', '-c', '%s --basedir %s --destination %s --auxClasspath %s %s' % (cmd, basedir, instpath, auxcp, jarname)])
assert os.path.exists('cobertura.ser')
- return instrumented
+ return destjar
+
+def removeInstrumentedAccumuloJars():
+ instpath = os.path.join(ACCUMULO_HOME, 'instrumented')
+ shutil.rmtree(instpath, ignore_errors=True)
def mergeCoverage():
"Most of the coverage ends up in $HOME due to ssh'ing around"
fname = 'cobertura.ser'
- run(['sh', '-c', ' '.join([
- os.path.join(COBERTURA_HOME, "cobertura-merge.sh"),
- os.path.join(os.environ['HOME'], fname),
- fname])])
+ if os.access(os.path.join(os.environ['HOME'], fname), os.F_OK):
+ run(['sh', '-c', ' '.join([
+ os.path.join(COBERTURA_HOME, "cobertura-merge.sh"),
+ os.path.join(os.environ['HOME'], fname),
+ fname])])
+
def produceCoverageReport(sourceDirectories):
reporter = os.path.join(COBERTURA_HOME, 'cobertura-report.sh')
@@ -285,11 +298,21 @@ def main():
map(suite.addTest, filtered)
if options.coverage:
+ cobertura_jar = os.path.join(COBERTURA_HOME, findCoberturaJar())
+ if not cobertura_jar or not os.access(cobertura_jar, os.F_OK):
+ print "Install Cobertura under %s" % COBERTURA_HOME
+ sys.exit(1)
fixCoberturaShellScripts()
removeCoverageFromPreviousRun()
- os.environ['HADOOP_CLASSPATH'] = os.path.join(COBERTURA_HOME,
- 'cobertura.jar')
+ os.environ['HADOOP_CLASSPATH'] = cobertura_jar
sleep.scale = 2.0
+ libpath = os.path.join(ACCUMULO_HOME,'lib')
+ libs = os.listdir(libpath)
+ for l in libs:
+ if re.search(r'^accumulo.*\.jar$', l):
+ instrumentAccumuloJar(os.path.join(libpath, l))
+ else:
+ removeInstrumentedAccumuloJars()
for i in range(options.repeat):
runner.run(suite)