You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2011/12/31 14:35:28 UTC
svn commit: r1226083 - in /hbase/trunk: dev-support/test-patch.sh pom.xml
src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
src/test/resources/hbase-site.xml
Author: tedyu
Date: Sat Dec 31 13:35:28 2011
New Revision: 1226083
URL: http://svn.apache.org/viewvc?rev=1226083&view=rev
Log:
HBASE-5064 utilize surefire tests parallelization (N Keywal)
Modified:
hbase/trunk/dev-support/test-patch.sh
hbase/trunk/pom.xml
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
hbase/trunk/src/test/resources/hbase-site.xml
Modified: hbase/trunk/dev-support/test-patch.sh
URL: http://svn.apache.org/viewvc/hbase/trunk/dev-support/test-patch.sh?rev=1226083&r1=1226082&r2=1226083&view=diff
==============================================================================
--- hbase/trunk/dev-support/test-patch.sh (original)
+++ hbase/trunk/dev-support/test-patch.sh Sat Dec 31 13:35:28 2011
@@ -567,9 +567,9 @@ runTests () {
failed_tests=""
### Kill any rogue build processes from the last attempt
$PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
- echo "$MVN clean test -D${PROJECT_NAME}PatchProcess"
+ echo "$MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess -Dsurefire.secondPartThreadCount=4"
ulimit -a
- $MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess
+ $MVN clean test -P runAllTests -D${PROJECT_NAME}PatchProcess -Dsurefire.secondPartThreadCount=4
if [[ $? != 0 ]] ; then
### Find and format names of failed tests
failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
Modified: hbase/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/pom.xml?rev=1226083&r1=1226082&r2=1226083&view=diff
==============================================================================
--- hbase/trunk/pom.xml (original)
+++ hbase/trunk/pom.xml Sat Dec 31 13:35:28 2011
@@ -627,6 +627,7 @@
<parallel>${surefire.firstPartParallel}</parallel>
<perCoreThreadCount>false</perCoreThreadCount>
<threadCount>${surefire.firstPartThreadCount}</threadCount>
+ <parallel>classes</parallel><!-- surefire hack, if not we're using method parallelisation class !-->
<groups>${surefire.firstPartGroups}</groups>
<testFailureIgnore>false</testFailureIgnore>
</configuration>
@@ -638,8 +639,10 @@
<configuration>
<skip>${surefire.skipSecondPart}</skip>
<testFailureIgnore>false</testFailureIgnore>
- <forkMode>always</forkMode>
- <parallel>none</parallel>
+ <forkMode>perThread</forkMode>
+ <perCoreThreadCount>false</perCoreThreadCount>
+ <threadCount>${surefire.secondPartThreadCount}</threadCount>
+ <parallel>classes</parallel><!-- surefire hack, if not we're using method parallelisation class !-->
<groups>${surefire.secondPartGroups}</groups>
</configuration>
</execution>
@@ -901,16 +904,17 @@
<unittest.include>**/Test*.java</unittest.include>
<integrationtest.include>**/IntegrationTest*.java</integrationtest.include>
- <surefire.version>2.11-TRUNK-HBASE-2</surefire.version>
+ <surefire.version>2.12-TRUNK-HBASE-2</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>
- <!-- default: run small & medium -->
+ <!-- default: run small & medium, medium with 2 threads -->
<surefire.skipFirstPart>false</surefire.skipFirstPart>
<surefire.skipSecondPart>false</surefire.skipSecondPart>
<surefire.firstPartForkMode>once</surefire.firstPartForkMode>
- <surefire.firstPartParallel>none</surefire.firstPartParallel>
+ <surefire.firstPartParallel>classes</surefire.firstPartParallel>
<surefire.firstPartThreadCount>1</surefire.firstPartThreadCount>
+ <surefire.secondPartThreadCount>2</surefire.secondPartThreadCount>
<surefire.firstPartGroups>org.apache.hadoop.hbase.SmallTests</surefire.firstPartGroups>
<surefire.secondPartGroups>org.apache.hadoop.hbase.MediumTests</surefire.secondPartGroups>
@@ -2000,6 +2004,7 @@
<surefire.firstPartForkMode>once</surefire.firstPartForkMode>
<surefire.firstPartParallel>none</surefire.firstPartParallel>
<surefire.firstPartThreadCount>1</surefire.firstPartThreadCount>
+ <surefire.secondPartThreadCount>2</surefire.secondPartThreadCount>
<surefire.skipFirstPart>false</surefire.skipFirstPart>
<surefire.skipSecondPart>false</surefire.skipSecondPart>
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1226083&r1=1226082&r2=1226083&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Sat Dec 31 13:35:28 2011
@@ -262,6 +262,11 @@ public class HBaseTestingUtility {
* instances -- another instance could grab the temporary
* value unintentionally -- but not anything can do about it at moment;
* single instance only is how the minidfscluster works.
+ *
+ * We also create the underlying directory for
+ * hadoop.log.dir, mapred.local.dir and hadoop.tmp.dir, and set the values
+ * in the conf, and as a system property for hadoop.tmp.dir
+ *
* @return The calculated data test build directory.
*/
private void setupDataTestDir() {
@@ -272,13 +277,62 @@ public class HBaseTestingUtility {
}
String randomStr = UUID.randomUUID().toString();
- Path testPath= new Path(
- getBaseTestDir(),
- randomStr
- );
+ Path testPath= new Path(getBaseTestDir(), randomStr);
dataTestDir = new File(testPath.toString()).getAbsoluteFile();
dataTestDir.deleteOnExit();
+
+ createSubDirAndSystemProperty(
+ "hadoop.log.dir",
+ testPath, "hadoop-log-dir");
+
+ // This is defaulted in core-default.xml to /tmp/hadoop-${user.name}, but
+ // we want our own value to ensure uniqueness on the same machine
+ createSubDirAndSystemProperty(
+ "hadoop.tmp.dir",
+ testPath, "hadoop-tmp-dir");
+
+ // Read and modified in org.apache.hadoop.mapred.MiniMRCluster
+ createSubDir(
+ "mapred.local.dir",
+ testPath, "mapred-local-dir");
+
+ createSubDirAndSystemProperty(
+ "mapred.working.dir",
+ testPath, "mapred-working-dir");
+ }
+
+ private void createSubDir(String propertyName, Path parent, String subDirName){
+ Path newPath= new Path(parent, subDirName);
+ File newDir = new File(newPath.toString()).getAbsoluteFile();
+ newDir.deleteOnExit();
+ conf.set(propertyName, newDir.getAbsolutePath());
+ }
+
+ private void createSubDirAndSystemProperty(
+ String propertyName, Path parent, String subDirName){
+
+ String sysValue = System.getProperty(propertyName);
+
+ if (sysValue != null) {
+ // There is already a value set. So we do nothing but hope
+ // that there will be no conflicts
+ LOG.info("System.getProperty(\""+propertyName+"\") already set to: "+
+ sysValue + " so I do NOT create it in "+dataTestDir.getAbsolutePath());
+ String confValue = conf.get(propertyName);
+ if (confValue != null && !confValue.endsWith(sysValue)){
+ LOG.warn(
+ propertyName + " property value differs in configuration and system: "+
+ "Configuration="+confValue+" while System="+sysValue+
+ " Erasing configuration value by system value."
+ );
+ }
+ conf.set(propertyName, sysValue);
+ } else {
+ // Ok, it's not set, so we create it as a subdirectory
+ createSubDir(propertyName, parent, subDirName);
+ System.setProperty(propertyName, conf.get(propertyName));
+ }
}
/**
@@ -1215,13 +1269,11 @@ public class HBaseTestingUtility {
public void startMiniMapReduceCluster(final int servers) throws IOException {
LOG.info("Starting mini mapreduce cluster...");
// These are needed for the new and improved Map/Reduce framework
- Configuration c = getConfiguration();
- System.setProperty("hadoop.log.dir", c.get("hadoop.log.dir"));
- c.set("mapred.output.dir", c.get("hadoop.tmp.dir"));
+ conf.set("mapred.output.dir", conf.get("hadoop.tmp.dir"));
mrCluster = new MiniMRCluster(servers,
- FileSystem.get(c).getUri().toString(), 1);
+ FileSystem.get(conf).getUri().toString(), 1);
LOG.info("Mini mapreduce cluster started");
- c.set("mapred.job.tracker",
+ conf.set("mapred.job.tracker",
mrCluster.createJobConf().get("mapred.job.tracker"));
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java?rev=1226083&r1=1226082&r2=1226083&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java Sat Dec 31 13:35:28 2011
@@ -80,10 +80,6 @@ public class TestTimeRangeMapRed {
@BeforeClass
public static void beforeClass() throws Exception {
- System.setProperty("hadoop.log.dir",
- UTIL.getConfiguration().get("hadoop.log.dir"));
- UTIL.getConfiguration().set("mapred.output.dir",
- UTIL.getConfiguration().get("hadoop.tmp.dir"));
UTIL.startMiniCluster();
}
Modified: hbase/trunk/src/test/resources/hbase-site.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/resources/hbase-site.xml?rev=1226083&r1=1226082&r2=1226083&view=diff
==============================================================================
--- hbase/trunk/src/test/resources/hbase-site.xml (original)
+++ hbase/trunk/src/test/resources/hbase-site.xml Sat Dec 31 13:35:28 2011
@@ -123,10 +123,6 @@
</description>
</property>
<property>
- <name>hadoop.log.dir</name>
- <value>${user.dir}/../logs</value>
- </property>
- <property>
<name>hbase.zookeeper.property.clientPort</name>
<value>21818</value>
<description>Property from ZooKeeper's config zoo.cfg.