You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ja...@apache.org on 2014/06/25 23:12:03 UTC
git commit: Paralellize the unit tests
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 1fb88c0b5 -> 23893ead5
Paralellize the unit tests
patch by tjake; reviewed by thobbs for CASSANDRA-7442
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/23893ead
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/23893ead
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/23893ead
Branch: refs/heads/cassandra-2.1
Commit: 23893ead557989bb74d699e1028c9c6a1b689f3b
Parents: 1fb88c0
Author: Jake Luciani <ja...@apache.org>
Authored: Wed Jun 25 17:09:31 2014 -0400
Committer: Jake Luciani <ja...@apache.org>
Committed: Wed Jun 25 17:09:31 2014 -0400
----------------------------------------------------------------------
build.xml | 78 +++++++++++++++++++-
.../OffsetAwareConfigurationLoader.java | 63 ++++++++++++++++
test/unit/org/apache/cassandra/cli/CliTest.java | 3 +-
3 files changed, 140 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 1f975f0..c34bcfc 100644
--- a/build.xml
+++ b/build.xml
@@ -57,6 +57,7 @@
<property name="test.data" value="${test.dir}/data"/>
<property name="test.name" value="*Test"/>
<property name="test.methods" value=""/>
+ <property name="test.runners" value="1"/>
<property name="test.unit.src" value="${test.dir}/unit"/>
<property name="test.long.src" value="${test.dir}/long"/>
<property name="test.pig.src" value="${test.dir}/pig"/>
@@ -1060,11 +1061,13 @@
<macrodef name="testmacro">
<attribute name="suitename" />
<attribute name="inputdir" />
- <attribute name="timeout" />
+ <attribute name="timeout" default="${test.timeout}" />
<attribute name="forkmode" default="perTest"/>
<element name="optjvmargs" implicit="true" optional="true" />
<attribute name="filter" default="**/${test.name}.java"/>
<attribute name="exclude" default="" />
+ <attribute name="filelist" default="" />
+ <attribute name="poffset" default="0"/>
<sequential>
<echo message="running @{suitename} tests"/>
<mkdir dir="${build.test.dir}/cassandra"/>
@@ -1080,7 +1083,8 @@
<jvmarg value="-Xss256k"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>
<jvmarg value="-Dcassandra.test.use_prepared=${cassandra.test.use_prepared}"/>
- <optjvmargs/>
+ <jvmarg value="-Dcassandra.test.offsetseed=@{poffset}"/>
+ <optjvmargs/>
<classpath>
<path refid="cassandra.classpath" />
<pathelement location="${test.classes}"/>
@@ -1092,6 +1096,7 @@
</classpath>
<batchtest todir="${build.test.dir}/output">
<fileset dir="@{inputdir}" includes="@{filter}" excludes="@{exclude}"/>
+ <filelist dir="@{inputdir}" files="@{filelist}"/>
</batchtest>
</junit>
<fail message="Some @{suitename} test(s) failed.">
@@ -1134,7 +1139,7 @@
</junit>
</target>
- <target name="test" depends="build-test" description="Execute unit tests">
+ <target name="testold" depends="build-test" description="Execute unit tests">
<testmacro suitename="unit" inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
@@ -1146,6 +1151,17 @@
<exclude name="**/pig/*.java" />
</fileset>
</target>
+
+ <target name="testlist">
+ <testmacro suitename="${testlist.name}" inputdir="${test.unit.src}" filelist="${test.file.list}" poffset="${testlist.offset}" exclude="**/*.java" timeout="${test.timeout}">
+ <jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
+ <jvmarg value="-Dcorrupt-sstable-root=${test.data}/corrupt-sstables"/>
+ <jvmarg value="-Dmigration-sstable-root=${test.data}/migration-sstables"/>
+ <jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
+ <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
+ <jvmarg value="-Dcassandra.config.loader=org.apache.cassandra.OffsetAwareConfigurationLoader"/>
+ </testmacro>
+ </target>
<target name="testsome" depends="build-test" description="Execute specific unit tests" >
<testmacro suitename="unit" inputdir="${test.unit.src}" exclude="**/pig/*.java" timeout="${test.timeout}">
@@ -1334,6 +1350,62 @@
</create-javadoc>
</target>
+ <!-- Split test classes into n buckets and run across processes -->
+ <target name="test" depends="build-test" description="Parallel Test Runner">
+ <path id="all-test-classes-path">
+ <fileset dir="${test.unit.src}" excludes="**/pig/*.java" includes="**/${test.name}.java" />
+ </path>
+ <property name="all-test-classes" refid="all-test-classes-path"/>
+ <script language="javascript"> <![CDATA[
+ importClass(java.lang.Integer)
+ sep = project.getProperty("path.separator");
+ all = project.getProperty("all-test-classes").split(sep);
+ dir = project.getProperty("test.unit.src");
+
+ numRunners = parseInt(project.getProperty("test.runners"));
+
+ buckets = new Array(numRunners);
+ for (i = 0; i < all.length; i++) {
+ bucketNum = i % numRunners;
+ if (buckets[bucketNum] == undefined)
+ buckets[bucketNum] = "";
+ else
+ buckets[bucketNum] += ",";
+
+ buckets[bucketNum] += all[i];
+ }
+
+
+ var p = project.createTask('parallel');
+ p.setThreadCount(numRunners);
+
+ for (i = 0; i < buckets.length; i++) {
+
+ if (buckets[i] == undefined) continue;
+
+ task = project.createTask( 'antcall' );
+
+ task.setTarget("testlist");
+ param = task.createParam();
+ param.setName("test.file.list");
+ param.setValue(buckets[i]);
+
+ param = task.createParam();
+ param.setName("testlist.name");
+ param.setValue("test bucket "+i);
+
+ param = task.createParam();
+ param.setName("testlist.offset");
+ param.setValue(i);
+
+ p.addTask(task);
+ }
+
+ p.perform();
+
+]]> </script>
+ </target>
+
<!-- Generate Eclipse project description files -->
<target name="generate-eclipse-files" depends="build" description="Generate eclipse files">
<echo file=".project"><![CDATA[<?xml version="1.0" encoding="UTF-8"?>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
new file mode 100644
index 0000000..9023b11
--- /dev/null
+++ b/test/unit/org/apache/cassandra/OffsetAwareConfigurationLoader.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.YamlConfigurationLoader;
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+import java.io.File;
+
+
+public class OffsetAwareConfigurationLoader extends YamlConfigurationLoader
+{
+
+ static final String OFFSET_PROPERTY = "cassandra.test.offsetseed";
+ int offset = 0;
+
+ public OffsetAwareConfigurationLoader()
+ {
+ String offsetStr = System.getProperty(OFFSET_PROPERTY);
+
+ if (offsetStr == null)
+ throw new RuntimeException("offset property is not set: "+OFFSET_PROPERTY);
+
+ offset = Integer.valueOf(offsetStr);
+
+ assert offset >= 0;
+ }
+
+ @Override
+ public Config loadConfig() throws ConfigurationException
+ {
+ Config config = super.loadConfig();
+
+
+ config.rpc_port += offset;
+ config.native_transport_port += offset;
+ config.storage_port += offset;
+
+ config.commitlog_directory += File.pathSeparator + offset;
+ config.saved_caches_directory += File.pathSeparator + offset;
+ for (int i = 0; i < config.data_file_directories.length; i++)
+ config.data_file_directories[i] += File.pathSeparator + offset;
+
+
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/23893ead/test/unit/org/apache/cassandra/cli/CliTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/cli/CliTest.java b/test/unit/org/apache/cassandra/cli/CliTest.java
index 5d28c70..38f567b 100644
--- a/test/unit/org/apache/cassandra/cli/CliTest.java
+++ b/test/unit/org/apache/cassandra/cli/CliTest.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.cli;
import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.service.EmbeddedCassandraService;
import org.apache.cassandra.thrift.*;
@@ -226,7 +227,7 @@ public class CliTest extends SchemaLoader
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
// checking if we can connect to the running cassandra node on localhost
- CliMain.connect("127.0.0.1", 9170);
+ CliMain.connect("127.0.0.1", DatabaseDescriptor.getRpcPort());
// setting new output stream
CliMain.sessionState.setOut(new PrintStream(outStream));