You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by aw...@apache.org on 2017/07/11 18:29:25 UTC
[2/6] cassandra git commit: Set test.runners based on cores and
memory size
Set test.runners based on cores and memory size
patch by Jay Zhuang; reviewed by Ariel Weisberg for CASSANDRA-13078
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2400d07b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2400d07b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2400d07b
Branch: refs/heads/cassandra-3.11
Commit: 2400d07bf52829b25a7c03c19b22ddd3301899be
Parents: 97fb4d1
Author: Jay Zhuang <ja...@yahoo.com>
Authored: Thu Jul 6 18:01:38 2017 -0700
Committer: Ariel Weisberg <aw...@apple.com>
Committed: Tue Jul 11 14:08:14 2017 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
build.xml | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
circle.yml | 2 +-
3 files changed, 55 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 8095e25..ce2324d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.0.15
+ * Set test.runners based on cores and memory size (CASSANDRA-13078)
* Allow different NUMACTL_ARGS to be passed in (CASSANDRA-13557)
* Allow native function calls in CQLSSTableWriter (CASSANDRA-12606)
* Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 5eb6572..53c2cea 100644
--- a/build.xml
+++ b/build.xml
@@ -59,7 +59,6 @@
<property name="test.classlistfile" value="testlist.txt"/>
<property name="benchmark.name" value=""/>
<property name="test.methods" value=""/>
- <property name="test.runners" value="1"/>
<property name="test.unit.src" value="${test.dir}/unit"/>
<property name="test.long.src" value="${test.dir}/long"/>
<property name="test.burn.src" value="${test.dir}/burn"/>
@@ -1625,12 +1624,25 @@
<scriptdef name="testparallelhelper" language="javascript">
<attribute name="testdelegate"/>
<![CDATA[
- var Integer = java.lang.Integer;
sep = project.getProperty("path.separator");
all = project.getProperty("all-test-classes").split(sep);
- dir = project.getProperty("test.unit.src");
+ runners = project.getProperty("test.runners")
+ cores = project.getProperty("cores.count")
+ mem = project.getProperty("mem.size")
- numRunners = parseInt(project.getProperty("test.runners"));
+ numRunners = 1
+ if (runners != null) // there's test.runners override
+ numRunners = parseInt(runners) || 1;
+ else if (cores != null && mem != null) // only if cores and memory size is set
+ numRunners = Math.min(Math.floor(Math.sqrt(parseInt(cores) || 1)),
+ Math.floor((parseInt(mem) || 1)/(4*1024*1024*1024)));
+
+ if (numRunners < 1)
+ numRunners = 1
+
+ var echo = project.createTask("echo");
+ echo.setMessage("Number of test runners: " + numRunners);
+ echo.perform();
var p = project.createTask('parallel');
p.setThreadCount(numRunners);
@@ -1652,7 +1664,42 @@
]]>
</scriptdef>
- <target name="test" depends="build-test" description="Parallel Test Runner">
+ <target name="get-cores">
+ <property environment="env"/>
+ <!-- support for Windows -->
+ <condition property="cores.count" value="${env.NUMBER_OF_PROCESSORS}">
+ <os family="windows" />
+ </condition>
+ <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) -->
+ <exec executable="nproc" outputproperty="cores.count" os="Linux,SunOS,Solaris" failifexecutionfails="false">
+ <arg value="--all"/>
+ </exec>
+ <!-- support for Mac OS X -->
+ <exec executable="sysctl" outputproperty="cores.count" os="Mac,Mac OS X,Darwin" failifexecutionfails="false">
+ <arg value="-n"/>
+ <arg value="hw.ncpu"/>
+ </exec>
+ <echo message="Number of cores: ${cores.count}"/>
+ </target>
+
+ <target name="get-mem">
+ <condition property="mem.size" value="unknown">
+ <os family="windows" />
+ </condition>
+ <!-- support for Linux and Solaris (package SUNWgnu-coreutils is required) -->
+ <exec executable="bash" outputproperty="mem.size" os="Linux,SunOS,Solaris" failifexecutionfails="false">
+ <arg value="-c"/>
+ <arg value="free -b | grep Mem: | awk '{print $2}'"/>
+ </exec>
+ <!-- support for Mac OS X -->
+ <exec executable="sysctl" outputproperty="mem.size" os="Mac,Mac OS X,Darwin" failifexecutionfails="false">
+ <arg value="-n"/>
+ <arg value="hw.memsize"/>
+ </exec>
+ <echo message="Mem size : ${mem.size}"/>
+ </target>
+
+ <target name="test" depends="build-test,get-cores,get-mem" description="Parallel Test Runner">
<path id="all-test-classes-path">
<fileset dir="${test.unit.src}" includes="**/${test.name}.java" />
</path>
@@ -1933,4 +1980,5 @@
file="${build.dir}/${final.name}-javadoc.jar"
classifier="javadoc"/>
</target>
+
</project>
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2400d07b/circle.yml
----------------------------------------------------------------------
diff --git a/circle.yml b/circle.yml
index 9d31277..f4801b7 100644
--- a/circle.yml
+++ b/circle.yml
@@ -7,7 +7,7 @@ test:
- sudo apt-get update; sudo apt-get install wamerican:
parallel: true
override:
- - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test ;; 1) ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac:
+ - case $CIRCLE_NODE_INDEX in 0) ant eclipse-warnings; ant test -Dtest.runners=1;; 1) ant long-test ;; 2) ant test-compression ;; 3) ant stress-test ;;esac:
parallel: true
post:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org