You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2018/03/13 03:11:56 UTC
[3/6] hadoop git commit: HADOOP-14696. parallel tests don't work for
Windows. Contributed by Allen Wittenauer
HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
(cherry picked from commit 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b)
(cherry picked from commit 09940b1eb3b7ed764149f4a993c1857e9c6ad938)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a03c8ea6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a03c8ea6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a03c8ea6
Branch: refs/heads/branch-3.0
Commit: a03c8ea61f6e30a9d462571ace23858b6e0fd1c9
Parents: 430cdfe
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Mar 12 19:47:42 2018 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Mar 12 20:06:22 2018 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/pom.xml | 26 +----
.../apache/hadoop/test/GenericTestUtils.java | 68 +++++++++----
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +----
.../plugin/paralleltests/CreateDirsMojo.java | 100 +++++++++++++++++++
hadoop-tools/hadoop-aws/pom.xml | 26 +----
5 files changed, 161 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a03c8ea6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 82cd558..b085e36 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -979,30 +979,13 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-antrun-plugin</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
- <id>create-parallel-tests-dirs</id>
- <phase>test-compile</phase>
- <configuration>
- <target>
- <script language="javascript"><![CDATA[
- var baseDirs = [
- "${test.build.data}",
- "${test.build.dir}",
- "${hadoop.tmp.dir}" ];
- for (var i in baseDirs) {
- for (var j = 1; j <= ${testsThreadCount}; ++j) {
- var mkdir = project.createTask("mkdir");
- mkdir.setDir(new java.io.File(baseDirs[i], j));
- mkdir.perform();
- }
- }
- ]]></script>
- </target>
- </configuration>
+ <id>parallel-tests-createdir</id>
<goals>
- <goal>run</goal>
+ <goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@@ -1015,6 +998,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
+ <testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a03c8ea6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 72c8d41..b6f49da 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -286,7 +286,7 @@ public abstract class GenericTestUtils {
public static void assertExists(File f) {
Assert.assertTrue("File " + f + " should exist", f.exists());
}
-
+
/**
* List all of the files in 'dir' that match the regex 'pattern'.
* Then check that this list is identical to 'expectedMatches'.
@@ -294,7 +294,7 @@ public abstract class GenericTestUtils {
*/
public static void assertGlobEquals(File dir, String pattern,
String ... expectedMatches) throws IOException {
-
+
Set<String> found = Sets.newTreeSet();
for (File f : FileUtil.listFiles(dir)) {
if (f.getName().matches(pattern)) {
@@ -332,7 +332,7 @@ public abstract class GenericTestUtils {
+ StringUtils.stringifyException(t),
t);
}
- }
+ }
/**
* Wait for the specified test to return true. The test will be performed
@@ -482,18 +482,18 @@ public abstract class GenericTestUtils {
*/
public static class DelayAnswer implements Answer<Object> {
private final Log LOG;
-
+
private final CountDownLatch fireLatch = new CountDownLatch(1);
private final CountDownLatch waitLatch = new CountDownLatch(1);
private final CountDownLatch resultLatch = new CountDownLatch(1);
-
+
private final AtomicInteger fireCounter = new AtomicInteger(0);
private final AtomicInteger resultCounter = new AtomicInteger(0);
-
+
// Result fields set after proceed() is called.
private volatile Throwable thrown;
private volatile Object returnValue;
-
+
public DelayAnswer(Log log) {
this.LOG = log;
}
@@ -504,7 +504,7 @@ public abstract class GenericTestUtils {
public void waitForCall() throws InterruptedException {
fireLatch.await();
}
-
+
/**
* Tell the method to proceed.
* This should only be called after waitForCall()
@@ -512,7 +512,7 @@ public abstract class GenericTestUtils {
public void proceed() {
waitLatch.countDown();
}
-
+
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
LOG.info("DelayAnswer firing fireLatch");
@@ -541,7 +541,7 @@ public abstract class GenericTestUtils {
resultLatch.countDown();
}
}
-
+
/**
* After calling proceed(), this will wait until the call has
* completed and a result has been returned to the caller.
@@ -549,7 +549,7 @@ public abstract class GenericTestUtils {
public void waitForResult() throws InterruptedException {
resultLatch.await();
}
-
+
/**
* After the call has gone through, return any exception that
* was thrown, or null if no exception was thrown.
@@ -557,7 +557,7 @@ public abstract class GenericTestUtils {
public Throwable getThrown() {
return thrown;
}
-
+
/**
* After the call has gone through, return the call's return value,
* or null in case it was void or an exception was thrown.
@@ -565,20 +565,20 @@ public abstract class GenericTestUtils {
public Object getReturnValue() {
return returnValue;
}
-
+
public int getFireCount() {
return fireCounter.get();
}
-
+
public int getResultCount() {
return resultCounter.get();
}
}
-
+
/**
* An Answer implementation that simply forwards all calls through
* to a delegate.
- *
+ *
* This is useful as the default Answer for a mock object, to create
* something like a spy on an RPC proxy. For example:
* <code>
@@ -589,14 +589,14 @@ public abstract class GenericTestUtils {
* ...
* </code>
*/
- public static class DelegateAnswer implements Answer<Object> {
+ public static class DelegateAnswer implements Answer<Object> {
private final Object delegate;
private final Log log;
-
+
public DelegateAnswer(Object delegate) {
this(null, delegate);
}
-
+
public DelegateAnswer(Log log, Object delegate) {
this.log = log;
this.delegate = delegate;
@@ -636,7 +636,7 @@ public abstract class GenericTestUtils {
this.minSleepTime = minSleepTime;
this.maxSleepTime = maxSleepTime;
}
-
+
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
boolean interrupted = false;
@@ -666,11 +666,11 @@ public abstract class GenericTestUtils {
" but got:\n" + output,
Pattern.compile(pattern).matcher(output).find());
}
-
+
public static void assertValueNear(long expected, long actual, long allowedError) {
assertValueWithinRange(expected - allowedError, expected + allowedError, actual);
}
-
+
public static void assertValueWithinRange(long expectedMin, long expectedMax,
long actual) {
Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + ","
@@ -825,4 +825,28 @@ public abstract class GenericTestUtils {
failf(format, args);
}
}
+
+ /**
+ * Retreive the max number of parallel test threads when running under maven.
+ * @return int number of threads
+ */
+ public static int getTestsThreadCount() {
+ String propString = System.getProperty("testsThreadCount", "1");
+ int threadCount = 1;
+ if (propString != null) {
+ String trimProp = propString.trim();
+ if (trimProp.endsWith("C")) {
+ double multiplier = Double.parseDouble(
+ trimProp.substring(0, trimProp.length()-1));
+ double calculated = multiplier * ((double) Runtime
+ .getRuntime()
+ .availableProcessors());
+ threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
+ } else {
+ threadCount = Integer.parseInt(trimProp);
+ }
+ }
+ return threadCount;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a03c8ea6/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 6e062ef..36088dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -509,30 +509,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<build>
<plugins>
<plugin>
- <artifactId>maven-antrun-plugin</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
- <id>create-parallel-tests-dirs</id>
- <phase>test-compile</phase>
- <configuration>
- <target>
- <script language="javascript"><![CDATA[
- var baseDirs = [
- "${test.build.data}",
- "${test.build.dir}",
- "${hadoop.tmp.dir}" ];
- for (var i in baseDirs) {
- for (var j = 1; j <= ${testsThreadCount}; ++j) {
- var mkdir = project.createTask("mkdir");
- mkdir.setDir(new java.io.File(baseDirs[i], j));
- mkdir.perform();
- }
- }
- ]]></script>
- </target>
- </configuration>
+ <id>parallel-tests-createdir</id>
<goals>
- <goal>run</goal>
+ <goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@@ -545,6 +528,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
+ <testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a03c8ea6/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java
new file mode 100644
index 0000000..4c9b9d7
--- /dev/null
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.maven.plugin.paralleltests;
+
+import java.io.File;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+
+
+/**
+ * Goal which creates the parallel-test directories.
+ */
+@Mojo(name="parallel-tests-createdir",
+ defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
+public class CreateDirsMojo extends AbstractMojo {
+
+ /**
+ * Location of the test.build.dir.
+ */
+ @Parameter(defaultValue="${project.build.directory}/test-dir")
+ private File testBuildDir;
+
+ /**
+ * Location of the test.build.data.
+ */
+ @Parameter(defaultValue="${project.build.directory}/test-dir")
+ private File testBuildData;
+
+ /**
+ * Location of the test.build.data.
+ */
+ @Parameter(defaultValue="${project.build.directory}/tmp")
+ private File hadoopTmpDir;
+
+ /**
+ * Thread count.
+ */
+ @Parameter(defaultValue="${testsThreadCount}")
+ private String testsThreadCount;
+
+ public void execute() throws MojoExecutionException {
+ int numDirs=getTestsThreadCount();
+
+ mkParallelDirs(testBuildDir, numDirs);
+ mkParallelDirs(testBuildData, numDirs);
+ mkParallelDirs(hadoopTmpDir, numDirs);
+ }
+
+ /**
+ * Get the real number of parallel threads.
+ * @return int number of threads
+ */
+
+ public int getTestsThreadCount() {
+ int threadCount = 1;
+ if (testsThreadCount != null) {
+ String trimProp = testsThreadCount.trim();
+ if (trimProp.endsWith("C")) {
+ double multiplier = Double.parseDouble(
+ trimProp.substring(0, trimProp.length()-1));
+ double calculated = multiplier * ((double) Runtime
+ .getRuntime()
+ .availableProcessors());
+ threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
+ } else {
+ threadCount = Integer.parseInt(testsThreadCount);
+ }
+ }
+ return threadCount;
+ }
+
+ private void mkParallelDirs(File testDir, int numDirs)
+ throws MojoExecutionException {
+ for (int i=1; i<=numDirs; i++) {
+ File newDir = new File(testDir, String.valueOf(i));
+ if (!newDir.exists()) {
+ getLog().info("Creating " + newDir.toString());
+ if (!newDir.mkdirs()) {
+ throw new MojoExecutionException("Unable to create "
+ + newDir.toString());
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a03c8ea6/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 875db3e..1b94963 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -85,30 +85,13 @@
<build>
<plugins>
<plugin>
- <artifactId>maven-antrun-plugin</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
- <id>create-parallel-tests-dirs</id>
- <phase>test-compile</phase>
- <configuration>
- <target>
- <script language="javascript"><![CDATA[
- var baseDirs = [
- "${test.build.data}",
- "${test.build.dir}",
- "${hadoop.tmp.dir}" ];
- for (var i in baseDirs) {
- for (var j = 1; j <= ${testsThreadCount}; ++j) {
- var mkdir = project.createTask("mkdir");
- mkdir.setDir(new java.io.File(baseDirs[i], j));
- mkdir.perform();
- }
- }
- ]]></script>
- </target>
- </configuration>
+ <id>parallel-tests-createdir</id>
<goals>
- <goal>run</goal>
+ <goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@@ -121,6 +104,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
+ <testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org