You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/07/12 17:53:39 UTC
svn commit: r1360737 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/ hadoop-yarn/...
Author: bobby
Date: Thu Jul 12 15:53:38 2012
New Revision: 1360737
URL: http://svn.apache.org/viewvc?rev=1360737&view=rev
Log:
svn merge -c 1360735 FIXES: MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1360737&r1=1360736&r2=1360737&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Thu Jul 12 15:53:38 2012
@@ -86,6 +86,8 @@ Release 2.0.1-alpha - UNRELEASED
MAPREDUCE-3993. Graceful handling of codec errors during decompression
(kkambatl via tucu)
+ MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java?rev=1360737&r1=1360736&r2=1360737&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java Thu Jul 12 15:53:38 2012
@@ -35,16 +35,23 @@ import org.apache.hadoop.mapreduce.serve
import junit.framework.TestCase;
public class TestIndexCache extends TestCase {
+ private JobConf conf;
+ private FileSystem fs;
+ private Path p;
+
+ @Override
+ public void setUp() throws IOException {
+ conf = new JobConf();
+ fs = FileSystem.getLocal(conf).getRaw();
+ p = new Path(System.getProperty("test.build.data", "/tmp"),
+ "cache").makeQualified(fs.getUri(), fs.getWorkingDirectory());
+ }
public void testLRCPolicy() throws Exception {
Random r = new Random();
long seed = r.nextLong();
r.setSeed(seed);
System.out.println("seed: " + seed);
- JobConf conf = new JobConf();
- FileSystem fs = FileSystem.getLocal(conf).getRaw();
- Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "cache").makeQualified(fs);
fs.delete(p, true);
conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
final int partsPerMap = 1000;
@@ -115,10 +122,6 @@ public class TestIndexCache extends Test
public void testBadIndex() throws Exception {
final int parts = 30;
- JobConf conf = new JobConf();
- FileSystem fs = FileSystem.getLocal(conf).getRaw();
- Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "cache").makeQualified(fs);
fs.delete(p, true);
conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
IndexCache cache = new IndexCache(conf);
@@ -150,10 +153,6 @@ public class TestIndexCache extends Test
}
public void testInvalidReduceNumberOrLength() throws Exception {
- JobConf conf = new JobConf();
- FileSystem fs = FileSystem.getLocal(conf).getRaw();
- Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "cache").makeQualified(fs);
fs.delete(p, true);
conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
final int partsPerMap = 1000;
@@ -199,10 +198,6 @@ public class TestIndexCache extends Test
// This test case may not repeatable. But on my macbook this test
// fails with probability of 100% on code before MAPREDUCE-2541,
// so it is repeatable in practice.
- JobConf conf = new JobConf();
- FileSystem fs = FileSystem.getLocal(conf).getRaw();
- Path p = new Path(System.getProperty("test.build.data", "/tmp"),
- "cache").makeQualified(fs);
fs.delete(p, true);
conf.setInt(TTConfig.TT_INDEX_CACHE, 10);
// Make a big file so removeMapThread almost surely runs faster than
@@ -247,6 +242,66 @@ public class TestIndexCache extends Test
}
}
+ public void testCreateRace() throws Exception {
+ fs.delete(p, true);
+ conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+ final int partsPerMap = 1000;
+ final int bytesPerFile = partsPerMap * 24;
+ final IndexCache cache = new IndexCache(conf);
+
+ final Path racy = new Path(p, "racyIndex");
+ final String user =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ writeFile(fs, racy, bytesPerFile, partsPerMap);
+
+ // run multiple instances
+ Thread[] getInfoThreads = new Thread[50];
+ for (int i = 0; i < 50; i++) {
+ getInfoThreads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ cache.getIndexInformation("racyIndex", partsPerMap, racy, user);
+ cache.removeMap("racyIndex");
+ } catch (Exception e) {
+ // should not be here
+ }
+ }
+ };
+ }
+
+ for (int i = 0; i < 50; i++) {
+ getInfoThreads[i].start();
+ }
+
+ final Thread mainTestThread = Thread.currentThread();
+
+ Thread timeoutThread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(15000);
+ mainTestThread.interrupt();
+ } catch (InterruptedException ie) {
+ // we are done;
+ }
+ }
+ };
+
+ for (int i = 0; i < 50; i++) {
+ try {
+ getInfoThreads[i].join();
+ } catch (InterruptedException ie) {
+ // we haven't finished in time. Potential deadlock/race.
+ fail("Unexpectedly long delay during concurrent cache entry creations");
+ }
+ }
+ // stop the timeoutThread. If we get interrupted before stopping, there
+ // must be something wrong, although it wasn't a deadlock. No need to
+ // catch and swallow.
+ timeoutThread.interrupt();
+ }
+
private static void checkRecord(IndexRecord rec, long fill) {
assertEquals(fill, rec.startOffset);
assertEquals(fill, rec.rawLength);
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1360737&r1=1360736&r2=1360737&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml Thu Jul 12 15:53:38 2012
@@ -99,6 +99,23 @@
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>clover</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>clover</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.cenqua.clover</groupId>
+ <artifactId>clover</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+</profiles>
<build>
<plugins>
<plugin>
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml?rev=1360737&r1=1360736&r2=1360737&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml Thu Jul 12 15:53:38 2012
@@ -31,4 +31,21 @@
<modules>
<module>hadoop-yarn-applications-distributedshell</module>
</modules>
+ <profiles>
+ <profile>
+ <id>clover</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>clover</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.cenqua.clover</groupId>
+ <artifactId>clover</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+</profiles>
</project>