You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2012/07/24 23:16:42 UTC
svn commit: r1365292 - in /hadoop/common/branches/branch-0.22/mapreduce:
CHANGES.txt
src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
Author: shv
Date: Tue Jul 24 21:16:41 2012
New Revision: 1365292
URL: http://svn.apache.org/viewvc?rev=1365292&view=rev
Log:
MAPREDUCE-4349. Add test to verify Distributed Cache consistency when cached archives are deleted. Contributed by Mayank Bansal.
Modified:
hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
hadoop/common/branches/branch-0.22/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt?rev=1365292&r1=1365291&r2=1365292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt Tue Jul 24 21:16:41 2012
@@ -17,6 +17,9 @@ Release 0.22.1 - Unreleased
MAPREDUCE-4405. Test case for HierarchicalQueue in TestJobQueueClient.
(Mayank Bansal via shv)
+ MAPREDUCE-4349. Add test to verify Distributed Cache consistency when
+ cached archives are deleted. (Mayank Bansal via shv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java?rev=1365292&r1=1365291&r2=1365292&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/test/mapred/org/apache/hadoop/mapreduce/filecache/TestTrackerDistributedCacheManager.java Tue Jul 24 21:16:41 2012
@@ -46,7 +46,6 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.filecache.TaskDistributedCacheManager.CacheFile;
-import org.apache.hadoop.mapreduce.filecache.TrackerDistributedCacheManager.CacheStatus;
import org.apache.hadoop.mapreduce.filecache.TestTrackerDistributedCacheManager;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -227,7 +226,8 @@ public class TestTrackerDistributedCache
* @throws InterruptedException
*/
@SuppressWarnings("deprecation")
- public void testCacheConsistency() throws IOException, LoginException, InterruptedException {
+ public void testCacheConsistency() throws IOException, LoginException,
+ InterruptedException {
if (!canRun()) {
return;
}
@@ -238,6 +238,7 @@ public class TestTrackerDistributedCache
subConf.set(MRJobConfig.USER_NAME, userName);
JobID jobid = new JobID("jt", 1);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
+ DistributedCache.addCacheArchive(secondCacheFile.toUri(), subConf);
TrackerDistributedCacheManager.determineTimestamps(subConf);
TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
// ****** End of imitating JobClient code
@@ -253,6 +254,7 @@ public class TestTrackerDistributedCache
TaskDistributedCacheManager handle = manager
.newTaskDistributedCacheManager(jobid, subConf);
assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
+ assertNull(null, DistributedCache.getLocalCacheArchives(subConf));
handle.setupCache(subConf, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
JobLocalizer.downloadPrivateCache(subConf);
@@ -269,28 +271,52 @@ public class TestTrackerDistributedCache
File f1 = new File(cachedFirstFile.toString());
assertTrue(f1.delete());
+ Path[] localCacheArchiveFiles = DistributedCache
+ .getLocalCacheArchives(subConf);
+ assertNotNull(null, localCacheArchiveFiles);
+ assertEquals(1, localCacheArchiveFiles.length);
+ Path cachedSecondFile = localCacheArchiveFiles[0];
+ assertFileLengthEquals(secondCacheFile, new Path(cachedSecondFile,
+ "secondcachefile"));
+ assertFalse("Paths should be different.",
+ secondCacheFile.equals(cachedSecondFile));
+ File f2 = new File(cachedSecondFile.toString());
+ FileUtil.fullyDelete(f2);
+ assertFalse(f2.exists());
// ****** Imitate JobClient code
// Configures a task/job with both a regular file and a "classpath" file.
subConf = new Configuration(conf);
userName = getJobOwnerName();
subConf.set(MRJobConfig.USER_NAME, userName);
DistributedCache.addCacheFile(firstCacheFile.toUri(), subConf);
+ DistributedCache.addCacheArchive(secondCacheFile.toUri(), subConf);
TrackerDistributedCacheManager.determineTimestamps(subConf);
TrackerDistributedCacheManager.determineCacheVisibilities(subConf);
JobID jobidnew = new JobID("jt", 2);
handle = manager.newTaskDistributedCacheManager(jobidnew, subConf);
assertNull(null, DistributedCache.getLocalCacheFiles(subConf));
+ assertNull(null, DistributedCache.getLocalCacheArchives(subConf));
handle.setupCache(subConf, TaskTracker.getPublicDistributedCacheDir(),
TaskTracker.getPrivateDistributedCacheDir(userName));
JobLocalizer.downloadPrivateCache(subConf);
// ****** End of imitating TaskRunner code
Path[] localCacheFilesagain = DistributedCache.getLocalCacheFiles(subConf);
+ Path[] localCacheArchivesagain = DistributedCache
+ .getLocalCacheArchives(subConf);
assertNotNull(null, localCacheFilesagain);
+ assertNotNull(null, localCacheArchivesagain);
assertEquals(1, localCacheFilesagain.length);
+ assertEquals(1, localCacheArchivesagain.length);
Path cachedFirstFileAgain = localCacheFilesagain[0];
assertFileLengthEquals(firstCacheFile, cachedFirstFileAgain);
assertFalse("Paths should be different.",
firstCacheFile.equals(cachedFirstFileAgain));
+
+ Path cachedSecondFileAgain = localCacheArchivesagain[0];
+ assertFileLengthEquals(secondCacheFile, new Path(cachedSecondFileAgain,
+ "secondcachefile"));
+ assertFalse("Paths should be different.",
+ secondCacheFile.equals(cachedSecondFileAgain));
}
/**