You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:40:45 UTC
svn commit: r1077102 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
core/org/apache/hadoop/filecache/DistributedCache.java
mapred/org/apache/hadoop/mapred/TaskRunner.java
Author: omalley
Date: Fri Mar 4 03:40:45 2011
New Revision: 1077102
URL: http://svn.apache.org/viewvc?rev=1077102&view=rev
Log:
commit d1705ffb5c5cc115bf9d95281b4cee1cfac15a30
Author: Hemanth Yamijala <yh...@yahoo-inc.com>
Date: Mon Jan 11 20:28:02 2010 +0530
Reverting patch https://issues.apache.org/jira/secure/attachment/12426383/patch-1140-2-ydist.txt for MAPREDUCE:1140
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java?rev=1077102&r1=1077101&r2=1077102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/filecache/DistributedCache.java Fri Mar 4 03:40:45 2011
@@ -216,49 +216,38 @@ public class DistributedCache {
lcacheStatus.refcount++;
}
- boolean initSuccessful = false;
- try {
- synchronized (lcacheStatus) {
- if (!lcacheStatus.isInited()) {
- localizedPath = localizeCache(conf, cache, confFileStamp,
- lcacheStatus, fileStatus, isArchive);
- lcacheStatus.initComplete();
- } else {
- localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
- lcacheStatus, fileStatus, isArchive);
- }
- createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir,
- honorSymLinkConf);
- }
-
- // try deleting stuff if you can
- long size = 0;
- synchronized (lcacheStatus) {
- synchronized (baseDirSize) {
- Long get = baseDirSize.get(lcacheStatus.getBaseDir());
- if (get != null) {
- size = get.longValue();
- } else {
- LOG.warn("Cannot find size of baseDir: "
- + lcacheStatus.getBaseDir());
- }
- }
+ synchronized (lcacheStatus) {
+ if (!lcacheStatus.isInited()) {
+ localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus,
+ fileStatus, isArchive);
+ lcacheStatus.initComplete();
+ } else {
+ localizedPath = checkCacheStatusValidity(conf, cache, confFileStamp,
+ lcacheStatus, fileStatus, isArchive);
}
- // setting the cache size to a default of 10GB
- long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
- if (allowedSize < size) {
- // try some cache deletions
- deleteCache(conf);
- }
- initSuccessful = true;
- return localizedPath;
- } finally {
- if (!initSuccessful) {
- synchronized (cachedArchives) {
- lcacheStatus.refcount--;
+ createSymlink(conf, cache, lcacheStatus, isArchive,
+ currentWorkDir, honorSymLinkConf);
+ }
+
+ // try deleting stuff if you can
+ long size = 0;
+ synchronized (lcacheStatus) {
+ synchronized (baseDirSize) {
+ Long get = baseDirSize.get(lcacheStatus.getBaseDir());
+ if ( get != null ) {
+ size = get.longValue();
+ } else {
+ LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir());
}
}
}
+ // setting the cache size to a default of 10GB
+ long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
+ if (allowedSize < size) {
+ // try some cache deletions
+ deleteCache(conf);
+ }
+ return localizedPath;
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1077102&r1=1077101&r2=1077102&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Fri Mar 4 03:40:45 2011
@@ -141,19 +141,9 @@ abstract class TaskRunner extends Thread
return jobConf.get(JobConf.MAPRED_TASK_ENV);
}
- private static class CacheFile {
- URI uri;
- long timeStamp;
- CacheFile (URI uri, long timeStamp) {
- this.uri = uri;
- this.timeStamp = timeStamp;
- }
- }
-
@Override
public final void run() {
String errorInfo = "Child Error";
- List<CacheFile> localizedCacheFiles = new ArrayList<CacheFile>();
try {
//before preparing the job localize
@@ -198,8 +188,6 @@ abstract class TaskRunner extends Thread
getAbsolutePath()),
false,
lDirAlloc);
- localizedCacheFiles.add(new CacheFile(archives[i], Long
- .parseLong(archivesTimestamps[i])));
}
DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
@@ -220,8 +208,6 @@ abstract class TaskRunner extends Thread
getAbsolutePath()),
false,
lDirAlloc);
- localizedCacheFiles.add(new CacheFile(files[i], Long
- .parseLong(fileTimestamps[i])));
}
DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
}
@@ -542,8 +528,22 @@ abstract class TaskRunner extends Thread
}
} finally {
try{
- for (CacheFile cf : localizedCacheFiles){
- DistributedCache.releaseCache(cf.uri, conf, cf.timeStamp);
+ URI[] archives = DistributedCache.getCacheArchives(conf);
+ URI[] files = DistributedCache.getCacheFiles(conf);
+ String[] archivesTimestamps =
+ DistributedCache.getArchiveTimestamps(conf);
+ String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
+ if (archives != null){
+ for (int i = 0; i < archives.length; i++){
+ DistributedCache.releaseCache(archives[i], conf,
+ Long.parseLong(archivesTimestamps[i]));
+ }
+ }
+ if (files != null){
+ for(int i = 0; i < files.length; i++){
+ DistributedCache.releaseCache(files[i], conf,
+ Long.parseLong(fileTimestamps[i]));
+ }
}
}catch(IOException ie){
LOG.warn("Error releasing caches : Cache files might not have been cleaned up");