You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2011/08/18 00:08:31 UTC
svn commit: r1158935 [2/3] - in
/hadoop/common/branches/MR-279-merge/hadoop-mapreduce: ./ assembly/
hadoop-mr-client/ hadoop-mr-client/hadoop-mapreduce-client-app/
hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
had...
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml Wed Aug 17 22:08:08 2011
@@ -9,7 +9,7 @@
<!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
<fileSets>
<fileSet>
- <directory>yarn/yarn-server/yarn-server-nodemanager/target/classes/bin</directory>
+ <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>container-executor</include>
@@ -17,7 +17,7 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
- <directory>yarn/bin</directory>
+ <directory>hadoop-yarn/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>*</include>
@@ -33,7 +33,7 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
- <directory>yarn/conf</directory>
+ <directory>hadoop-yarn/conf</directory>
<outputDirectory>conf</outputDirectory>
<includes>
<include>**/*</include>
@@ -62,7 +62,7 @@
<include>INSTALL</include>
<include>LICENSE.txt</include>
<include>mr-client/**</include>
- <include>yarn/**</include>
+ <include>hadoop-yarn/**</include>
<include>src/**</include>
</includes>
</fileSet>
@@ -78,7 +78,7 @@
<moduleSets>
<moduleSet>
<excludes>
- <exclude>org.apache.hadoop:yarn-server-tests</exclude>
+ <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
</excludes>
<binaries>
<outputDirectory>modules</outputDirectory>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml Wed Aug 17 22:08:08 2011
@@ -19,7 +19,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -29,17 +29,17 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-common</artifactId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-nodemanager</artifactId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-resourcemanager</artifactId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Wed Aug 17 22:08:08 2011
@@ -406,12 +406,6 @@ public class TaskAttemptListenerImpl ext
}
@Override
- public void updatePrivateDistributedCacheSizes(
- org.apache.hadoop.mapreduce.JobID jobId, long[] sizes) throws IOException {
- // TODO Auto-generated method stub
- }
-
- @Override
public void register(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
org.apache.hadoop.mapred.Task task, WrappedJvmID jvmID) {
//create the mapping so that it is easy to look up
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Wed Aug 17 22:08:08 2011
@@ -803,7 +803,7 @@ public class JobImpl implements org.apac
job.conf.get(MRJobConfig.USER_NAME, "mapred"),
job.submitTime,
job.remoteJobConfFile.toString(),
- job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "test"));
+ job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "default"));
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
//TODO JH Verify jobACLs, UserName via UGI?
@@ -1111,7 +1111,6 @@ public class JobImpl implements org.apac
new JobInitedEvent(job.oldJobId,
job.startTime,
job.numMapTasks, job.numReduceTasks,
- job.isUber, 0, 0, // FIXME: lose latter two args again (old-style uber junk: needs to go along with 98% of other old-style uber junk)
job.getState().toString()); //Will transition to state running. Currently in INITED
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Aug 17 22:08:08 2011
@@ -642,13 +642,24 @@ public abstract class TaskAttemptImpl im
return container;
}
+ private static long[] parseTimeStamps(String[] strs) {
+ if (null == strs) {
+ return null;
+ }
+ long[] result = new long[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Long.parseLong(strs[i]);
+ }
+ return result;
+ }
+
private void setupDistributedCache(FileSystem remoteFS, Configuration conf,
ContainerLaunchContext container) throws IOException {
// Cache archives
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
- DistributedCache.getArchiveTimestamps(conf),
+ parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
DistributedCache.getArchiveVisibilities(conf),
DistributedCache.getArchiveClassPaths(conf));
@@ -656,7 +667,7 @@ public abstract class TaskAttemptImpl im
// Cache files
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
- DistributedCache.getFileTimestamps(conf),
+ parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
DistributedCache.getFileVisibilities(conf),
DistributedCache.getFileClassPaths(conf));
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml Wed Aug 17 22:08:08 2011
@@ -18,7 +18,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -56,7 +56,7 @@
<configuration>
<executable>protoc</executable>
<arguments>
- <argument>-I../../yarn/yarn-api/src/main/proto/</argument>
+ <argument>-I../../hadoop-yarn/hadoop-yarn-api/src/main/proto/</argument>
<argument>-Isrc/main/proto/</argument>
<argument>--java_out=target/generated-sources/proto</argument>
<argument>src/main/proto/mr_protos.proto</argument>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml (from r1158914, hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml&r1=1158914&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml Wed Aug 17 22:08:08 2011
@@ -18,7 +18,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java Wed Aug 17 22:08:08 2011
@@ -560,7 +560,7 @@ public class BackupStore<K,V> {
private Writer<K,V> createSpillFile() throws IOException {
Path tmp =
- new Path(TaskTracker.OUTPUT + "/backup_" + tid.getId() + "_"
+ new Path(Constants.OUTPUT + "/backup_" + tid.getId() + "_"
+ (spillNumber++) + ".out");
LOG.info("Created file: " + tmp);
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java Wed Aug 17 22:08:08 2011
@@ -48,7 +48,7 @@ public class JobACLsManager {
*
* @return JobACL to AccessControlList map.
*/
- Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
+ public Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
Map<JobACL, AccessControlList> acls =
new HashMap<JobACL, AccessControlList>();
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java Wed Aug 17 22:08:08 2011
@@ -54,7 +54,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputFile()
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING, getConf());
}
@@ -68,7 +68,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputFileForWrite(long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING, size, getConf());
}
@@ -89,7 +89,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputIndexFile()
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
getConf());
}
@@ -104,7 +104,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputIndexFileForWrite(long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
size, getConf());
}
@@ -128,7 +128,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillFile(int spillNumber)
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
+ spillNumber + ".out", getConf());
}
@@ -143,7 +143,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillFileForWrite(int spillNumber, long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
+ spillNumber + ".out", size, getConf());
}
@@ -157,7 +157,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillIndexFile(int spillNumber)
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
+ spillNumber + ".out.index", getConf());
}
@@ -172,7 +172,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillIndexFileForWrite(int spillNumber, long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
+ spillNumber + ".out.index", size, getConf());
}
@@ -187,7 +187,7 @@ public class MROutputFiles extends MapOu
public Path getInputFile(int mapId)
throws IOException {
return lDirAlloc.getLocalPathToRead(String.format(
- REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer
+ REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, Integer
.valueOf(mapId)), getConf());
}
@@ -204,7 +204,7 @@ public class MROutputFiles extends MapOu
long size)
throws IOException {
return lDirAlloc.getLocalPathForWrite(String.format(
- REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()),
+ REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, mapId.getId()),
size, getConf());
}
@@ -212,7 +212,7 @@ public class MROutputFiles extends MapOu
@Override
public void removeAll()
throws IOException {
- ((JobConf)getConf()).deleteLocalFiles(TaskTracker.OUTPUT);
+ ((JobConf)getConf()).deleteLocalFiles(Constants.OUTPUT);
}
@Override
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Wed Aug 17 22:08:08 2011
@@ -107,13 +107,6 @@ class MapTask extends Task {
throws IOException {
super.localizeConfiguration(conf);
}
-
-
- @Override
- public TaskRunner createRunner(TaskTracker tracker,
- TaskTracker.TaskInProgress tip) {
- return new MapTaskRunner(tip, tracker, this.conf);
- }
@Override
public void write(DataOutput out) throws IOException {
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java Wed Aug 17 22:08:08 2011
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.TaskScheduler.QueueRefresher;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.security.UserGroupInformation;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Wed Aug 17 22:08:08 2011
@@ -46,11 +46,10 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
-import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.task.reduce.Shuffle;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -145,12 +144,6 @@ public class ReduceTask extends Task {
}
@Override
- public TaskRunner createRunner(TaskTracker tracker, TaskInProgress tip)
- throws IOException {
- return new ReduceTaskRunner(tip, tracker, this.conf);
- }
-
- @Override
public boolean isMapTask() {
return false;
}
@@ -347,7 +340,7 @@ public class ReduceTask extends Task {
// Initialize the codec
codec = initCodec();
RawKeyValueIterator rIter = null;
- boolean isLocal = "local".equals(job.get(JTConfig.JT_IPC_ADDRESS, "local"));
+ boolean isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
if (!isLocal) {
Class combinerClass = conf.getCombinerClass();
CombineOutputCollector combineCollector =
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java Wed Aug 17 22:08:08 2011
@@ -34,7 +34,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.PureJavaCrc32;
-
+import org.apache.hadoop.fs.FSDataInputStream;
import static org.apache.hadoop.mapred.MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH;
class SpillRecord {
@@ -64,9 +64,7 @@ class SpillRecord {
throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
- final DataInputStream in =
- new DataInputStream(SecureIOUtils.openForRead(
- new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
+ final FSDataInputStream in = rfs.open(indexFileName);
try {
final long length = rfs.getFileStatus(indexFileName).getLen();
final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Wed Aug 17 22:08:08 2011
@@ -61,9 +61,8 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin.*;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.*;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -487,12 +486,6 @@ abstract public class Task implements Wr
public abstract void run(JobConf job, TaskUmbilicalProtocol umbilical)
throws IOException, ClassNotFoundException, InterruptedException;
-
- /** Return an approprate thread runner for this task.
- * @param tip TODO*/
- public abstract TaskRunner createRunner(TaskTracker tracker,
- TaskTracker.TaskInProgress tip) throws IOException;
-
/** The number of milliseconds between progress reports. */
public static final int PROGRESS_INTERVAL = 3000;
@@ -539,7 +532,7 @@ abstract public class Task implements Wr
}
committer.setupTask(taskContext);
Class<? extends ResourceCalculatorPlugin> clazz =
- conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
+ conf.getClass(MRConfig.RESOURCE_CALCULATOR_PLUGIN,
null, ResourceCalculatorPlugin.class);
resourceCalculator = ResourceCalculatorPlugin
.getResourceCalculatorPlugin(clazz, conf);
@@ -1161,7 +1154,7 @@ abstract public class Task implements Wr
// add the static resolutions (this is required for the junit to
// work on testcases that simulate multiple nodes on a single physical
// node.
- String hostToResolved[] = conf.getStrings(TTConfig.TT_STATIC_RESOLUTIONS);
+ String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
if (hostToResolved != null) {
for (String str : hostToResolved) {
String name = str.substring(0, str.indexOf('='));
@@ -1175,6 +1168,10 @@ abstract public class Task implements Wr
return this.conf;
}
+ public MapOutputFile getMapOutputFile() {
+ return mapOutputFile;
+ }
+
/**
* OutputCollector for the combiner.
*/
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java Wed Aug 17 22:08:08 2011
@@ -33,12 +33,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
+import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -281,8 +279,8 @@ public class Cluster {
if (jobHistoryDir == null) {
jobHistoryDir = new Path(client.getJobHistoryDir());
}
- return JobHistory.getJobHistoryFile(jobHistoryDir, jobId,
- ugi.getShortUserName()).toString();
+ return new Path(jobHistoryDir, jobId.toString() + "_"
+ + ugi.getShortUserName()).toString();
}
/**
@@ -314,19 +312,6 @@ public class Cluster {
throws IOException, InterruptedException {
return client.getChildQueues(queueName);
}
-
- /**
- * Get JobTracker's state
- *
- * @return {@link State} of the JobTracker
- * @throws IOException
- * @throws InterruptedException
- * @deprecated Use {@link #getJobTrackerStatus()} instead.
- */
- @Deprecated
- public State getJobTrackerState() throws IOException, InterruptedException {
- return client.getJobTrackerState();
- }
/**
* Get the JobTracker's status.
@@ -359,8 +344,7 @@ public class Cluster {
getDelegationToken(Text renewer) throws IOException, InterruptedException{
Token<DelegationTokenIdentifier> result =
client.getDelegationToken(renewer);
- InetSocketAddress addr = NetUtils.createSocketAddr(
- conf.get(JTConfig.JT_IPC_ADDRESS, "localhost:8012"));
+ InetSocketAddress addr = Master.getMasterAddress(conf);
StringBuilder service = new StringBuilder();
service.append(NetUtils.normalizeHostName(addr.getAddress().
getHostAddress()));
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java Wed Aug 17 22:08:08 2011
@@ -35,5 +35,9 @@ public enum JobCounter {
SLOTS_MILLIS_MAPS,
SLOTS_MILLIS_REDUCES,
FALLOW_SLOTS_MILLIS_MAPS,
- FALLOW_SLOTS_MILLIS_REDUCES
+ FALLOW_SLOTS_MILLIS_REDUCES,
+ TOTAL_LAUNCHED_UBERTASKS,
+ NUM_UBER_SUBMAPS,
+ NUM_UBER_SUBREDUCES,
+ NUM_FAILED_UBERTASKS
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Wed Aug 17 22:08:08 2011
@@ -18,8 +18,6 @@
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapred.TaskTracker;
/**
* Place holder for cluster level configuration keys.
@@ -58,6 +56,15 @@ public interface MRConfig {
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
+ public static final String RESOURCE_CALCULATOR_PLUGIN =
+ "mapreduce.job.resourcecalculatorplugin";
+ public static final String STATIC_RESOLUTIONS =
+ "mapreduce.job.net.static.resolutions";
+
+ public static final String MASTER_ADDRESS = "mapreduce.jobtracker.address";
+ public static final String MASTER_USER_NAME =
+ "mapreduce.jobtracker.kerberos.principal";
+
public static final String FRAMEWORK_NAME = "mapreduce.framework.name";
public static final String TASK_LOCAL_OUTPUT_CLASS =
"mapreduce.task.local.output.class";
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Aug 17 22:08:08 2011
@@ -287,4 +287,18 @@ public interface MRJobConfig {
public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max";
public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
+ public static final String JOB_UBERTASK_ENABLE =
+ "mapreduce.job.ubertask.enable";
+ public static final String JOB_UBERTASK_MAXMAPS =
+ "mapreduce.job.ubertask.maxmaps";
+ public static final String JOB_UBERTASK_MAXREDUCES =
+ "mapreduce.job.ubertask.maxreduces";
+ public static final String JOB_UBERTASK_MAXBYTES =
+ "mapreduce.job.ubertask.maxbytes";
+ public static final String UBERTASK_JAVA_OPTS =
+ "mapreduce.ubertask.child.java.opts"; // or mapreduce.uber.java.opts?
+ public static final String UBERTASK_ULIMIT =
+ "mapreduce.ubertask.child.ulimit"; // or mapreduce.uber.ulimit?
+ public static final String UBERTASK_ENV =
+ "mapreduce.ubertask.child.env"; // or mapreduce.uber.env?
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java Wed Aug 17 22:08:08 2011
@@ -377,6 +377,38 @@ public class DistributedCache {
return false;
}
+ private static boolean[] parseBooleans(String[] strs) {
+ if (null == strs) {
+ return null;
+ }
+ boolean[] result = new boolean[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Boolean.parseBoolean(strs[i]);
+ }
+ return result;
+ }
+
+ /**
+ * Get the booleans on whether the files are public or not. Used by
+ * internal DistributedCache and MapReduce code.
+ * @param conf The configuration which stored the timestamps
+ * @return a string array of booleans
+ * @throws IOException
+ */
+ public static boolean[] getFileVisibilities(Configuration conf) {
+ return parseBooleans(conf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES));
+ }
+
+ /**
+ * Get the booleans on whether the archives are public or not. Used by
+ * internal DistributedCache and MapReduce code.
+ * @param conf The configuration which stored the timestamps
+ * @return a string array of booleans
+ */
+ public static boolean[] getArchiveVisibilities(Configuration conf) {
+ return parseBooleans(conf.getStrings(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES));
+ }
+
/**
* This method checks if there is a conflict in the fragment names
* of the uris. Also makes sure that each uri has a fragment. It
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java Wed Aug 17 22:08:08 2011
@@ -18,12 +18,11 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.lang.Integer;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import org.apache.avro.Schema;
-
-import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
public class AvroArrayUtils {
@@ -31,13 +30,12 @@ public class AvroArrayUtils {
private static final Schema ARRAY_INT
= Schema.createArray(Schema.create(Schema.Type.INT));
- static public GenericArray<Integer> NULL_PROGRESS_SPLITS_ARRAY
+ static public List<Integer> NULL_PROGRESS_SPLITS_ARRAY
= new GenericData.Array<Integer>(0, ARRAY_INT);
- public static GenericArray<Integer>
+ public static List<Integer>
toAvro(int values[]) {
- GenericData.Array<Integer> result
- = new GenericData.Array<Integer>(values.length, ARRAY_INT);
+ List<Integer> result = new ArrayList<Integer>(values.length);
for (int i = 0; i < values.length; ++i) {
result.add(values[i]);
@@ -46,7 +44,7 @@ public class AvroArrayUtils {
return result;
}
- public static int[] fromAvro(GenericArray<Integer> avro) {
+ public static int[] fromAvro(List<Integer> avro) {
int[] result = new int[(int)avro.size()];
int i = 0;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Wed Aug 17 22:08:08 2011
@@ -19,23 +19,21 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
+import java.util.ArrayList;
import org.apache.avro.Schema;
+import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.JsonEncoder;
-import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
/**
* Event Writer is an utility class used to write events to the underlying
@@ -97,13 +95,13 @@ class EventWriter {
static JhCounters toAvro(Counters counters, String name) {
JhCounters result = new JhCounters();
result.name = new Utf8(name);
- result.groups = new GenericData.Array<JhCounterGroup>(0, GROUPS);
+ result.groups = new ArrayList<JhCounterGroup>(0);
if (counters == null) return result;
for (CounterGroup group : counters) {
JhCounterGroup g = new JhCounterGroup();
g.name = new Utf8(group.getName());
g.displayName = new Utf8(group.getDisplayName());
- g.counts = new GenericData.Array<JhCounter>(group.size(), COUNTERS);
+ g.counts = new ArrayList<JhCounter>(group.size());
for (Counter counter : group) {
JhCounter c = new JhCounter();
c.name = new Utf8(counter.getName());
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java Wed Aug 17 22:08:08 2011
@@ -34,9 +34,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobInProgress;
import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.TaskLogServlet;
import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
@@ -45,6 +43,7 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.util.StringUtils;
/**
@@ -456,9 +455,9 @@ public class HistoryViewer {
}
String taskTrackerName =
- JobInProgress.convertTrackerNameToHostName(
+ HostUtil.convertTrackerNameToHostName(
attempt.getTrackerName());
- return TaskLogServlet.getTaskLogUrl(taskTrackerName,
+ return HostUtil.getTaskLogUrl(taskTrackerName,
Integer.toString(attempt.getHttpPort()),
attempt.getAttemptId().toString());
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Wed Aug 17 22:08:08 2011
@@ -57,7 +57,7 @@ public class JobSubmittedEvent implement
datum.userName = new Utf8(userName);
datum.submitTime = submitTime;
datum.jobConfPath = new Utf8(jobConfPath);
- Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+ Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
entry.getValue().getAclString()));
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Wed Aug 17 22:08:08 2011
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.TaskT
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -140,17 +139,6 @@ public interface ClientProtocol extends
*/
public ClusterMetrics getClusterMetrics()
throws IOException, InterruptedException;
-
- /**
- * Get JobTracker's state
- *
- * @return {@link State} of the JobTracker
- * @throws IOException
- * @throws InterruptedException
- * @deprecated Use {@link #getJobTrackerStatus()} instead.
- */
- @Deprecated
- public State getJobTrackerState() throws IOException, InterruptedException;
/**
* Get the JobTracker's status.
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Wed Aug 17 22:08:08 2011
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.Credentials;
@@ -91,13 +91,6 @@ public class TokenCache {
}
}
- static String getJTPrincipal(Configuration conf) throws IOException {
- String jtHostname = JobTracker.getAddress(conf).getHostName();
- // get jobtracker principal for use as delegation token renewer
- return SecurityUtil.getServerPrincipal(conf.get(JTConfig.JT_USER_NAME),
- jtHostname);
- }
-
/**
* get delegation token for a specific FS
* @param fs
@@ -108,7 +101,7 @@ public class TokenCache {
*/
static void obtainTokensForNamenodesInternal(FileSystem fs,
Credentials credentials, Configuration conf) throws IOException {
- String delegTokenRenewer = getJTPrincipal(conf);
+ String delegTokenRenewer = Master.getMasterPrincipal(conf);
if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
throw new IOException(
"Can't get JobTracker Kerberos principal for use as renewer");
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java Wed Aug 17 22:08:08 2011
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate;
import org.apache.hadoop.mapred.TaskCompletionEvent;
-import org.apache.hadoop.mapred.TaskTracker;
import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
import org.apache.hadoop.mapreduce.TaskAttemptID;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java Wed Aug 17 22:08:08 2011
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BoundedByteArrayOutputStream;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskTracker;
+import org.apache.hadoop.mapred.MapOutputFile;
import org.apache.hadoop.mapreduce.TaskAttemptID;
class MapOutput<K,V> {
@@ -64,7 +64,8 @@ class MapOutput<K,V> {
MapOutput(TaskAttemptID mapId, MergeManager<K,V> merger, long size,
JobConf conf, LocalDirAllocator localDirAllocator,
- int fetcher, boolean primaryMapOutput) throws IOException {
+ int fetcher, boolean primaryMapOutput, MapOutputFile mapOutputFile)
+ throws IOException {
this.id = ID.incrementAndGet();
this.mapId = mapId;
this.merger = merger;
@@ -77,18 +78,10 @@ class MapOutput<K,V> {
this.size = size;
this.localFS = FileSystem.getLocal(conf);
- String filename = "map_" + mapId.getTaskID().getId() + ".out";
- String tmpOutput = Path.SEPARATOR +
- TaskTracker.getJobCacheSubdir(conf.getUser()) +
- Path.SEPARATOR + mapId.getJobID() +
- Path.SEPARATOR + merger.getReduceId() +
- Path.SEPARATOR + "output" +
- Path.SEPARATOR + filename +
- "." + fetcher;
-
- tmpOutputPath =
- localDirAllocator.getLocalPathForWrite(tmpOutput, size, conf);
- outputPath = new Path(tmpOutputPath.getParent(), filename);
+ outputPath =
+ mapOutputFile.getInputFileForWrite(mapId.getTaskID(),size);
+ tmpOutputPath = outputPath.suffix(String.valueOf(fetcher));
+
disk = localFS.create(tmpOutputPath);
this.primaryMapOutput = primaryMapOutput;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Wed Aug 17 22:08:08 2011
@@ -225,7 +225,8 @@ public class MergeManager<K, V> {
" is greater than maxSingleShuffleLimit (" +
maxSingleShuffleLimit + ")");
return new MapOutput<K,V>(mapId, this, requestedSize, jobConf,
- localDirAllocator, fetcher, true);
+ localDirAllocator, fetcher, true,
+ mapOutputFile);
}
// Stall shuffle if we are above the memory limit
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (from r1158894, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&r1=1158894&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java Wed Aug 17 22:08:08 2011
@@ -29,7 +29,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.mapred.TaskTrackerStatus;
/**
* Plugin to calculate resource information on Linux systems.
@@ -40,6 +39,8 @@ public class LinuxResourceCalculatorPlug
private static final Log LOG =
LogFactory.getLog(LinuxResourceCalculatorPlugin.class);
+ public static final int UNAVAILABLE = -1;
+
/**
* proc's meminfo virtual file has keys-values in the format
* "key:[ \t]*value[ \t]kB".
@@ -87,9 +88,9 @@ public class LinuxResourceCalculatorPlug
private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms)
private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms)
// Unix timestamp while reading the CPU time (ms)
- private float cpuUsage = TaskTrackerStatus.UNAVAILABLE;
- private long sampleTime = TaskTrackerStatus.UNAVAILABLE;
- private long lastSampleTime = TaskTrackerStatus.UNAVAILABLE;
+ private float cpuUsage = UNAVAILABLE;
+ private long sampleTime = UNAVAILABLE;
+ private long lastSampleTime = UNAVAILABLE;
private ProcfsBasedProcessTree pTree = null;
boolean readMemInfoFile = false;
@@ -352,7 +353,7 @@ public class LinuxResourceCalculatorPlug
public float getCpuUsage() {
readProcStatFile();
sampleTime = getCurrentTime();
- if (lastSampleTime == TaskTrackerStatus.UNAVAILABLE ||
+ if (lastSampleTime == UNAVAILABLE ||
lastSampleTime > sampleTime) {
// lastSampleTime > sampleTime may happen when the system time is changed
lastSampleTime = sampleTime;
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Wed Aug 17 22:08:08 2011
@@ -94,12 +94,12 @@ public class CompletedJob implements org
@Override
public int getCompletedMaps() {
- return jobInfo.getFinishedMaps();
+ return (int) jobInfo.getFinishedMaps();
}
@Override
public int getCompletedReduces() {
- return jobInfo.getFinishedReduces();
+ return (int) jobInfo.getFinishedReduces();
}
@Override
@@ -264,17 +264,17 @@ public class CompletedJob implements org
@Override
public int getTotalMaps() {
- return jobInfo.getTotalMaps();
+ return (int) jobInfo.getTotalMaps();
}
@Override
public int getTotalReduces() {
- return jobInfo.getTotalReduces();
+ return (int) jobInfo.getTotalReduces();
}
@Override
public boolean isUber() {
- return jobInfo.getIsUber();
+ throw new YarnException("Not yet implemented!");
}
@Override
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/pom.xml Wed Aug 17 22:08:08 2011
@@ -43,22 +43,22 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-nodemanager</artifactId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-resourcemanager</artifactId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-common</artifactId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-tests</artifactId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Wed Aug 17 22:08:08 2011
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.TaskR
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
@@ -371,6 +372,22 @@ public class YARNRunner implements Clien
}
/**
+ * * TODO: Copied for now from TaskAttemptImpl.java ... fixme
+ * @param strs
+ * @return
+ */
+ private static long[] parseTimeStamps(String[] strs) {
+ if (null == strs) {
+ return null;
+ }
+ long[] result = new long[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Long.parseLong(strs[i]);
+ }
+ return result;
+ }
+
+ /**
* TODO: Copied for now from TaskAttemptImpl.java ... fixme
*
* TODO: This is currently needed in YarnRunner as user code like setupJob,
@@ -384,7 +401,7 @@ public class YARNRunner implements Clien
// Cache archives
parseDistributedCacheArtifacts(conf, container, LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
- DistributedCache.getArchiveTimestamps(conf),
+ parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
DistributedCache.getArchiveVisibilities(conf),
DistributedCache.getArchiveClassPaths(conf));
@@ -392,7 +409,7 @@ public class YARNRunner implements Clien
// Cache files
parseDistributedCacheArtifacts(conf, container, LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
- DistributedCache.getFileTimestamps(conf),
+ parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
DistributedCache.getFileVisibilities(conf),
DistributedCache.getFileClassPaths(conf));
@@ -553,7 +570,13 @@ public class YARNRunner implements Clien
public AccessControlList getQueueAdmins(String arg0) throws IOException {
return new AccessControlList("*");
}
-
+
+ @Override
+ public JobTrackerStatus getJobTrackerStatus() throws IOException,
+ InterruptedException {
+ return JobTrackerStatus.RUNNING;
+ }
+
@Override
public ProtocolSignature getProtocolSignature(String protocol,
long clientVersion, int clientMethodsHash) throws IOException {
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java Wed Aug 17 22:08:08 2011
@@ -35,8 +35,10 @@ import org.apache.hadoop.mapreduce.TaskC
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
+@Ignore
public class TestUberAM extends TestMRJobs {
private static final Log LOG = LogFactory.getLog(TestUberAM.class);
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/pom.xml?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/pom.xml Wed Aug 17 22:08:08 2011
@@ -18,7 +18,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-nodemanager</artifactId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Wed Aug 17 22:08:08 2011
@@ -394,7 +394,7 @@ public class ShuffleHandler extends Abst
LOG.debug("DEBUG1 " + base + " : " + mapOutputFileName + " : " +
indexFileName);
IndexRecord info =
- indexCache.getIndexInformation(mapId, reduce, indexFileName);
+ indexCache.getIndexInformation(mapId, reduce, indexFileName, user);
final ShuffleHeader header =
new ShuffleHeader(mapId, info.partLength, info.rawLength, reduce);
final DataOutputBuffer dob = new DataOutputBuffer();
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/pom.xml (from r1158914, hadoop/common/branches/MR-279/mapreduce/mr-client/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/mr-client/pom.xml&r1=1158914&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/pom.xml Wed Aug 17 22:08:08 2011
@@ -21,24 +21,24 @@
<!-- begin MNG-4223 workaround -->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn</artifactId>
+ <artifactId>hadoop-yarn</artifactId>
<version>${yarn.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-api</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server</artifactId>
+ <artifactId>hadoop-yarn-server</artifactId>
<version>${yarn.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-common</artifactId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
<version>${yarn.version}</version>
</dependency>
<!-- end MNG-4223 workaround -->
@@ -50,29 +50,29 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
<version>${yarn.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
<version>${yarn.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-tests</artifactId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
<version>${yarn.version}</version>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-nodemanager</artifactId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
<version>${yarn.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-resourcemanager</artifactId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<version>${yarn.version}</version>
</dependency>
<dependency>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-api/pom.xml (from r1158914, hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-api/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-api/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/pom.xml&r1=1158914&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-api/pom.xml Wed Aug 17 22:08:08 2011
@@ -1,14 +1,14 @@
<?xml version="1.0"?>
<project xmlns:pom="http://maven.apache.org/POM/4.0.0">
<parent>
- <artifactId>yarn</artifactId>
+ <artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
<version>${yarn.version}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-api</artifactId>
- <name>yarn-api</name>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <name>hadoop-yarn-api</name>
<properties>
<install.file>${project.artifact.file}</install.file>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/pom.xml (from r1158914, hadoop/common/branches/MR-279/mapreduce/yarn/yarn-common/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/yarn/yarn-common/pom.xml&r1=1158914&r2=1158935&rev=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-common/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/pom.xml Wed Aug 17 22:08:08 2011
@@ -1,14 +1,14 @@
<?xml version="1.0"?>
<project>
<parent>
- <artifactId>yarn</artifactId>
+ <artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
<version>${yarn.version}</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
- <name>yarn-common</name>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <name>hadoop-yarn-common</name>
<properties>
<install.file>${project.artifact.file}</install.file>
@@ -34,7 +34,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-api</artifactId>
+ <artifactId>hadoop-yarn-api</artifactId>
</dependency>
</dependencies>
@@ -80,7 +80,7 @@
<configuration>
<executable>protoc</executable>
<arguments>
- <argument>-I../yarn-api/src/main/proto/</argument>
+ <argument>-I../hadoop-yarn-api/src/main/proto/</argument>
<argument>-Isrc/main/proto/</argument>
<argument>--java_out=target/generated-sources/proto</argument>
<argument>src/main/proto/yarnprototunnelrpc.proto</argument>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LinuxResourceCalculatorPlugin.java Wed Aug 17 22:08:08 2011
@@ -29,8 +29,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
-import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
* Plugin to calculate resource information on Linux systems.
@@ -41,6 +39,8 @@ public class LinuxResourceCalculatorPlug
private static final Log LOG =
LogFactory.getLog(LinuxResourceCalculatorPlugin.class);
+ public static final int UNAVAILABLE = -1;
+
/**
* proc's meminfo virtual file has keys-values in the format
* "key:[ \t]*value[ \t]kB".
@@ -88,9 +88,9 @@ public class LinuxResourceCalculatorPlug
private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms)
private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms)
// Unix timestamp while reading the CPU time (ms)
- private float cpuUsage = ResourceCalculatorPlugin.UNAVAILABLE;
- private long sampleTime = ResourceCalculatorPlugin.UNAVAILABLE;
- private long lastSampleTime = ResourceCalculatorPlugin.UNAVAILABLE;
+ private float cpuUsage = UNAVAILABLE;
+ private long sampleTime = UNAVAILABLE;
+ private long lastSampleTime = UNAVAILABLE;
private ProcfsBasedProcessTree pTree = null;
boolean readMemInfoFile = false;
@@ -353,7 +353,7 @@ public class LinuxResourceCalculatorPlug
public float getCpuUsage() {
readProcStatFile();
sampleTime = getCurrentTime();
- if (lastSampleTime == ResourceCalculatorPlugin.UNAVAILABLE ||
+ if (lastSampleTime == UNAVAILABLE ||
lastSampleTime > sampleTime) {
// lastSampleTime > sampleTime may happen when the system time is changed
lastSampleTime = sampleTime;
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java?rev=1158935&r1=1158914&r2=1158935&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java Wed Aug 17 22:08:08 2011
@@ -32,8 +32,6 @@ import org.apache.hadoop.util.Reflection
@InterfaceStability.Unstable
public abstract class ResourceCalculatorPlugin extends Configured {
- public static final int UNAVAILABLE = -1;
-
/**
* Obtain the total size of the virtual memory present in the system.
*
@@ -99,7 +97,7 @@ public abstract class ResourceCalculator
@InterfaceStability.Unstable
public abstract ProcResourceValues getProcResourceValues();
- public static class ProcResourceValues {
+ public class ProcResourceValues {
private final long cumulativeCpuTime;
private final long physicalMemorySize;
private final long virtualMemorySize;
@@ -150,7 +148,6 @@ public abstract class ResourceCalculator
return ReflectionUtils.newInstance(clazz, conf);
}
- // FIXME
// No class given, try a os specific class
try {
String osName = System.getProperty("os.name");