You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2011/08/18 00:57:55 UTC
svn commit: r1158962 [2/3] - in
/hadoop/common/branches/MR-279-merge/hadoop-mapreduce: ./ assembly/
hadoop-mr-client/ hadoop-mr-client/hadoop-mapreduce-client-app/
hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
had...
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/assembly/all.xml Wed Aug 17 22:57:31 2011
@@ -9,7 +9,7 @@
<!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
<fileSets>
<fileSet>
- <directory>yarn/yarn-server/yarn-server-nodemanager/target/classes/bin</directory>
+ <directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>container-executor</include>
@@ -17,7 +17,7 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
- <directory>yarn/bin</directory>
+ <directory>hadoop-yarn/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>*</include>
@@ -33,7 +33,7 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
- <directory>yarn/conf</directory>
+ <directory>hadoop-yarn/conf</directory>
<outputDirectory>conf</outputDirectory>
<includes>
<include>**/*</include>
@@ -62,7 +62,7 @@
<include>INSTALL</include>
<include>LICENSE.txt</include>
<include>mr-client/**</include>
- <include>yarn/**</include>
+ <include>hadoop-yarn/**</include>
<include>src/**</include>
</includes>
</fileSet>
@@ -78,7 +78,7 @@
<moduleSets>
<moduleSet>
<excludes>
- <exclude>org.apache.hadoop:yarn-server-tests</exclude>
+ <exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
</excludes>
<binaries>
<outputDirectory>modules</outputDirectory>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/pom.xml Wed Aug 17 22:57:31 2011
@@ -19,7 +19,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
@@ -29,17 +29,17 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-common</artifactId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-nodemanager</artifactId>
+ <artifactId>hadoop-yarn-server-nodemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-server-resourcemanager</artifactId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Wed Aug 17 22:57:31 2011
@@ -406,12 +406,6 @@ public class TaskAttemptListenerImpl ext
}
@Override
- public void updatePrivateDistributedCacheSizes(
- org.apache.hadoop.mapreduce.JobID jobId, long[] sizes) throws IOException {
- // TODO Auto-generated method stub
- }
-
- @Override
public void register(org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID,
org.apache.hadoop.mapred.Task task, WrappedJvmID jvmID) {
//create the mapping so that it is easy to look up
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Wed Aug 17 22:57:31 2011
@@ -803,7 +803,7 @@ public class JobImpl implements org.apac
job.conf.get(MRJobConfig.USER_NAME, "mapred"),
job.submitTime,
job.remoteJobConfFile.toString(),
- job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "test"));
+ job.jobACLs, job.conf.get(MRJobConfig.QUEUE_NAME, "default"));
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
//TODO JH Verify jobACLs, UserName via UGI?
@@ -1111,7 +1111,6 @@ public class JobImpl implements org.apac
new JobInitedEvent(job.oldJobId,
job.startTime,
job.numMapTasks, job.numReduceTasks,
- job.isUber, 0, 0, // FIXME: lose latter two args again (old-style uber junk: needs to go along with 98% of other old-style uber junk)
job.getState().toString()); //Will transition to state running. Currently in INITED
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jie));
JobInfoChangeEvent jice = new JobInfoChangeEvent(job.oldJobId,
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Wed Aug 17 22:57:31 2011
@@ -642,13 +642,24 @@ public abstract class TaskAttemptImpl im
return container;
}
+ private static long[] parseTimeStamps(String[] strs) {
+ if (null == strs) {
+ return null;
+ }
+ long[] result = new long[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Long.parseLong(strs[i]);
+ }
+ return result;
+ }
+
private void setupDistributedCache(FileSystem remoteFS, Configuration conf,
ContainerLaunchContext container) throws IOException {
// Cache archives
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
- DistributedCache.getArchiveTimestamps(conf),
+ parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
DistributedCache.getArchiveVisibilities(conf),
DistributedCache.getArchiveClassPaths(conf));
@@ -656,7 +667,7 @@ public abstract class TaskAttemptImpl im
// Cache files
parseDistributedCacheArtifacts(remoteFS, container, LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
- DistributedCache.getFileTimestamps(conf),
+ parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
DistributedCache.getFileVisibilities(conf),
DistributedCache.getFileClassPaths(conf));
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-common/pom.xml Wed Aug 17 22:57:31 2011
@@ -18,7 +18,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -56,7 +56,7 @@
<configuration>
<executable>protoc</executable>
<arguments>
- <argument>-I../../yarn/yarn-api/src/main/proto/</argument>
+ <argument>-I../../hadoop-yarn/hadoop-yarn-api/src/main/proto/</argument>
<argument>-Isrc/main/proto/</argument>
<argument>--java_out=target/generated-sources/proto</argument>
<argument>src/main/proto/mr_protos.proto</argument>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml (from r1158949, hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml&p1=hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-core/pom.xml (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/pom.xml Wed Aug 17 22:57:31 2011
@@ -18,7 +18,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>yarn-common</artifactId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/BackupStore.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/BackupStore.java Wed Aug 17 22:57:31 2011
@@ -560,7 +560,7 @@ public class BackupStore<K,V> {
private Writer<K,V> createSpillFile() throws IOException {
Path tmp =
- new Path(TaskTracker.OUTPUT + "/backup_" + tid.getId() + "_"
+ new Path(Constants.OUTPUT + "/backup_" + tid.getId() + "_"
+ (spillNumber++) + ".out");
LOG.info("Created file: " + tmp);
Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Constants.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+public class Constants {
+ static final String OUTPUT = "output";
+ public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR";
+ public static final String JOBFILE = "job.xml";
+ public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV";
+ public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV";
+}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/JobACLsManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobACLsManager.java Wed Aug 17 22:57:31 2011
@@ -48,7 +48,7 @@ public class JobACLsManager {
*
* @return JobACL to AccessControlList map.
*/
- Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
+ public Map<JobACL, AccessControlList> constructJobACLs(Configuration conf) {
Map<JobACL, AccessControlList> acls =
new HashMap<JobACL, AccessControlList>();
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MROutputFiles.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MROutputFiles.java Wed Aug 17 22:57:31 2011
@@ -54,7 +54,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputFile()
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING, getConf());
}
@@ -68,7 +68,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputFileForWrite(long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING, size, getConf());
}
@@ -89,7 +89,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputIndexFile()
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
getConf());
}
@@ -104,7 +104,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getOutputIndexFileForWrite(long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + Path.SEPARATOR
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + Path.SEPARATOR
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING,
size, getConf());
}
@@ -128,7 +128,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillFile(int spillNumber)
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
+ spillNumber + ".out", getConf());
}
@@ -143,7 +143,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillFileForWrite(int spillNumber, long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
+ spillNumber + ".out", size, getConf());
}
@@ -157,7 +157,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillIndexFile(int spillNumber)
throws IOException {
- return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathToRead(Constants.OUTPUT + "/spill"
+ spillNumber + ".out.index", getConf());
}
@@ -172,7 +172,7 @@ public class MROutputFiles extends MapOu
@Override
public Path getSpillIndexFileForWrite(int spillNumber, long size)
throws IOException {
- return lDirAlloc.getLocalPathForWrite(TaskTracker.OUTPUT + "/spill"
+ return lDirAlloc.getLocalPathForWrite(Constants.OUTPUT + "/spill"
+ spillNumber + ".out.index", size, getConf());
}
@@ -187,7 +187,7 @@ public class MROutputFiles extends MapOu
public Path getInputFile(int mapId)
throws IOException {
return lDirAlloc.getLocalPathToRead(String.format(
- REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, Integer
+ REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, Integer
.valueOf(mapId)), getConf());
}
@@ -204,7 +204,7 @@ public class MROutputFiles extends MapOu
long size)
throws IOException {
return lDirAlloc.getLocalPathForWrite(String.format(
- REDUCE_INPUT_FILE_FORMAT_STRING, TaskTracker.OUTPUT, mapId.getId()),
+ REDUCE_INPUT_FILE_FORMAT_STRING, Constants.OUTPUT, mapId.getId()),
size, getConf());
}
@@ -212,7 +212,7 @@ public class MROutputFiles extends MapOu
@Override
public void removeAll()
throws IOException {
- ((JobConf)getConf()).deleteLocalFiles(TaskTracker.OUTPUT);
+ ((JobConf)getConf()).deleteLocalFiles(Constants.OUTPUT);
}
@Override
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Wed Aug 17 22:57:31 2011
@@ -107,13 +107,6 @@ class MapTask extends Task {
throws IOException {
super.localizeConfiguration(conf);
}
-
-
- @Override
- public TaskRunner createRunner(TaskTracker tracker,
- TaskTracker.TaskInProgress tip) {
- return new MapTaskRunner(tip, tracker, this.conf);
- }
@Override
public void write(DataOutput out) throws IOException {
Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,52 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+
+public class Master {
+
+ public enum State {
+ INITIALIZING, RUNNING;
+ }
+
+ public static String getMasterUserName(Configuration conf) {
+ return conf.get(MRConfig.MASTER_USER_NAME);
+ }
+
+ public static InetSocketAddress getMasterAddress(Configuration conf) {
+ String jobTrackerStr =
+ conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+ return NetUtils.createSocketAddr(jobTrackerStr);
+ }
+
+ public static String getMasterPrincipal(Configuration conf)
+ throws IOException {
+ String jtHostname = getMasterAddress(conf).getHostName();
+ // get jobtracker principal for use as delegation token renewer
+ return SecurityUtil.getServerPrincipal(getMasterUserName(conf), jtHostname);
+ }
+
+}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/QueueManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java Wed Aug 17 22:57:31 2011
@@ -23,7 +23,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapred.TaskScheduler.QueueRefresher;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.QueueState;
import org.apache.hadoop.security.UserGroupInformation;
Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueRefresher.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,49 @@
+package org.apache.hadoop.mapred;
+
+import java.util.List;
+
+/**
+ * Abstract QueueRefresher class. Scheduler's can extend this and return an
+ * instance of this in the {@link #getQueueRefresher()} method. The
+ * {@link #refreshQueues(List)} method of this instance will be invoked by the
+ * {@link QueueManager} whenever it gets a request from an administrator to
+ * refresh its own queue-configuration. This method has a documented contract
+ * between the {@link QueueManager} and the {@link TaskScheduler}.
+ *
+ * Before calling QueueRefresher, the caller must hold the lock to the
+ * corresponding {@link TaskScheduler} (generally in the {@link JobTracker}).
+ */
+abstract class QueueRefresher {
+
+ /**
+ * Refresh the queue-configuration in the scheduler. This method has the
+ * following contract.
+ * <ol>
+ * <li>Before this method, {@link QueueManager} does a validation of the new
+ * queue-configuration. For e.g, currently addition of new queues, or
+ * removal of queues at any level in the hierarchy is not supported by
+ * {@link QueueManager} and so are not supported for schedulers too.</li>
+ * <li>Schedulers will be passed a list of {@link JobQueueInfo}s of the root
+ * queues i.e. the queues at the top level. All the descendants are properly
+ * linked from these top-level queues.</li>
+ * <li>Schedulers should use the scheduler specific queue properties from
+ * the newRootQueues, validate the properties themselves and apply them
+ * internally.</li>
+ * <li>
+ * Once the method returns successfully from the schedulers, it is assumed
+ * that the refresh of queue properties is successful throughout and will be
+ * 'committed' internally to {@link QueueManager} too. It is guaranteed that
+ * at no point, after successful return from the scheduler, is the queue
+ * refresh in QueueManager failed. If ever, such abnormalities happen, the
+ * queue framework will be inconsistent and will need a JT restart.</li>
+ * <li>If scheduler throws an exception during {@link #refreshQueues()},
+ * {@link QueueManager} throws away the newly read configuration, retains
+ * the old (consistent) configuration and informs the request issuer about
+ * the error appropriately.</li>
+ * </ol>
+ *
+ * @param newRootQueues
+ */
+ abstract void refreshQueues(List<JobQueueInfo> newRootQueues)
+ throws Throwable;
+}
\ No newline at end of file
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Wed Aug 17 22:57:31 2011
@@ -46,11 +46,10 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
-import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.mapreduce.task.reduce.Shuffle;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -145,12 +144,6 @@ public class ReduceTask extends Task {
}
@Override
- public TaskRunner createRunner(TaskTracker tracker, TaskInProgress tip)
- throws IOException {
- return new ReduceTaskRunner(tip, tracker, this.conf);
- }
-
- @Override
public boolean isMapTask() {
return false;
}
@@ -347,7 +340,7 @@ public class ReduceTask extends Task {
// Initialize the codec
codec = initCodec();
RawKeyValueIterator rIter = null;
- boolean isLocal = "local".equals(job.get(JTConfig.JT_IPC_ADDRESS, "local"));
+ boolean isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
if (!isLocal) {
Class combinerClass = conf.getCombinerClass();
CombineOutputCollector combineCollector =
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/SpillRecord.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SpillRecord.java Wed Aug 17 22:57:31 2011
@@ -34,7 +34,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SecureIOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.PureJavaCrc32;
-
+import org.apache.hadoop.fs.FSDataInputStream;
import static org.apache.hadoop.mapred.MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH;
class SpillRecord {
@@ -64,9 +64,7 @@ class SpillRecord {
throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
- final DataInputStream in =
- new DataInputStream(SecureIOUtils.openForRead(
- new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
+ final FSDataInputStream in = rfs.open(indexFileName);
try {
final long length = rfs.getFileStatus(indexFileName).getLen();
final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Wed Aug 17 22:57:31 2011
@@ -61,9 +61,8 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin.*;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.*;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.Progressable;
@@ -487,12 +486,6 @@ abstract public class Task implements Wr
public abstract void run(JobConf job, TaskUmbilicalProtocol umbilical)
throws IOException, ClassNotFoundException, InterruptedException;
-
- /** Return an approprate thread runner for this task.
- * @param tip TODO*/
- public abstract TaskRunner createRunner(TaskTracker tracker,
- TaskTracker.TaskInProgress tip) throws IOException;
-
/** The number of milliseconds between progress reports. */
public static final int PROGRESS_INTERVAL = 3000;
@@ -539,7 +532,7 @@ abstract public class Task implements Wr
}
committer.setupTask(taskContext);
Class<? extends ResourceCalculatorPlugin> clazz =
- conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
+ conf.getClass(MRConfig.RESOURCE_CALCULATOR_PLUGIN,
null, ResourceCalculatorPlugin.class);
resourceCalculator = ResourceCalculatorPlugin
.getResourceCalculatorPlugin(clazz, conf);
@@ -1161,7 +1154,7 @@ abstract public class Task implements Wr
// add the static resolutions (this is required for the junit to
// work on testcases that simulate multiple nodes on a single physical
// node.
- String hostToResolved[] = conf.getStrings(TTConfig.TT_STATIC_RESOLUTIONS);
+ String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
if (hostToResolved != null) {
for (String str : hostToResolved) {
String name = str.substring(0, str.indexOf('='));
@@ -1175,6 +1168,10 @@ abstract public class Task implements Wr
return this.conf;
}
+ public MapOutputFile getMapOutputFile() {
+ return mapOutputFile;
+ }
+
/**
* OutputCollector for the combiner.
*/
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java Wed Aug 17 22:57:31 2011
@@ -33,12 +33,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistory;
+import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -281,8 +279,8 @@ public class Cluster {
if (jobHistoryDir == null) {
jobHistoryDir = new Path(client.getJobHistoryDir());
}
- return JobHistory.getJobHistoryFile(jobHistoryDir, jobId,
- ugi.getShortUserName()).toString();
+ return new Path(jobHistoryDir, jobId.toString() + "_"
+ + ugi.getShortUserName()).toString();
}
/**
@@ -314,19 +312,6 @@ public class Cluster {
throws IOException, InterruptedException {
return client.getChildQueues(queueName);
}
-
- /**
- * Get JobTracker's state
- *
- * @return {@link State} of the JobTracker
- * @throws IOException
- * @throws InterruptedException
- * @deprecated Use {@link #getJobTrackerStatus()} instead.
- */
- @Deprecated
- public State getJobTrackerState() throws IOException, InterruptedException {
- return client.getJobTrackerState();
- }
/**
* Get the JobTracker's status.
@@ -359,8 +344,7 @@ public class Cluster {
getDelegationToken(Text renewer) throws IOException, InterruptedException{
Token<DelegationTokenIdentifier> result =
client.getDelegationToken(renewer);
- InetSocketAddress addr = NetUtils.createSocketAddr(
- conf.get(JTConfig.JT_IPC_ADDRESS, "localhost:8012"));
+ InetSocketAddress addr = Master.getMasterAddress(conf);
StringBuilder service = new StringBuilder();
service.append(NetUtils.normalizeHostName(addr.getAddress().
getHostAddress()));
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/JobCounter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java Wed Aug 17 22:57:31 2011
@@ -35,5 +35,9 @@ public enum JobCounter {
SLOTS_MILLIS_MAPS,
SLOTS_MILLIS_REDUCES,
FALLOW_SLOTS_MILLIS_MAPS,
- FALLOW_SLOTS_MILLIS_REDUCES
+ FALLOW_SLOTS_MILLIS_REDUCES,
+ TOTAL_LAUNCHED_UBERTASKS,
+ NUM_UBER_SUBMAPS,
+ NUM_UBER_SUBREDUCES,
+ NUM_FAILED_UBERTASKS
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java Wed Aug 17 22:57:31 2011
@@ -18,8 +18,6 @@
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapred.TaskTracker;
/**
* Place holder for cluster level configuration keys.
@@ -58,6 +56,15 @@ public interface MRConfig {
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
+ public static final String RESOURCE_CALCULATOR_PLUGIN =
+ "mapreduce.job.resourcecalculatorplugin";
+ public static final String STATIC_RESOLUTIONS =
+ "mapreduce.job.net.static.resolutions";
+
+ public static final String MASTER_ADDRESS = "mapreduce.jobtracker.address";
+ public static final String MASTER_USER_NAME =
+ "mapreduce.jobtracker.kerberos.principal";
+
public static final String FRAMEWORK_NAME = "mapreduce.framework.name";
public static final String TASK_LOCAL_OUTPUT_CLASS =
"mapreduce.task.local.output.class";
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Wed Aug 17 22:57:31 2011
@@ -287,4 +287,18 @@ public interface MRJobConfig {
public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max";
public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
+ public static final String JOB_UBERTASK_ENABLE =
+ "mapreduce.job.ubertask.enable";
+ public static final String JOB_UBERTASK_MAXMAPS =
+ "mapreduce.job.ubertask.maxmaps";
+ public static final String JOB_UBERTASK_MAXREDUCES =
+ "mapreduce.job.ubertask.maxreduces";
+ public static final String JOB_UBERTASK_MAXBYTES =
+ "mapreduce.job.ubertask.maxbytes";
+ public static final String UBERTASK_JAVA_OPTS =
+ "mapreduce.ubertask.child.java.opts"; // or mapreduce.uber.java.opts?
+ public static final String UBERTASK_ULIMIT =
+ "mapreduce.ubertask.child.ulimit"; // or mapreduce.uber.ulimit?
+ public static final String UBERTASK_ENV =
+ "mapreduce.ubertask.child.env"; // or mapreduce.uber.env?
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java Wed Aug 17 22:57:31 2011
@@ -377,6 +377,38 @@ public class DistributedCache {
return false;
}
+ private static boolean[] parseBooleans(String[] strs) {
+ if (null == strs) {
+ return null;
+ }
+ boolean[] result = new boolean[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Boolean.parseBoolean(strs[i]);
+ }
+ return result;
+ }
+
+ /**
+ * Get the booleans on whether the files are public or not. Used by
+ * internal DistributedCache and MapReduce code.
+ * @param conf The configuration which stored the timestamps
+ * @return a string array of booleans
+ * @throws IOException
+ */
+ public static boolean[] getFileVisibilities(Configuration conf) {
+ return parseBooleans(conf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES));
+ }
+
+ /**
+ * Get the booleans on whether the archives are public or not. Used by
+ * internal DistributedCache and MapReduce code.
+ * @param conf The configuration which stored the timestamps
+ * @return a string array of booleans
+ */
+ public static boolean[] getArchiveVisibilities(Configuration conf) {
+ return parseBooleans(conf.getStrings(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES));
+ }
+
/**
* This method checks if there is a conflict in the fragment names
* of the uris. Also makes sure that each uri has a fragment. It
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AvroArrayUtils.java Wed Aug 17 22:57:31 2011
@@ -18,12 +18,11 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.lang.Integer;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import org.apache.avro.Schema;
-
-import org.apache.avro.generic.GenericArray;
import org.apache.avro.generic.GenericData;
public class AvroArrayUtils {
@@ -31,13 +30,12 @@ public class AvroArrayUtils {
private static final Schema ARRAY_INT
= Schema.createArray(Schema.create(Schema.Type.INT));
- static public GenericArray<Integer> NULL_PROGRESS_SPLITS_ARRAY
+ static public List<Integer> NULL_PROGRESS_SPLITS_ARRAY
= new GenericData.Array<Integer>(0, ARRAY_INT);
- public static GenericArray<Integer>
+ public static List<Integer>
toAvro(int values[]) {
- GenericData.Array<Integer> result
- = new GenericData.Array<Integer>(values.length, ARRAY_INT);
+ List<Integer> result = new ArrayList<Integer>(values.length);
for (int i = 0; i < values.length; ++i) {
result.add(values[i]);
@@ -46,7 +44,7 @@ public class AvroArrayUtils {
return result;
}
- public static int[] fromAvro(GenericArray<Integer> avro) {
+ public static int[] fromAvro(List<Integer> avro) {
int[] result = new int[(int)avro.size()];
int i = 0;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Wed Aug 17 22:57:31 2011
@@ -19,23 +19,21 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
+import java.util.ArrayList;
import org.apache.avro.Schema;
+import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Encoder;
import org.apache.avro.io.JsonEncoder;
-import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
import org.apache.avro.util.Utf8;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
/**
* Event Writer is an utility class used to write events to the underlying
@@ -97,13 +95,13 @@ class EventWriter {
static JhCounters toAvro(Counters counters, String name) {
JhCounters result = new JhCounters();
result.name = new Utf8(name);
- result.groups = new GenericData.Array<JhCounterGroup>(0, GROUPS);
+ result.groups = new ArrayList<JhCounterGroup>(0);
if (counters == null) return result;
for (CounterGroup group : counters) {
JhCounterGroup g = new JhCounterGroup();
g.name = new Utf8(group.getName());
g.displayName = new Utf8(group.getDisplayName());
- g.counts = new GenericData.Array<JhCounter>(group.size(), COUNTERS);
+ g.counts = new ArrayList<JhCounter>(group.size());
for (Counter counter : group) {
JhCounter c = new JhCounter();
c.name = new Utf8(counter.getName());
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java Wed Aug 17 22:57:31 2011
@@ -34,9 +34,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.JobInProgress;
import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.TaskLogServlet;
import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
@@ -45,6 +43,7 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.util.StringUtils;
/**
@@ -456,9 +455,9 @@ public class HistoryViewer {
}
String taskTrackerName =
- JobInProgress.convertTrackerNameToHostName(
+ HostUtil.convertTrackerNameToHostName(
attempt.getTrackerName());
- return TaskLogServlet.getTaskLogUrl(taskTrackerName,
+ return HostUtil.getTaskLogUrl(taskTrackerName,
Integer.toString(attempt.getHttpPort()),
attempt.getAttemptId().toString());
}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Wed Aug 17 22:57:31 2011
@@ -57,7 +57,7 @@ public class JobSubmittedEvent implement
datum.userName = new Utf8(userName);
datum.submitTime = submitTime;
datum.jobConfPath = new Utf8(jobConfPath);
- Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+ Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
entry.getValue().getAclString()));
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Wed Aug 17 22:57:31 2011
@@ -39,7 +39,6 @@ import org.apache.hadoop.mapreduce.TaskT
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
-import org.apache.hadoop.mapreduce.server.jobtracker.State;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -140,17 +139,6 @@ public interface ClientProtocol extends
*/
public ClusterMetrics getClusterMetrics()
throws IOException, InterruptedException;
-
- /**
- * Get JobTracker's state
- *
- * @return {@link State} of the JobTracker
- * @throws IOException
- * @throws InterruptedException
- * @deprecated Use {@link #getJobTrackerStatus()} instead.
- */
- @Deprecated
- public State getJobTrackerState() throws IOException, InterruptedException;
/**
* Get the JobTracker's status.
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Wed Aug 17 22:57:31 2011
@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.Master;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.Credentials;
@@ -91,13 +91,6 @@ public class TokenCache {
}
}
- static String getJTPrincipal(Configuration conf) throws IOException {
- String jtHostname = JobTracker.getAddress(conf).getHostName();
- // get jobtracker principal for use as delegation token renewer
- return SecurityUtil.getServerPrincipal(conf.get(JTConfig.JT_USER_NAME),
- jtHostname);
- }
-
/**
* get delegation token for a specific FS
* @param fs
@@ -108,7 +101,7 @@ public class TokenCache {
*/
static void obtainTokensForNamenodesInternal(FileSystem fs,
Credentials credentials, Configuration conf) throws IOException {
- String delegTokenRenewer = getJTPrincipal(conf);
+ String delegTokenRenewer = Master.getMasterPrincipal(conf);
if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
throw new IOException(
"Can't get JobTracker Kerberos principal for use as renewer");
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/EventFetcher.java Wed Aug 17 22:57:31 2011
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate;
import org.apache.hadoop.mapred.TaskCompletionEvent;
-import org.apache.hadoop.mapred.TaskTracker;
import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
import org.apache.hadoop.mapreduce.TaskAttemptID;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java Wed Aug 17 22:57:31 2011
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BoundedByteArrayOutputStream;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.TaskTracker;
+import org.apache.hadoop.mapred.MapOutputFile;
import org.apache.hadoop.mapreduce.TaskAttemptID;
class MapOutput<K,V> {
@@ -64,7 +64,8 @@ class MapOutput<K,V> {
MapOutput(TaskAttemptID mapId, MergeManager<K,V> merger, long size,
JobConf conf, LocalDirAllocator localDirAllocator,
- int fetcher, boolean primaryMapOutput) throws IOException {
+ int fetcher, boolean primaryMapOutput, MapOutputFile mapOutputFile)
+ throws IOException {
this.id = ID.incrementAndGet();
this.mapId = mapId;
this.merger = merger;
@@ -77,18 +78,10 @@ class MapOutput<K,V> {
this.size = size;
this.localFS = FileSystem.getLocal(conf);
- String filename = "map_" + mapId.getTaskID().getId() + ".out";
- String tmpOutput = Path.SEPARATOR +
- TaskTracker.getJobCacheSubdir(conf.getUser()) +
- Path.SEPARATOR + mapId.getJobID() +
- Path.SEPARATOR + merger.getReduceId() +
- Path.SEPARATOR + "output" +
- Path.SEPARATOR + filename +
- "." + fetcher;
-
- tmpOutputPath =
- localDirAllocator.getLocalPathForWrite(tmpOutput, size, conf);
- outputPath = new Path(tmpOutputPath.getParent(), filename);
+ outputPath =
+ mapOutputFile.getInputFileForWrite(mapId.getTaskID(),size);
+ tmpOutputPath = outputPath.suffix(String.valueOf(fetcher));
+
disk = localFS.create(tmpOutputPath);
this.primaryMapOutput = primaryMapOutput;
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Wed Aug 17 22:57:31 2011
@@ -225,7 +225,8 @@ public class MergeManager<K, V> {
" is greater than maxSingleShuffleLimit (" +
maxSingleShuffleLimit + ")");
return new MapOutput<K,V>(mapId, this, requestedSize, jobConf,
- localDirAllocator, fetcher, true);
+ localDirAllocator, fetcher, true,
+ mapOutputFile);
}
// Stall shuffle if we are above the memory limit
Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,33 @@
+package org.apache.hadoop.mapreduce.util;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public class HostUtil {
+
+ /**
+ * Construct the taskLogUrl
+ * @param taskTrackerHostName
+ * @param httpPort
+ * @param taskAttemptID
+ * @return the taskLogUrl
+ */
+ public static String getTaskLogUrl(String taskTrackerHostName,
+ String httpPort, String taskAttemptID) {
+ return ("http://" + taskTrackerHostName + ":" + httpPort
+ + "/tasklog?attemptid=" + taskAttemptID);
+ }
+
+ public static String convertTrackerNameToHostName(String trackerName) {
+ // Ugly!
+ // Convert the trackerName to its host name
+ int indexOfColon = trackerName.indexOf(":");
+ String trackerHostName = (indexOfColon == -1) ?
+ trackerName :
+ trackerName.substring(0, indexOfColon);
+ return trackerHostName.substring("tracker_".length());
+ }
+
+}
Copied: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (from r1158949, hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java)
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java?p2=hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&p1=hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java&r1=1158949&r2=1158962&rev=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/mapreduce/src/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java Wed Aug 17 22:57:31 2011
@@ -29,7 +29,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.mapred.TaskTrackerStatus;
/**
* Plugin to calculate resource information on Linux systems.
@@ -40,6 +39,8 @@ public class LinuxResourceCalculatorPlug
private static final Log LOG =
LogFactory.getLog(LinuxResourceCalculatorPlugin.class);
+ public static final int UNAVAILABLE = -1;
+
/**
* proc's meminfo virtual file has keys-values in the format
* "key:[ \t]*value[ \t]kB".
@@ -87,9 +88,9 @@ public class LinuxResourceCalculatorPlug
private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms)
private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms)
// Unix timestamp while reading the CPU time (ms)
- private float cpuUsage = TaskTrackerStatus.UNAVAILABLE;
- private long sampleTime = TaskTrackerStatus.UNAVAILABLE;
- private long lastSampleTime = TaskTrackerStatus.UNAVAILABLE;
+ private float cpuUsage = UNAVAILABLE;
+ private long sampleTime = UNAVAILABLE;
+ private long lastSampleTime = UNAVAILABLE;
private ProcfsBasedProcessTree pTree = null;
boolean readMemInfoFile = false;
@@ -352,7 +353,7 @@ public class LinuxResourceCalculatorPlug
public float getCpuUsage() {
readProcStatFile();
sampleTime = getCurrentTime();
- if (lastSampleTime == TaskTrackerStatus.UNAVAILABLE ||
+ if (lastSampleTime == UNAVAILABLE ||
lastSampleTime > sampleTime) {
// lastSampleTime > sampleTime may happen when the system time is changed
lastSampleTime = sampleTime;
Added: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java?rev=1158962&view=auto
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java (added)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java Wed Aug 17 22:57:31 2011
@@ -0,0 +1,236 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.util;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin;
+import org.junit.Test;
+
+/**
+ * A JUnit test to test {@link LinuxResourceCalculatorPlugin}
+ * Create the fake /proc/ information and verify the parsing and calculation
+ */
+public class TestLinuxResourceCalculatorPlugin extends TestCase {
+ /**
+ * LinuxResourceCalculatorPlugin with a fake timer
+ */
+ static class FakeLinuxResourceCalculatorPlugin extends
+ LinuxResourceCalculatorPlugin {
+
+ long currentTime = 0;
+ public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
+ String procfsCpuFile,
+ String procfsStatFile,
+ long jiffyLengthInMillis) {
+ super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis);
+ }
+ @Override
+ long getCurrentTime() {
+ return currentTime;
+ }
+ public void advanceTime(long adv) {
+ currentTime += adv * jiffyLengthInMillis;
+ }
+ }
+ private static final FakeLinuxResourceCalculatorPlugin plugin;
+ private static String TEST_ROOT_DIR = new Path(System.getProperty(
+ "test.build.data", "/tmp")).toString().replace(' ', '+');
+ private static final String FAKE_MEMFILE;
+ private static final String FAKE_CPUFILE;
+ private static final String FAKE_STATFILE;
+ private static final long FAKE_JIFFY_LENGTH = 10L;
+ static {
+ int randomNum = (new Random()).nextInt(1000000000);
+ FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
+ FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
+ FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
+ plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
+ FAKE_STATFILE,
+ FAKE_JIFFY_LENGTH);
+ }
+ static final String MEMINFO_FORMAT =
+ "MemTotal: %d kB\n" +
+ "MemFree: %d kB\n" +
+ "Buffers: 138244 kB\n" +
+ "Cached: 947780 kB\n" +
+ "SwapCached: 142880 kB\n" +
+ "Active: 3229888 kB\n" +
+ "Inactive: %d kB\n" +
+ "SwapTotal: %d kB\n" +
+ "SwapFree: %d kB\n" +
+ "Dirty: 122012 kB\n" +
+ "Writeback: 0 kB\n" +
+ "AnonPages: 2710792 kB\n" +
+ "Mapped: 24740 kB\n" +
+ "Slab: 132528 kB\n" +
+ "SReclaimable: 105096 kB\n" +
+ "SUnreclaim: 27432 kB\n" +
+ "PageTables: 11448 kB\n" +
+ "NFS_Unstable: 0 kB\n" +
+ "Bounce: 0 kB\n" +
+ "CommitLimit: 4125904 kB\n" +
+ "Committed_AS: 4143556 kB\n" +
+ "VmallocTotal: 34359738367 kB\n" +
+ "VmallocUsed: 1632 kB\n" +
+ "VmallocChunk: 34359736375 kB\n" +
+ "HugePages_Total: 0\n" +
+ "HugePages_Free: 0\n" +
+ "HugePages_Rsvd: 0\n" +
+ "Hugepagesize: 2048 kB";
+
+ static final String CPUINFO_FORMAT =
+ "processor : %s\n" +
+ "vendor_id : AuthenticAMD\n" +
+ "cpu family : 15\n" +
+ "model : 33\n" +
+ "model name : Dual Core AMD Opteron(tm) Processor 280\n" +
+ "stepping : 2\n" +
+ "cpu MHz : %f\n" +
+ "cache size : 1024 KB\n" +
+ "physical id : 0\n" +
+ "siblings : 2\n" +
+ "core id : 0\n" +
+ "cpu cores : 2\n" +
+ "fpu : yes\n" +
+ "fpu_exception : yes\n" +
+ "cpuid level : 1\n" +
+ "wp : yes\n" +
+ "flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " +
+ "pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " +
+ "3dnowext 3dnow pni lahf_lm cmp_legacy\n" +
+ "bogomips : 4792.41\n" +
+ "TLB size : 1024 4K pages\n" +
+ "clflush size : 64\n" +
+ "cache_alignment : 64\n" +
+ "address sizes : 40 bits physical, 48 bits virtual\n" +
+ "power management: ts fid vid ttp";
+
+ static final String STAT_FILE_FORMAT =
+ "cpu %d %d %d 1646495089 831319 48713 164346 0\n" +
+ "cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" +
+ "cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" +
+ "cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" +
+ "cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" +
+ "intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" +
+ "ctxt 242017731764\n" +
+ "btime 1257808753\n" +
+ "processes 26414943\n" +
+ "procs_running 1\n" +
+ "procs_blocked 0\n";
+
+ /**
+ * Test parsing /proc/stat and /proc/cpuinfo
+ * @throws IOException
+ */
+ @Test
+ public void testParsingProcStatAndCpuFile() throws IOException {
+ // Write fake /proc/cpuinfo file.
+ long numProcessors = 8;
+ long cpuFrequencyKHz = 2392781;
+ String fileContent = "";
+ for (int i = 0; i < numProcessors; i++) {
+ fileContent += String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D) +
+ "\n";
+ }
+ File tempFile = new File(FAKE_CPUFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
+ fWriter.write(fileContent);
+ fWriter.close();
+ assertEquals(plugin.getNumProcessors(), numProcessors);
+ assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz);
+
+ // Write fake /proc/stat file.
+ long uTime = 54972994;
+ long nTime = 188860;
+ long sTime = 19803373;
+ tempFile = new File(FAKE_STATFILE);
+ tempFile.deleteOnExit();
+ updateStatFile(uTime, nTime, sTime);
+ assertEquals(plugin.getCumulativeCpuTime(),
+ FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+ assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE));
+
+ // Advance the time and sample again to test the CPU usage calculation
+ uTime += 100L;
+ plugin.advanceTime(200L);
+ updateStatFile(uTime, nTime, sTime);
+ assertEquals(plugin.getCumulativeCpuTime(),
+ FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+ assertEquals(plugin.getCpuUsage(), 6.25F);
+
+ // Advance the time and sample again. This time, we call getCpuUsage() only.
+ uTime += 600L;
+ plugin.advanceTime(300L);
+ updateStatFile(uTime, nTime, sTime);
+ assertEquals(plugin.getCpuUsage(), 25F);
+
+ // Advance very short period of time (one jiffy length).
+ // In this case, CPU usage should not be updated.
+ uTime += 1L;
+ plugin.advanceTime(1L);
+ updateStatFile(uTime, nTime, sTime);
+ assertEquals(plugin.getCumulativeCpuTime(),
+ FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
+ assertEquals(plugin.getCpuUsage(), 25F); // CPU usage is not updated.
+ }
+
+ /**
+ * Write information to fake /proc/stat file
+ */
+ private void updateStatFile(long uTime, long nTime, long sTime)
+ throws IOException {
+ FileWriter fWriter = new FileWriter(FAKE_STATFILE);
+ fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime));
+ fWriter.close();
+ }
+
+ /**
+ * Test parsing /proc/meminfo
+ * @throws IOException
+ */
+ @Test
+ public void testParsingProcMemFile() throws IOException {
+ long memTotal = 4058864L;
+ long memFree = 99632L;
+ long inactive = 567732L;
+ long swapTotal = 2096472L;
+ long swapFree = 1818480L;
+ File tempFile = new File(FAKE_MEMFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+ fWriter.write(String.format(MEMINFO_FORMAT,
+ memTotal, memFree, inactive, swapTotal, swapFree));
+
+ fWriter.close();
+ assertEquals(plugin.getAvailablePhysicalMemorySize(),
+ 1024L * (memFree + inactive));
+ assertEquals(plugin.getAvailableVirtualMemorySize(),
+ 1024L * (memFree + inactive + swapFree));
+ assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
+ assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
+ }
+}
Modified: hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1158962&r1=1158949&r2=1158962&view=diff
==============================================================================
--- hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/MR-279-merge/hadoop-mapreduce/hadoop-mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Wed Aug 17 22:57:31 2011
@@ -94,12 +94,12 @@ public class CompletedJob implements org
@Override
public int getCompletedMaps() {
- return jobInfo.getFinishedMaps();
+ return (int) jobInfo.getFinishedMaps();
}
@Override
public int getCompletedReduces() {
- return jobInfo.getFinishedReduces();
+ return (int) jobInfo.getFinishedReduces();
}
@Override
@@ -264,17 +264,17 @@ public class CompletedJob implements org
@Override
public int getTotalMaps() {
- return jobInfo.getTotalMaps();
+ return (int) jobInfo.getTotalMaps();
}
@Override
public int getTotalReduces() {
- return jobInfo.getTotalReduces();
+ return (int) jobInfo.getTotalReduces();
}
@Override
public boolean isUber() {
- return jobInfo.getIsUber();
+ throw new YarnException("Not yet implemented!");
}
@Override