You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/04/24 21:05:20 UTC
svn commit: r1329947 [1/4] - in
/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/
hadoop-mapreduce-...
Author: szetszwo
Date: Tue Apr 24 19:05:09 2012
New Revision: 1329947
URL: http://svn.apache.org/viewvc?rev=1329947&view=rev
Log:
Merge r1327258 through r1329943 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
- copied unchanged from r1329943, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/resources/yarn-site.xml
Modified:
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryStorage.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/ipc/TestSocketFactory.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/RpcServerFactory.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RpcServerFactoryPBImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/YarnRPC.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/pom.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/mapred-default.xml
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1327258-1329943
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt Tue Apr 24 19:05:09 2012
@@ -50,8 +50,16 @@ Trunk (unreleased changes)
MAPREDUCE-3935. Annotate Counters.Counter and Counters.Group as @Public.
(tomwhite)
+ HADOOP-8285 MR changes for Use ProtoBuf for RpcPayLoadHeader (sanjay radia)
+
BUG FIXES
+ MAPREDUCE-4100. [Gridmix] Bug fixed in compression emulation feature for
+ map only jobs. (amarrk)
+
+ MAPREDUCE-4149. [Rumen] Rumen fails to parse certain counter
+ strings. (ravigummadi)
+
MAPREDUCE-4083. [Gridmix] NPE in cpu emulation. (amarrk)
MAPREDUCE-4087. [Gridmix] GenerateDistCacheData job of Gridmix can
@@ -143,6 +151,9 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4103. Fix HA docs for changes to shell command fencer args (todd)
+ MAPREDUCE-4093. Improve RM WebApp start up when proxy address is not set
+ (Devaraj K vai bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -244,6 +255,11 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4008. ResourceManager throws MetricsException on start up
saying QueueMetrics MBean already exists (Devaraj K via tgraves)
+ MAPREDUCE-3867. MiniMRYarn/MiniYarn uses fixed ports (tucu)
+
+ MAPREDUCE-4141. clover integration broken, also mapreduce poms are
+ pulling in clover as a dependency. (phunt via tucu)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -251,6 +267,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4072. User set java.library.path seems to overwrite default
creating problems native lib loading (Anupam Seth via bobby)
+ MAPREDUCE-3812. Lower default allocation sizes, fix allocation
+ configurations and document them (Harsh J via bobby)
+
NEW FEATURES
IMPROVEMENTS
@@ -262,6 +281,20 @@ Release 0.23.3 - UNRELEASED
ApplicationTokenSecretManager and roll it every so often. (Vinod Kumar
Vavilapalli via sseth)
+ MAPREDUCE-4151. RM scheduler web page should filter apps to those that
+ are relevant to scheduling (Jason Lowe via tgraves)
+
+ MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they
+ are not being used any more (Ravi Prakash via bobby)
+
+ MAPREDUCE-3972. Fix locking and exception issues in JobHistory server.
+ (Robert Joseph Evans via sseth)
+
+ MAPREDUCE-4161. create sockets consistently (Daryn Sharp via bobby)
+
+ MAPREDUCE-4079. Allow MR AppMaster to limit ephemeral port range.
+ (bobby via tgraves)
+
OPTIMIZATIONS
BUG FIXES
@@ -350,6 +383,26 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4160. some mrv1 ant tests fail with timeout - due to 4156
(tgraves)
+ MAPREDUCE-4074. Client continuously retries to RM When RM goes down
+ before launching Application Master (xieguiming via tgraves)
+
+ MAPREDUCE-4159. Job is running in Uber mode after setting
+ "mapreduce.job.ubertask.maxreduces" to zero (Devaraj K via bobby)
+
+ MAPREDUCE-4165. Committing is misspelled as commiting in task logs
+ (John Eagles via bobby)
+
+ MAPREDUCE-4129. Lots of unneeded counters log messages (Ahmed Radwan via
+ bobby)
+
+ MAPREDUCE-3947. yarn.app.mapreduce.am.resource.mb not documented
+ (Devaraj K via bobby)
+
+ MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
+ via bobby)
+
+ MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1327258-1329943
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Tue Apr 24 19:05:09 2012
@@ -139,7 +139,8 @@ public class MRClientService extends Abs
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
conf, secretManager,
conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT,
- MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT));
+ MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
+ MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
// Enable service authorization?
if (conf.getBoolean(
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java Tue Apr 24 19:05:09 2012
@@ -18,9 +18,11 @@
package org.apache.hadoop.mapreduce.v2.app.job;
+import java.io.IOException;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
@@ -72,6 +74,13 @@ public interface Job {
Path getConfFile();
/**
+ * @return a parsed version of the config files pointed to by
+ * {@link #getConfFile()}.
+ * @throws IOException on any error trying to load the conf file.
+ */
+ Configuration loadConfFile() throws IOException;
+
+ /**
* @return the ACLs for this job for each type of JobACL given.
*/
Map<JobACL, AccessControlList> getJobACLs();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Apr 24 19:05:09 2012
@@ -37,6 +37,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -821,12 +822,12 @@ public class JobImpl implements org.apac
//FIXME: handling multiple reduces within a single AM does not seem to
//work.
- // int sysMaxReduces =
- // job.conf.getInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 1);
- int sysMaxReduces = 1;
+ int sysMaxReduces = conf.getInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 1);
+ boolean isValidUberMaxReduces = (sysMaxReduces == 0)
+ || (sysMaxReduces == 1);
long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
- fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from
+ fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS from
// [File?]InputFormat and default block size
// from that
@@ -855,7 +856,7 @@ public class JobImpl implements org.apac
// while "uber-AM" (MR AM + LocalContainerLauncher) loops over tasks
// and thus requires sequential execution.
isUber = uberEnabled && smallNumMapTasks && smallNumReduceTasks
- && smallInput && smallMemory && notChainJob;
+ && smallInput && smallMemory && notChainJob && isValidUberMaxReduces;
if (isUber) {
LOG.info("Uberizing job " + jobId + ": " + numMapTasks + "m+"
@@ -888,7 +889,9 @@ public class JobImpl implements org.apac
if (!smallMemory)
msg.append(" too much RAM;");
if (!notChainJob)
- msg.append(" chainjob");
+ msg.append(" chainjob;");
+ if (!isValidUberMaxReduces)
+ msg.append(" not supported uber max reduces");
LOG.info(msg.toString());
}
}
@@ -1472,4 +1475,13 @@ public class JobImpl implements org.apac
job.finished(JobState.ERROR);
}
}
+
+ @Override
+ public Configuration loadConfFile() throws IOException {
+ Path confPath = getConfFile();
+ FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
+ Configuration jobConf = new Configuration(false);
+ jobConf.addResource(fc.open(confPath));
+ return jobConf;
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java Tue Apr 24 19:05:09 2012
@@ -19,6 +19,7 @@
package org.apache.hadoop.mapreduce.v2.app.rm;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -35,7 +36,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -245,11 +245,12 @@ public abstract class RMCommunicator ext
}
protected AMRMProtocol createSchedulerProxy() {
- final YarnRPC rpc = YarnRPC.create(getConfig());
final Configuration conf = getConfig();
- final String serviceAddr = conf.get(
+ final YarnRPC rpc = YarnRPC.create(conf);
+ final InetSocketAddress serviceAddr = conf.getSocketAddr(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
UserGroupInformation currentUser;
try {
@@ -279,7 +280,7 @@ public abstract class RMCommunicator ext
@Override
public AMRMProtocol run() {
return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
- NetUtils.createSocketAddr(serviceAddr), conf);
+ serviceAddr, conf);
}
});
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java Tue Apr 24 19:05:09 2012
@@ -31,7 +31,6 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -68,14 +67,11 @@ import com.google.inject.Inject;
public class AMWebServices {
private final AppContext appCtx;
private final App app;
- private final Configuration conf;
-
+
@Inject
- public AMWebServices(final App app, final AppContext context,
- final Configuration conf) {
+ public AMWebServices(final App app, final AppContext context) {
this.appCtx = context;
this.app = app;
- this.conf = conf;
}
Boolean hasAccess(Job job, HttpServletRequest request) {
@@ -272,7 +268,7 @@ public class AMWebServices {
checkAccess(job, hsr);
ConfInfo info;
try {
- info = new ConfInfo(job, this.conf);
+ info = new ConfInfo(job);
} catch (IOException e) {
throw new NotFoundException("unable to load configuration for job: "
+ jid);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Apr 24 19:05:09 2012
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.web
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
@@ -44,11 +43,9 @@ import com.google.inject.Inject;
*/
public class ConfBlock extends HtmlBlock {
final AppContext appContext;
- final Configuration conf;
- @Inject ConfBlock(AppContext appctx, Configuration conf) {
+ @Inject ConfBlock(AppContext appctx) {
appContext = appctx;
- this.conf = conf;
}
/*
@@ -71,7 +68,7 @@ public class ConfBlock extends HtmlBlock
}
Path confPath = job.getConfFile();
try {
- ConfInfo info = new ConfInfo(job, this.conf);
+ ConfInfo info = new ConfInfo(job);
html.div().h3(confPath.toString())._();
TBODY<TABLE<Hamlet>> tbody = html.
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Tue Apr 24 19:05:09 2012
@@ -66,7 +66,8 @@ public class TaskPage extends AppView {
th(".id", "Attempt").
th(".progress", "Progress").
th(".state", "State").
- th(".node", "node").
+ th(".node", "Node").
+ th(".logs", "Logs").
th(".tsh", "Started").
th(".tsh", "Finished").
th(".tsh", "Elapsed").
@@ -83,24 +84,28 @@ public class TaskPage extends AppView {
long finishTime = ta.getFinishTime();
long elapsed = ta.getElapsedTime();
String diag = ta.getNote() == null ? "" : ta.getNote();
- TD<TR<TBODY<TABLE<Hamlet>>>> nodeTd = tbody.
- tr().
- td(".id", taid).
- td(".progress", progress).
- td(".state", ta.getState()).td();
+ TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
+ TD<TR<TBODY<TABLE<Hamlet>>>> nodeTd = row.
+ td(".id", taid).
+ td(".progress", progress).
+ td(".state", ta.getState()).td();
if (nodeHttpAddr == null) {
nodeTd._("N/A");
} else {
nodeTd.
a(".nodelink", url("http://", nodeHttpAddr), nodeHttpAddr);
}
+ nodeTd._();
if (containerId != null) {
String containerIdStr = ta.getAssignedContainerIdStr();
- nodeTd._(" ").
+ row.td().
a(".logslink", url("http://", nodeHttpAddr, "node", "containerlogs",
- containerIdStr, app.getJob().getUserName()), "logs");
+ containerIdStr, app.getJob().getUserName()), "logs")._();
+ } else {
+ row.td()._("N/A")._();
}
- nodeTd._().
+
+ row.
td(".ts", Times.format(startTime)).
td(".ts", Times.format(finishTime)).
td(".dt", StringUtils.formatTime(elapsed)).
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java Tue Apr 24 19:05:09 2012
@@ -40,15 +40,11 @@ public class ConfInfo {
public ConfInfo() {
}
- public ConfInfo(Job job, Configuration conf) throws IOException {
+ public ConfInfo(Job job) throws IOException {
- Path confPath = job.getConfFile();
this.property = new ArrayList<ConfEntryInfo>();
- // Read in the configuration file and put it in a key/value table.
- FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
- Configuration jobConf = new Configuration(false);
- jobConf.addResource(fc.open(confPath));
- this.path = confPath.toString();
+ Configuration jobConf = job.loadConfFile();
+ this.path = job.getConfFile().toString();
for (Map.Entry<String, String> entry : jobConf) {
this.property.add(new ConfEntryInfo(entry.getKey(), entry.getValue()));
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Tue Apr 24 19:05:09 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app;
+import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -27,6 +28,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.ShuffleHandler;
@@ -442,7 +444,7 @@ public class MockJobs extends MockApps {
final Path configFile = confFile;
Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
- Configuration conf = new Configuration();
+ final Configuration conf = new Configuration();
conf.set(JobACL.VIEW_JOB.getAclName(), "testuser");
conf.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
@@ -564,6 +566,14 @@ public class MockJobs extends MockApps {
amInfoList.add(createAMInfo(2));
return amInfoList;
}
+
+ @Override
+ public Configuration loadConfFile() throws IOException {
+ FileContext fc = FileContext.getFileContext(configFile.toUri(), conf);
+ Configuration jobConf = new Configuration(false);
+ jobConf.addResource(fc.open(configFile));
+ return jobConf;
+ }
};
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Tue Apr 24 19:05:09 2012
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.Event;
@@ -341,7 +342,7 @@ public class TestRMContainerAllocator {
}
@Override
protected ResourceScheduler createScheduler() {
- return new MyFifoScheduler(getRMContext());
+ return new MyFifoScheduler();
}
}
@@ -997,18 +998,6 @@ public class TestRMContainerAllocator {
}
private static class MyFifoScheduler extends FifoScheduler {
-
- public MyFifoScheduler(RMContext rmContext) {
- super();
- try {
- reinitialize(new Configuration(), new ContainerTokenSecretManager(),
- rmContext);
- } catch (IOException ie) {
- LOG.info("add application failed with ", ie);
- assert (false);
- }
- }
-
// override this to copy the objects otherwise FifoScheduler updates the
// numContainers in same objects as kept by RMContainerAllocator
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Tue Apr 24 19:05:09 2012
@@ -489,6 +489,11 @@ public class TestRuntimeEstimators {
public List<AMInfo> getAMInfos() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ @Override
+ public Configuration loadConfFile() {
+ throw new UnsupportedOperationException();
+ }
}
/*
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java Tue Apr 24 19:05:09 2012
@@ -37,14 +37,20 @@ import org.apache.hadoop.mapreduce.MRJob
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
+import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.InitTransition;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.JobNoTasksCompletedTransition;
+import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -233,4 +239,69 @@ public class TestJobImpl {
Assert.assertTrue(job5.checkAccess(ugi1, null));
Assert.assertTrue(job5.checkAccess(ugi2, null));
}
+ @Test
+ public void testUberDecision() throws Exception {
+
+ // with default values, no of maps is 2
+ Configuration conf = new Configuration();
+ boolean isUber = testUberDecision(conf);
+ Assert.assertFalse(isUber);
+
+ // enable uber mode, no of maps is 2
+ conf = new Configuration();
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
+ isUber = testUberDecision(conf);
+ Assert.assertTrue(isUber);
+
+ // enable uber mode, no of maps is 2, no of reduces is 1 and uber task max
+ // reduces is 0
+ conf = new Configuration();
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
+ conf.setInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 0);
+ conf.setInt(MRJobConfig.NUM_REDUCES, 1);
+ isUber = testUberDecision(conf);
+ Assert.assertFalse(isUber);
+
+ // enable uber mode, no of maps is 2, no of reduces is 1 and uber task max
+ // reduces is 1
+ conf = new Configuration();
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
+ conf.setInt(MRJobConfig.JOB_UBERTASK_MAXREDUCES, 1);
+ conf.setInt(MRJobConfig.NUM_REDUCES, 1);
+ isUber = testUberDecision(conf);
+ Assert.assertTrue(isUber);
+
+ // enable uber mode, no of maps is 2 and uber task max maps is 0
+ conf = new Configuration();
+ conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true);
+ conf.setInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 1);
+ isUber = testUberDecision(conf);
+ Assert.assertFalse(isUber);
+ }
+
+ private boolean testUberDecision(Configuration conf) {
+ JobID jobID = JobID.forName("job_1234567890000_0001");
+ JobId jobId = TypeConverter.toYarn(jobID);
+ MRAppMetrics mrAppMetrics = MRAppMetrics.create();
+ JobImpl job = new JobImpl(jobId, Records
+ .newRecord(ApplicationAttemptId.class), conf, mock(EventHandler.class),
+ null, mock(JobTokenSecretManager.class), null, null, null,
+ mrAppMetrics, mock(OutputCommitter.class), true, null, 0, null, null);
+ InitTransition initTransition = getInitTransition();
+ JobEvent mockJobEvent = mock(JobEvent.class);
+ initTransition.transition(job, mockJobEvent);
+ boolean isUber = job.isUber();
+ return isUber;
+ }
+
+ private InitTransition getInitTransition() {
+ InitTransition initTransition = new InitTransition() {
+ @Override
+ protected TaskSplitMetaInfo[] createSplits(JobImpl job, JobId jobId) {
+ return new TaskSplitMetaInfo[] { new TaskSplitMetaInfo(),
+ new TaskSplitMetaInfo() };
+ }
+ };
+ return initTransition;
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Tue Apr 24 19:05:09 2012
@@ -187,9 +187,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_TASK_ULIMIT} can be used to
- * control the maximum virtual memory of the child processes.
- *
* The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass
* other environment variables to the child processes.
*
@@ -210,9 +207,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_MAP_TASK_ULIMIT} can be used to
- * control the maximum virtual memory of the map processes.
- *
* The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass
* other environment variables to the map processes.
*/
@@ -230,9 +224,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_REDUCE_TASK_ULIMIT} can be used
- * to control the maximum virtual memory of the reduce processes.
- *
* The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to
* pass process environment variables to the reduce processes.
*/
@@ -242,36 +233,32 @@ public class JobConf extends Configurati
public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m";
/**
- * Configuration key to set the maximum virutal memory available to the child
- * map and reduce tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_TASK_JAVA_OPTS}, else the VM might not start.
- *
- * @deprecated Use {@link #MAPRED_MAP_TASK_ULIMIT} or
- * {@link #MAPRED_REDUCE_TASK_ULIMIT}
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the child
+ * map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
@Deprecated
public static final String MAPRED_TASK_ULIMIT = "mapred.child.ulimit";
/**
- * Configuration key to set the maximum virutal memory available to the
- * map tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_MAP_TASK_JAVA_OPTS}, else the VM might not start.
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the
+ * map tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
- public static final String MAPRED_MAP_TASK_ULIMIT = JobContext.MAP_ULIMIT;
+ @Deprecated
+ public static final String MAPRED_MAP_TASK_ULIMIT = "mapreduce.map.ulimit";
/**
- * Configuration key to set the maximum virutal memory available to the
- * reduce tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}, else the VM might not start.
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the
+ * reduce tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
- public static final String MAPRED_REDUCE_TASK_ULIMIT =
- JobContext.REDUCE_ULIMIT;
+ @Deprecated
+ public static final String MAPRED_REDUCE_TASK_ULIMIT =
+ "mapreduce.reduce.ulimit";
/**
@@ -1966,6 +1953,15 @@ public class JobConf extends Configurati
+ " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY
+ " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
}
+ if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT));
+ }
+ if(get(JobConf.MAPRED_MAP_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_MAP_TASK_ULIMIT));
+ }
+ if(get(JobConf.MAPRED_REDUCE_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT));
+ }
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java Tue Apr 24 19:05:09 2012
@@ -55,10 +55,10 @@ public class Master {
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
}
else {
- masterAddress = conf.get(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS);
- return NetUtils.createSocketAddr(masterAddress, YarnConfiguration.DEFAULT_RM_PORT,
- YarnConfiguration.RM_ADDRESS);
+ return conf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java Tue Apr 24 19:05:09 2012
@@ -964,7 +964,7 @@ abstract public class Task implements Wr
TaskReporter reporter
) throws IOException, InterruptedException {
LOG.info("Task:" + taskId + " is done."
- + " And is in the process of commiting");
+ + " And is in the process of committing");
updateCounters();
boolean commitRequired = isCommitRequired();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Apr 24 19:05:09 2012
@@ -183,8 +183,6 @@ public interface MRJobConfig {
public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts";
- public static final String MAP_ULIMIT = "mapreduce.map.ulimit";
-
public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts";
public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script";
@@ -243,8 +241,6 @@ public interface MRJobConfig {
public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts";
- public static final String REDUCE_ULIMIT = "mapreduce.reduce.ulimit";
-
public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir";
public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts";
@@ -329,6 +325,13 @@ public interface MRJobConfig {
MR_PREFIX + "client-am.ipc.max-retries";
public static final int DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 3;
+ /**
+ * The number of client retries to the RM/HS/AM before throwing exception.
+ */
+ public static final String MR_CLIENT_MAX_RETRIES =
+ MR_PREFIX + "client.max-retries";
+ public static final int DEFAULT_MR_CLIENT_MAX_RETRIES = 3;
+
/** The staging directory for map reduce.*/
public static final String MR_AM_STAGING_DIR =
MR_AM_PREFIX+"staging-dir";
@@ -338,12 +341,12 @@ public interface MRJobConfig {
/** The amount of memory the MR app master needs.*/
public static final String MR_AM_VMEM_MB =
MR_AM_PREFIX+"resource.mb";
- public static final int DEFAULT_MR_AM_VMEM_MB = 2048;
+ public static final int DEFAULT_MR_AM_VMEM_MB = 1536;
/** Command line arguments passed to the MR app master.*/
public static final String MR_AM_COMMAND_OPTS =
MR_AM_PREFIX+"command-opts";
- public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1536m";
+ public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m";
/** Root Logging level passed to the MR app master.*/
public static final String MR_AM_LOG_LEVEL =
@@ -370,6 +373,13 @@ public interface MRJobConfig {
MR_AM_PREFIX + "job.client.thread-count";
public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1;
+ /**
+ * Range of ports that the MapReduce AM can use when binding. Leave blank
+ * if you want all possible ports.
+ */
+ public static final String MR_AM_JOB_CLIENT_PORT_RANGE =
+ MR_AM_PREFIX + "job.client.port-range";
+
/** Enable blacklisting of nodes in the job.*/
public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE =
MR_AM_PREFIX + "job.node-blacklisting.enable";
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Tue Apr 24 19:05:09 2012
@@ -201,7 +201,15 @@ public abstract class AbstractCounters<C
* @return the group
*/
public synchronized G getGroup(String groupName) {
- String newGroupName = filterGroupName(groupName);
+
+ // filterGroupName
+ boolean groupNameInLegacyMap = true;
+ String newGroupName = legacyMap.get(groupName);
+ if (newGroupName == null) {
+ groupNameInLegacyMap = false;
+ newGroupName = Limits.filterGroupName(groupName);
+ }
+
boolean isFGroup = isFrameworkGroup(newGroupName);
G group = isFGroup ? fgroups.get(newGroupName) : groups.get(newGroupName);
if (group == null) {
@@ -212,19 +220,14 @@ public abstract class AbstractCounters<C
limits.checkGroups(groups.size() + 1);
groups.put(newGroupName, group);
}
+ if (groupNameInLegacyMap) {
+ LOG.warn("Group " + groupName + " is deprecated. Use " + newGroupName
+ + " instead");
+ }
}
return group;
}
- private String filterGroupName(String oldName) {
- String newName = legacyMap.get(oldName);
- if (newName == null) {
- return Limits.filterGroupName(oldName);
- }
- LOG.warn("Group "+ oldName +" is deprecated. Use "+ newName +" instead");
- return newName;
- }
-
/**
* Returns the total number of counters, by summing the number of counters
* in each group.
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java Tue Apr 24 19:05:09 2012
@@ -314,8 +314,6 @@ public class ConfigUtil {
new String[] {MRJobConfig.MAP_ENV});
Configuration.addDeprecation("mapred.map.child.java.opts",
new String[] {MRJobConfig.MAP_JAVA_OPTS});
- Configuration.addDeprecation("mapred.map.child.ulimit",
- new String[] {MRJobConfig.MAP_ULIMIT});
Configuration.addDeprecation("mapred.map.max.attempts",
new String[] {MRJobConfig.MAP_MAX_ATTEMPTS});
Configuration.addDeprecation("mapred.map.task.debug.script",
@@ -362,8 +360,6 @@ public class ConfigUtil {
new String[] {MRJobConfig.REDUCE_ENV});
Configuration.addDeprecation("mapred.reduce.child.java.opts",
new String[] {MRJobConfig.REDUCE_JAVA_OPTS});
- Configuration.addDeprecation("mapred.reduce.child.ulimit",
- new String[] {MRJobConfig.REDUCE_ULIMIT});
Configuration.addDeprecation("mapred.reduce.max.attempts",
new String[] {MRJobConfig.REDUCE_MAX_ATTEMPTS});
Configuration.addDeprecation("mapred.reduce.parallel.copies",
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Tue Apr 24 19:05:09 2012
@@ -411,9 +411,6 @@
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
- The configuration variable mapred.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
-
Usage of -Djava.library.path can cause programs to no longer function if
hadoop native libraries are used. These values should instead be set as part
of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
@@ -432,20 +429,6 @@
</property>
<property>
- <name>mapred.child.ulimit</name>
- <value></value>
- <description>The maximum virtual memory, in KB, of a process launched by the
- Map-Reduce framework. This can be used to control both the Mapper/Reducer
- tasks and applications using Hadoop Pipes, Hadoop Streaming etc.
- By default it is left unspecified to let cluster admins control it via
- limits.conf and other such relevant mechanisms.
-
- Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to
- JavaVM, else the VM might not start.
- </description>
-</property>
-
-<property>
<name>mapreduce.admin.user.env</name>
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native</value>
<description>Expert: Additional execution environment entries for
@@ -1254,6 +1237,14 @@
</property>
<property>
+ <name>yarn.app.mapreduce.am.job.client.port-range</name>
+ <value></value>
+ <description>Range of ports that the MapReduce AM can use when binding.
+ Leave blank if you want all possible ports.
+ For example 50000-50050,50100-50200</description>
+</property>
+
+<property>
<name>yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms</name>
<value>1000</value>
<description>The interval in ms at which the MR AppMaster should send
@@ -1267,6 +1258,19 @@
to the RM to fetch Application Status.</description>
</property>
+<property>
+ <name>yarn.app.mapreduce.client.max-retries</name>
+ <value>3</value>
+ <description>The number of client retries to the RM/HS/AM before
+ throwing exception. This is a layer above the ipc.</description>
+</property>
+
+<property>
+ <name>yarn.app.mapreduce.am.resource.mb</name>
+ <value>1536</value>
+ <description>The amount of memory the MR AppMaster needs.</description>
+</property>
+
<!-- jobhistory properties -->
<property>
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1327258-1329943
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CachedHistoryStorage.java Tue Apr 24 19:05:09 2012
@@ -34,7 +34,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
-import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.MetaInfo;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.YarnException;
@@ -82,32 +82,41 @@ public class CachedHistoryStorage extend
super(CachedHistoryStorage.class.getName());
}
- private Job loadJob(MetaInfo metaInfo) {
+ private Job loadJob(HistoryFileInfo fileInfo) {
try {
- Job job = hsManager.loadJob(metaInfo);
+ Job job = fileInfo.loadJob();
if (LOG.isDebugEnabled()) {
LOG.debug("Adding " + job.getID() + " to loaded job cache");
}
+ // We can clobber results here, but that should be OK, because it only
+ // means that we may have two identical copies of the same job floating
+ // around for a while.
loadedJobCache.put(job.getID(), job);
return job;
} catch (IOException e) {
throw new YarnException(
- "Could not find/load job: " + metaInfo.getJobId(), e);
+ "Could not find/load job: " + fileInfo.getJobId(), e);
}
}
@Override
- public synchronized Job getFullJob(JobId jobId) {
+ public Job getFullJob(JobId jobId) {
if (LOG.isDebugEnabled()) {
LOG.debug("Looking for Job " + jobId);
}
try {
- Job result = loadedJobCache.get(jobId);
- if (result == null) {
- MetaInfo metaInfo = hsManager.getMetaInfo(jobId);
- if (metaInfo != null) {
- result = loadJob(metaInfo);
+ HistoryFileInfo fileInfo = hsManager.getFileInfo(jobId);
+ Job result = null;
+ if (fileInfo != null) {
+ result = loadedJobCache.get(jobId);
+ if (result == null) {
+ result = loadJob(fileInfo);
+ } else if(fileInfo.isDeleted()) {
+ loadedJobCache.remove(jobId);
+ result = null;
}
+ } else {
+ loadedJobCache.remove(jobId);
}
return result;
} catch (IOException e) {
@@ -120,25 +129,20 @@ public class CachedHistoryStorage extend
LOG.debug("Called getAllPartialJobs()");
SortedMap<JobId, Job> result = new TreeMap<JobId, Job>();
try {
- for (MetaInfo mi : hsManager.getAllMetaInfo()) {
+ for (HistoryFileInfo mi : hsManager.getAllFileInfo()) {
if (mi != null) {
JobId id = mi.getJobId();
result.put(id, new PartialJob(mi.getJobIndexInfo(), id));
}
}
} catch (IOException e) {
- LOG.warn("Error trying to scan for all MetaInfos", e);
+ LOG.warn("Error trying to scan for all FileInfos", e);
throw new YarnException(e);
}
return result;
}
@Override
- public void jobRemovedFromHDFS(JobId jobId) {
- loadedJobCache.remove(jobId);
- }
-
- @Override
public JobsInfo getPartialJobs(Long offset, Long count, String user,
String queue, Long sBegin, Long sEnd, Long fBegin, Long fEnd,
JobState jobState) {
@@ -173,6 +177,7 @@ public class CachedHistoryStorage extend
if (end < 0) { // due to overflow
end = Long.MAX_VALUE;
}
+
for (Job job : jobs) {
if (at > end) {
break;
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Tue Apr 24 19:05:09 2012
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -71,7 +72,7 @@ public class CompletedJob implements org
private final Configuration conf;
private final JobId jobId; //Can be picked from JobInfo with a conversion.
private final String user; //Can be picked up from JobInfo
- private final Path confFile;
+ private final HistoryFileInfo info;
private JobInfo jobInfo;
private JobReport report;
AtomicBoolean tasksLoaded = new AtomicBoolean(false);
@@ -84,13 +85,14 @@ public class CompletedJob implements org
public CompletedJob(Configuration conf, JobId jobId, Path historyFile,
- boolean loadTasks, String userName, Path confFile, JobACLsManager aclsMgr)
+ boolean loadTasks, String userName, HistoryFileInfo info,
+ JobACLsManager aclsMgr)
throws IOException {
LOG.info("Loading job: " + jobId + " from file: " + historyFile);
this.conf = conf;
this.jobId = jobId;
this.user = userName;
- this.confFile = confFile;
+ this.info = info;
this.aclsMgr = aclsMgr;
loadFullHistoryData(loadTasks, historyFile);
}
@@ -134,7 +136,7 @@ public class CompletedJob implements org
report.setUser(jobInfo.getUsername());
report.setMapProgress((float) getCompletedMaps() / getTotalMaps());
report.setReduceProgress((float) getCompletedReduces() / getTotalReduces());
- report.setJobFile(confFile.toString());
+ report.setJobFile(getConfFile().toString());
String historyUrl = "N/A";
try {
historyUrl = JobHistoryUtils.getHistoryUrl(conf, jobId.getAppId());
@@ -392,7 +394,16 @@ public class CompletedJob implements org
*/
@Override
public Path getConfFile() {
- return confFile;
+ return info.getConfFile();
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.hadoop.mapreduce.v2.app.job.Job#loadConfFile()
+ */
+ @Override
+ public Configuration loadConfFile() throws IOException {
+ return info.loadConfFile();
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Tue Apr 24 19:05:09 2012
@@ -78,6 +78,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.DelegationToken;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -147,6 +148,18 @@ public class HistoryClientService extend
+ ":" + server.getPort());
LOG.info("Instantiated MRClientService at " + this.bindAddress);
+ if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ String resolvedAddress = bindAddress.getHostName() + ":" + bindAddress.getPort();
+ conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, resolvedAddress);
+
+ String hostname = getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
+ hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname;
+ int port = webApp.port();
+ resolvedAddress = hostname + ":" + port;
+ conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, resolvedAddress);
+ }
+
super.start();
}