You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ss...@apache.org on 2012/10/16 02:03:53 UTC
svn commit: r1398581 [1/9] - in
/hadoop/common/branches/MR-3902/hadoop-mapreduce-project: ./ bin/ conf/
hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-...
Author: sseth
Date: Tue Oct 16 00:02:55 2012
New Revision: 1398581
URL: http://svn.apache.org/viewvc?rev=1398581&view=rev
Log:
merge from trunk to branch MR-3902
Added:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/conf/mapred-env.sh
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttemptContainerRequest.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUtils.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUtils.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
- copied unchanged from r1398579, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
Removed:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIFile.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestUtils.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
Modified:
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/INSTALL
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred-config.sh
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRApp.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/LazyOutputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/MultipleOutputs.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/IOMapperBase.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestUmbilicalProtocolWithJobToken.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/SecondarySort.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/pom.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/sample/data.txt
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/sample/data2.txt
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/HashingDistributionPolicy.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/IdentityLocalAnalysis.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocInputFormat.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocLocalAnalysis.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocRecordReader.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/LineDocTextAndOp.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/example/RoundRobinDistributionPolicy.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/LuceneIndexFileNameFilter.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/LuceneUtil.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/MixedDeletionPolicy.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/MixedDirectory.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/RAMDirectoryUtil.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/lucene/ShardWriter.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/main/UpdateIndex.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/DocumentAndOp.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/DocumentID.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IDistributionPolicy.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IIndexUpdater.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/ILocalAnalysis.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateCombiner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateConfiguration.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateMapper.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdatePartitioner.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IndexUpdateReducer.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/IntermediateForm.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/lucene/TestMixedDirectory.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/mapred/TestDistributionPolicy.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/index/src/test/org/apache/hadoop/contrib/index/mapred/TestIndexUpdater.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/java/mapred-default.xml
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
hadoop/common/branches/MR-3902/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt Tue Oct 16 00:02:55 2012
@@ -1,6 +1,6 @@
Hadoop MapReduce Change Log
-Trunk (unreleased changes)
+Trunk (Unreleased)
INCOMPATIBLE CHANGES
@@ -61,6 +61,8 @@ Trunk (unreleased changes)
MAPREDUCE-4371. Check for cyclic dependencies in Jobcontrol job DAG
(madhukara phatak via bobby)
+ MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes (Chris Nauroth via harsh)
+
BUG FIXES
MAPREDUCE-4356. [Rumen] Provide access to the method
@@ -126,62 +128,70 @@ Trunk (unreleased changes)
MAPREDUCE-3868. Make Raid Compile. (Weiyan Wang via schen)
-Branch-2 ( Unreleased changes )
+ MAPREDUCE-4685. DBCount should not use ACCESS. (Viji via harsh)
- INCOMPATIBLE CHANGES
+ MAPREDUCE-3223. Remove MR1 configs from mapred-default.xml (tlipcon via harsh)
- NEW FEATURES
+ MAPREDUCE-4678. Running the Pentomino example with defaults throws
+ java.lang.NegativeArraySizeException (Chris McConnell via harsh)
- MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process
- command-line. (ahmed via tucu)
+ MAPREDUCE-4695. Fix LocalRunner on trunk after MAPREDUCE-3223 broke it
+ (harsh)
- MAPREDUCE-4417. add support for encrypted shuffle (tucu)
+ MAPREDUCE-4574. Fix TotalOrderParitioner to work with
+ non-WritableComparable key types. (harsh)
- IMPROVEMENTS
+Release 2.0.3-alpha - Unreleased
- MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved
- (Jason Lowe via bobby)
+ INCOMPATIBLE CHANGES
- MAPREDUCE-4511. Add IFile readahead (ahmed via tucu)
+ MAPREDUCE-4123. Remove the 'mapred groups' command, which is no longer
+ supported. (Devaraj K via sseth)
- BUG FIXES
+ NEW FEATURES
- MAPREDUCE-4422. YARN_APPLICATION_CLASSPATH needs a documented default value in
- YarnConfiguration. (ahmed via tucu)
+ IMPROVEMENTS
- MAPREDUCE-4406. Users should be able to specify the MiniCluster ResourceManager
- and JobHistoryServer ports. (ahmed via tucu)
+ MAPREDUCE-3678. The Map tasks logs should have the value of input
+ split it processed. (harsh)
- MAPREDUCE-4407. Add hadoop-yarn-server-tests-<version>-tests.jar to hadoop dist
- package. (ahmed via tucu)
+ MAPREDUCE-4616. Improve javadoc for MultipleOutputs. (Tony Burton via
+ acmurthy)
- MAPREDUCE-4465. Update description of yarn.nodemanager.address property.
- (bowang via tucu)
+ HADOOP-8911. CRLF characters in source and text files.
+ (Raja Aluri via suresh)
- MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files
- get deleted from tasktracker. (mayank_bansal via tucu)
+ OPTIMIZATIONS
- MAPREDUCE-4498. Remove hsqldb jar from Hadoop runtime classpath. (rkanter via tucu)
+ BUG FIXES
- MAPREDUCE-4494. TestFifoScheduler failing with Metrics source QueueMetrics,q0=default
- already exists!. (ahmed.radwan via tucu)
+ MAPREDUCE-4607. Race condition in ReduceTask completion can result in Task
+ being incorrectly failed. (Bikas Saha via tomwhite)
- MAPREDUCE-4484. Incorrect IS_MINI_YARN_CLUSTER property name in YarnConfiguration.
- (ahmed.radwan via tucu)
+ MAPREDUCE-4646. Fixed MR framework to send diagnostic information correctly
+ to clients in case of failed jobs also. (Jason Lowe via vinodkv)
- MAPREDUCE-4562. Support for "FileSystemCounter" legacy counter group name
- for compatibility reasons is creating incorrect counter name.
- (Jarek Jarcec Cecho via tomwhite)
+ MAPREDUCE-4674. Hadoop examples secondarysort has a typo
+ "secondarysrot" in the usage. (Robert Justice via eli)
- MAPREDUCE-4068. Jars in lib subdirectory of the submittable JAR are not added to the
- classpath (rkanter via tucu)
+ MAPREDUCE-4681. Fix unit tests broken by HDFS-3910. (acmurthy)
+
+ MAPREDUCE-4712. mr-jobhistory-daemon.sh doesn't accept --config
+ (Vinod Kumar Vavilapalli via tgraves)
-Release 2.1.0-alpha - Unreleased
+ MAPREDUCE-4654. TestDistCp is ignored. (Sandy Ryza via tomwhite)
+
+Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
NEW FEATURES
+ MAPREDUCE-987. Exposing MiniDFS and MiniMR clusters as a single process
+ command-line. (ahmed via tucu)
+
+ MAPREDUCE-4417. add support for encrypted shuffle (tucu)
+
MAPREDUCE-4355. Add RunningJob.getJobStatus() (kkambatl via tucu)
MAPREDUCE-3451. Port Fair Scheduler to MR2 (pwendell via tucu)
@@ -191,6 +201,14 @@ Release 2.1.0-alpha - Unreleased
IMPROVEMENTS
+ MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved
+ (Jason Lowe via bobby)
+
+ MAPREDUCE-4511. Add IFile readahead (ahmed via tucu)
+
+ MAPREDUCE-4408. allow jobs to set a JAR that is in the distributed cached
+ (rkanter via tucu)
+
MAPREDUCE-4440. Changed SchedulerApp and SchedulerNode to be a minimal
interface to allow schedulers to maintain their own. (acmurthy)
@@ -203,9 +221,6 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-3921. MR AM should act on node health status changes.
(Bikas Saha via sseth)
- MAPREDUCE-4253. Tests for mapreduce-client-core are lying under
- mapreduce-client-jobclient (Tsuyoshi Ozawa via harsh)
-
MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in
mapred-default.xml (Rui Kubo via harsh)
@@ -226,10 +241,62 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-3289. Make use of fadvise in the NM's shuffle handler.
(Todd Lipcon and Siddharth Seth via sseth)
- OPTIMIZATIONS
+ MAPREDUCE-4580. Change MapReduce to use the yarn-client module.
+ (Vinod Kumar Vavilapalli via sseth)
+
+ MAPREDUCE-4579. Split TestTaskAttempt into two so as to pass tests on
+ jdk7. (Thomas Graves via vinodkv)
+
+ MAPREDUCE-4638. MR AM supplies MapReduce jars in classpath rather than
+ rely on YARN. (acmurthy)
BUG FIXES
+ MAPREDUCE-4422. YARN_APPLICATION_CLASSPATH needs a documented default value in
+ YarnConfiguration. (ahmed via tucu)
+
+ MAPREDUCE-4406. Users should be able to specify the MiniCluster ResourceManager
+ and JobHistoryServer ports. (ahmed via tucu)
+
+ MAPREDUCE-4407. Add hadoop-yarn-server-tests-<version>-tests.jar to hadoop dist
+ package. (ahmed via tucu)
+
+ MAPREDUCE-4465. Update description of yarn.nodemanager.address property.
+ (bowang via tucu)
+
+ MAPREDUCE-4342. Distributed Cache gives inconsistent result if cache files
+ get deleted from tasktracker. (mayank_bansal via tucu)
+
+ MAPREDUCE-4498. Remove hsqldb jar from Hadoop runtime classpath. (rkanter via tucu)
+
+ MAPREDUCE-4494. TestFifoScheduler failing with Metrics source QueueMetrics,q0=default
+ already exists!. (ahmed.radwan via tucu)
+
+ MAPREDUCE-4484. Incorrect IS_MINI_YARN_CLUSTER property name in YarnConfiguration.
+ (ahmed.radwan via tucu)
+
+ MAPREDUCE-4562. Support for "FileSystemCounter" legacy counter group name
+ for compatibility reasons is creating incorrect counter name.
+ (Jarek Jarcec Cecho via tomwhite)
+
+ MAPREDUCE-4068. Jars in lib subdirectory of the submittable JAR are not added to the
+ classpath (rkanter via tucu)
+
+ MAPREDUCE-4577. HDFS-3672 broke
+ TestCombineFileInputFormat.testMissingBlocks() test. (atm)
+
+ MAPREDUCE-4470. Fix TestCombineFileInputFormat.testForEmptyFile (ikatsov via tucu)
+
+ MAPREDUCE-4608. hadoop-mapreduce-client is missing some dependencies.
+ (tucu via tomwhite)
+
+ MAPREDUCE-4610. Support deprecated mapreduce.job.counters.limit property in
+ MR2. (tomwhite)
+
+ MAPREDUCE-4629. Remove JobHistory.DEBUG_MODE (Karthik Kambatla via bobby)
+
+ MAPREDUCE-4642. MiniMRClientClusterFactory should not use job.setJar() (rkanter via tucu)
+
MAPREDUCE-4148. MapReduce should not have a compile-time dependency on
HDFS. (tomwhite)
@@ -303,6 +370,10 @@ Release 2.1.0-alpha - Unreleased
MAPREDUCE-4380. Empty Userlogs directory is getting created under logs
directory (Devaraj K via bobby)
+ MAPREDUCE-4649. Ensure MapReduce JobHistory Daemon doens't assume
+ HADOOP_YARN_HOME and HADOOP_MAPRED_HOME are the same. (vinodkv via
+ acmurthy)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -486,7 +557,55 @@ Release 2.0.0-alpha - 05-23-2012
MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is
bad (Jason Lowe via bobby)
-Release 0.23.3 - UNRELEASED
+Release 0.23.5 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-4554. Job Credentials are not transmitted if security is turned
+ off (Benoy Antony via bobby)
+
+ MAPREDUCE-4705. Fix a bug in job history lookup, which makes older jobs
+ inaccessible despite the presence of a valid history file. (Jason Lowe
+ via sseth)
+
+Release 0.23.4 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ MAPREDUCE-2786. Add compression option for TestDFSIO.
+ (Plamen Jeliazkov via shv)
+
+ MAPREDUCE-4645. Provide a random seed to Slive to make the sequence
+ of file names deterministic. (Ravi Prakash via shv)
+
+ MAPREDUCE-4651. Benchmarking random reads with DFSIO. (shv)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
+ in it. (Robert Evans via tgraves)
+
+ MAPREDUCE-4691. Historyserver can report "Unknown job" after RM says job
+ has completed (Robert Joseph Evans via jlowe)
+
+ MAPREDUCE-4689. JobClient.getMapTaskReports on failed job results in NPE
+ (jlowe via bobby)
+
+Release 0.23.3
INCOMPATIBLE CHANGES
@@ -533,6 +652,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby
via tgraves)
+ MAPREDUCE-4569. Fixed TestHsWebServicesJobsQuery to pass on JDK7 by not
+ depending on test order. (Thomas Graves via vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
@@ -827,6 +949,27 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4570. ProcfsBasedProcessTree#constructProcessInfo() prints a
warning if procfsDir/<pid>/stat is not found. (Ahmed Radwan via bobby)
+ MAPREDUCE-4600. TestTokenCache.java from MRV1 no longer compiles (daryn
+ via bobby)
+
+ MAPREDUCE-4612. job summary file permissions not set when its created
+ (tgraves via bobby)
+
+ MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+
+ MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+ Evans via tgraves)
+
+ MAPREDUCE-4604. In mapred-default, mapreduce.map.maxattempts &
+ mapreduce.reduce.maxattempts defaults are set to 4 as well as
+ mapreduce.job.maxtaskfailures.per.tracker. (Ravi Prakash via jeagles)
+
+ MAPREDUCE-4633. history server doesn't set permissions on all subdirs
+ (tgraves via bobby)
+
+ MAPREDUCE-4641. Exception in commitJob marks job as successful in job
+ history (Jason Lowe via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/INSTALL
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/INSTALL?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/INSTALL (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/INSTALL Tue Oct 16 00:02:55 2012
@@ -19,7 +19,7 @@ mvn clean package -Pdist -Dtar -DskipTes
You can omit -Pnative it you don't want to build native packages.
Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different
-directory, say YARN_HOME.
+directory, say HADOOP_YARN_HOME.
Step 5)
Start hdfs
@@ -32,7 +32,7 @@ You probably want to export these in had
export HADOOP_MAPRED_HOME=<mapred loc>
export HADOOP_COMMON_HOME=<common loc>
export HADOOP_HDFS_HOME=<hdfs loc>
-export YARN_HOME=directory where you untarred yarn
+export HADOOP_YARN_HOME=directory where you untarred yarn
export HADOOP_CONF_DIR=<conf loc>
export YARN_CONF_DIR=$HADOOP_CONF_DIR
@@ -53,7 +53,7 @@ Step 8) Modify mapred-site.xml to use ya
<value>yarn</value>
</property>
-Step 9) cd $YARN_HOME
+Step 9) cd $HADOOP_YARN_HOME
Step 10) sbin/yarn-daemon.sh start resourcemanager
@@ -64,7 +64,7 @@ Step 12) sbin/mr-jobhistory-daemon.sh st
Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME
ant examples -Dresolvers=internal
-$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output
+$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $HADOOP_YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output
The output on the command line should be almost similar to what you see in the JT/TT setup (Hadoop 0.20/0.21)
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred Tue Oct 16 00:02:55 2012
@@ -17,7 +17,7 @@
bin=`which $0`
bin=`dirname ${bin}`
-bin=`cd "$bin"; pwd`
+bin=`cd "$bin" > /dev/null; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
@@ -35,7 +35,6 @@ function print_usage(){
echo " queue get information regarding JobQueues"
echo " classpath prints the class path needed for running"
echo " mapreduce subcommands"
- echo " groups get the groups which users belong to"
echo " historyserver run job history servers as a standalone daemon"
echo " distcp <srcurl> <desturl> copy file or directories recursively"
echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
@@ -63,9 +62,6 @@ elif [ "$COMMAND" = "sampler" ] ; then
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "classpath" ] ; then
echo -n
-elif [ "$COMMAND" = "groups" ] ; then
- CLASS=org.apache.hadoop.mapred.tools.GetGroups
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "historyserver" ] ; then
CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS"
@@ -74,7 +70,8 @@ elif [ "$COMMAND" = "historyserver" ] ;
fi
elif [ "$COMMAND" = "mradmin" ] \
|| [ "$COMMAND" = "jobtracker" ] \
- || [ "$COMMAND" = "tasktracker" ] ; then
+ || [ "$COMMAND" = "tasktracker" ] \
+ || [ "$COMMAND" = "groups" ] ; then
echo "Sorry, the $COMMAND command is no longer supported."
echo "You may find similar functionality with the \"yarn\" shell command."
print_usage
@@ -115,6 +112,11 @@ for f in $HADOOP_MAPRED_HOME/${MAPRED_DI
CLASSPATH=${CLASSPATH}:$f;
done
+# Need YARN jars also
+for f in $HADOOP_YARN_HOME/${YARN_DIR}/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+done
+
# add libs to CLASSPATH
for f in $HADOOP_MAPRED_HOME/${MAPRED_LIB_JARS_DIR}/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred-config.sh?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred-config.sh (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mapred-config.sh Tue Oct 16 00:02:55 2012
@@ -38,3 +38,15 @@ else
echo "Hadoop common not found."
exit
fi
+
+# Only set locally to use in HADOOP_OPTS. No need to export.
+# The following defaults are useful when somebody directly invokes bin/mapred.
+HADOOP_MAPRED_LOG_DIR=${HADOOP_MAPRED_LOG_DIR:-${HADOOP_MAPRED_HOME}/logs}
+HADOOP_MAPRED_LOGFILE=${HADOOP_MAPRED_LOGFILE:-hadoop.log}
+HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_MAPRED_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_MAPRED_LOGFILE"
+export HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_MAPRED_ROOT_LOGGER}"
+
+
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh Tue Oct 16 00:02:55 2012
@@ -16,22 +16,16 @@
# limitations under the License.
-# Runs a yarn command as a daemon.
#
# Environment Variables
#
-# HADOOP_LOGFILE Hadoop log file.
-# HADOOP_ROOT_LOGGER Hadoop root logger.
# HADOOP_JHS_LOGGER Hadoop JobSummary logger.
-# YARN_CONF_DIR Alternate conf dir. Default is ${YARN_HOME}/conf.
-# YARN_LOG_DIR Where log files are stored. PWD by default.
-# YARN_MASTER host:path where hadoop code should be rsync'd from
-# YARN_PID_DIR The pid files are stored. /tmp by default.
-# YARN_IDENT_STRING A string representing this instance of hadoop. $USER by default
-# YARN_NICENESS The scheduling priority for daemons. Defaults to 0.
+# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_MAPRED_HOME}/conf.
+# HADOOP_MAPRED_PID_DIR The pid files are stored. /tmp by default.
+# HADOOP_MAPRED_NICENESS The scheduling priority for daemons. Defaults to 0.
##
-usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <mapred-command> "
+usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] (start|stop) <mapred-command> "
# if no args specified, show usage
if [ $# -le 1 ]; then
@@ -44,7 +38,9 @@ bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+if [ -e ${HADOOP_LIBEXEC_DIR}/mapred-config.sh ]; then
+ . $HADOOP_LIBEXEC_DIR/mapred-config.sh
+fi
# get arguments
startStop=$1
@@ -69,43 +65,43 @@ hadoop_rotate_log ()
fi
}
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
- . "${YARN_CONF_DIR}/yarn-env.sh"
+if [ "$HADOOP_MAPRED_IDENT_STRING" = "" ]; then
+ export HADOOP_MAPRED_IDENT_STRING="$USER"
fi
-if [ "$YARN_IDENT_STRING" = "" ]; then
- export YARN_IDENT_STRING="$USER"
-fi
+export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_PREFIX}}
+export HADOOP_MAPRED_LOGFILE=mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.log
+export HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,RFA}
+export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA}
-# get log directory
-if [ "$YARN_LOG_DIR" = "" ]; then
- export YARN_LOG_DIR="$YARN_HOME/logs"
+if [ -f "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then
+ . "${HADOOP_CONF_DIR}/mapred-env.sh"
fi
-mkdir -p "$YARN_LOG_DIR"
-chown $YARN_IDENT_STRING $YARN_LOG_DIR
-if [ "$YARN_PID_DIR" = "" ]; then
- YARN_PID_DIR=/tmp
+mkdir -p "$HADOOP_MAPRED_LOG_DIR"
+chown $HADOOP_MAPRED_IDENT_STRING $HADOOP_MAPRED_LOG_DIR
+
+if [ "$HADOOP_MAPRED_PID_DIR" = "" ]; then
+ HADOOP_MAPRED_PID_DIR=/tmp
fi
-# some variables
-export HADOOP_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log
-export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,RFA}
-export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA}
-log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out
-pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid
-YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5}
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_MAPRED_IDENT_STRING"
+
+log=$HADOOP_MAPRED_LOG_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.out
+pid=$HADOOP_MAPRED_PID_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command.pid
+
+HADOOP_MAPRED_STOP_TIMEOUT=${HADOOP_MAPRED_STOP_TIMEOUT:-5}
# Set default scheduling priority
-if [ "$YARN_NICENESS" = "" ]; then
- export YARN_NICENESS=0
+if [ "$HADOOP_MAPRED_NICENESS" = "" ]; then
+ export HADOOP_MAPRED_NICENESS=0
fi
case $startStop in
(start)
- mkdir -p "$YARN_PID_DIR"
+ mkdir -p "$HADOOP_MAPRED_PID_DIR"
if [ -f $pid ]; then
if kill -0 `cat $pid` > /dev/null 2>&1; then
@@ -114,15 +110,10 @@ case $startStop in
fi
fi
- if [ "$YARN_MASTER" != "" ]; then
- echo rsync from $YARN_MASTER
- rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $YARN_MASTER/ "$YARN_HOME"
- fi
-
hadoop_rotate_log $log
echo starting $command, logging to $log
- cd "$YARN_HOME"
- nohup nice -n $YARN_NICENESS "$YARN_HOME"/bin/mapred --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
+ cd "$HADOOP_MAPRED_HOME"
+ nohup nice -n $HADOOP_MAPRED_NICENESS "$HADOOP_MAPRED_HOME"/bin/mapred --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
echo $! > $pid
sleep 1; head "$log"
;;
@@ -134,9 +125,9 @@ case $startStop in
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $command
kill $TARGET_PID
- sleep $YARN_STOP_TIMEOUT
+ sleep $HADOOP_MAPRED_STOP_TIMEOUT
if kill -0 $TARGET_PID > /dev/null 2>&1; then
- echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing with kill -9"
+ echo "$command did not stop gracefully after $HADOOP_MAPRED_STOP_TIMEOUT seconds: killing with kill -9"
kill -9 $TARGET_PID
fi
else
Propchange: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf:r1363593-1396941
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Tue Oct 16 00:02:55 2012
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -55,7 +54,6 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -92,11 +90,19 @@ class YarnChild {
DefaultMetricsSystem.initialize(
StringUtils.camelize(firstTaskid.getTaskType().name()) +"Task");
- Token<JobTokenIdentifier> jt = loadCredentials(defaultConf, address);
+ // Security framework already loaded the tokens into current ugi
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
+ LOG.info("Executing with tokens:");
+ for (Token<?> token: credentials.getAllTokens()) {
+ LOG.info(token);
+ }
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner =
UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
+ Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
+ SecurityUtil.setTokenService(jt, address);
taskOwner.addToken(jt);
final TaskUmbilicalProtocol umbilical =
taskOwner.doAs(new PrivilegedExceptionAction<TaskUmbilicalProtocol>() {
@@ -132,17 +138,14 @@ class YarnChild {
YarnChild.taskid = task.getTaskID();
// Create the job-conf and set credentials
- final JobConf job =
- configureTask(task, defaultConf.getCredentials(), jt);
+ final JobConf job = configureTask(task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
childUGI = UserGroupInformation.createRemoteUser(System
.getenv(ApplicationConstants.Environment.USER.toString()));
// Add tokens to new user so that it may execute its task correctly.
- for(Token<?> token : UserGroupInformation.getCurrentUser().getTokens()) {
- childUGI.addToken(token);
- }
+ childUGI.addCredentials(credentials);
// Create a final reference to the task for the doAs block
final Task taskFinal = task;
@@ -206,30 +209,6 @@ class YarnChild {
}
}
- private static Token<JobTokenIdentifier> loadCredentials(JobConf conf,
- InetSocketAddress address) throws IOException {
- //load token cache storage
- String tokenFileLocation =
- System.getenv(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME);
- String jobTokenFile =
- new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf))
- .toUri().getPath();
- Credentials credentials =
- TokenCache.loadTokens(jobTokenFile, conf);
- LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() +
- "; from file=" + jobTokenFile);
- Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
- SecurityUtil.setTokenService(jt, address);
- UserGroupInformation current = UserGroupInformation.getCurrentUser();
- current.addToken(jt);
- for (Token<? extends TokenIdentifier> tok : credentials.getAllTokens()) {
- current.addToken(tok);
- }
- // Set the credentials
- conf.setCredentials(credentials);
- return jt;
- }
-
/**
* Configure mapred-local dirs. This config is used by the task for finding
* out an output directory.
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Tue Oct 16 00:02:55 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final Map<JobId, MetaInfo> fileMap =
Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super("JobHistoryEventHandler");
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
Iterator<JobId> jobIt = fileMap.keySet().iterator();
- if(isSignalled) {
+ if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -661,6 +661,8 @@ public class JobHistoryEventHandler exte
summaryFileOut = doneDirFS.create(qualifiedSummaryDoneFile, true);
summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
summaryFileOut.close();
+ doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
+ JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
} catch (IOException e) {
LOG.info("Unable to write out JobSummaryInfo to ["
+ qualifiedSummaryDoneFile + "]", e);
@@ -894,7 +896,7 @@ public class JobHistoryEventHandler exte
stagingDirFS.delete(fromPath, false);
}
- }
+ }
boolean pathExists(FileSystem fileSys, Path path) throws IOException {
return fileSys.exists(path);
@@ -909,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
- this.isSignalled = isSignalled;
- LOG.info("JobHistoryEventHandler notified that isSignalled was "
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+ this.forceJobCompletion = forceJobCompletion;
+ LOG.info("JobHistoryEventHandler notified that forceJobCompletion is "
+ + forceJobCompletion);
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Tue Oct 16 00:02:55 2012
@@ -87,8 +87,6 @@ import org.apache.hadoop.mapreduce.v2.ut
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.Clock;
@@ -172,6 +170,8 @@ public class MRAppMaster extends Composi
private Credentials fsTokens = new Credentials(); // Filled during init
private UserGroupInformation currentUser; // Will be setup during init
+ private volatile boolean isLastAMRetry = false;
+
public MRAppMaster(ApplicationAttemptId applicationAttemptId,
ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
long appSubmitTime) {
@@ -197,11 +197,21 @@ public class MRAppMaster extends Composi
@Override
public void init(final Configuration conf) {
-
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
downloadTokensAndSetupUGI(conf);
-
+
+ //TODO this is a hack, we really need the RM to inform us when we
+ // are the last one. This would allow us to configure retries on
+ // a per application basis.
+ int numAMRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES);
+ isLastAMRetry = appAttemptID.getAttemptId() >= numAMRetries;
+ LOG.info("AM Retries: " + numAMRetries +
+ " attempt num: " + appAttemptID.getAttemptId() +
+ " is last retry: " + isLastAMRetry);
+
+
context = new RunningAppContext(conf);
// Job name is the same as the app name util we support DAG of jobs
@@ -419,6 +429,8 @@ public class MRAppMaster extends Composi
}
try {
+ //We are finishing cleanly so this is the last retry
+ isLastAMRetry = true;
// Stop all services
// This will also send the final report to the ResourceManager
LOG.info("Calling stop for all the services");
@@ -478,27 +490,17 @@ public class MRAppMaster extends Composi
try {
this.currentUser = UserGroupInformation.getCurrentUser();
- if (UserGroupInformation.isSecurityEnabled()) {
- // Read the file-system tokens from the localized tokens-file.
- Path jobSubmitDir =
- FileContext.getLocalFSFileContext().makeQualified(
- new Path(new File(MRJobConfig.JOB_SUBMIT_DIR)
- .getAbsolutePath()));
- Path jobTokenFile =
- new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
- fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
- LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
- + jobTokenFile);
-
- for (Token<? extends TokenIdentifier> tk : fsTokens.getAllTokens()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Token of kind " + tk.getKind()
- + "in current ugi in the AppMaster for service "
- + tk.getService());
- }
- currentUser.addToken(tk); // For use by AppMaster itself.
- }
- }
+ // Read the file-system tokens from the localized tokens-file.
+ Path jobSubmitDir =
+ FileContext.getLocalFSFileContext().makeQualified(
+ new Path(new File(MRJobConfig.JOB_SUBMIT_DIR)
+ .getAbsolutePath()));
+ Path jobTokenFile =
+ new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
+ fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
+ LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
+ + jobTokenFile);
+ currentUser.addCredentials(fsTokens); // For use by AppMaster itself.
} catch (IOException e) {
throw new YarnException(e);
}
@@ -676,7 +678,11 @@ public class MRAppMaster extends Composi
}
public void setSignalled(boolean isSignalled) {
- ((RMCommunicator) containerAllocator).setSignalled(true);
+ ((RMCommunicator) containerAllocator).setSignalled(isSignalled);
+ }
+
+ public void setShouldUnregister(boolean shouldUnregister) {
+ ((RMCommunicator) containerAllocator).setShouldUnregister(shouldUnregister);
}
}
@@ -727,7 +733,12 @@ public class MRAppMaster extends Composi
@Override
public synchronized void stop() {
try {
- cleanupStagingDir();
+ if(isLastAMRetry) {
+ cleanupStagingDir();
+ } else {
+ LOG.info("Skipping cleaning up the staging dir. "
+ + "assuming AM will be retried.");
+ }
} catch (IOException io) {
LOG.error("Failed to cleanup staging dir: ", io);
}
@@ -1026,14 +1037,19 @@ public class MRAppMaster extends Composi
public void run() {
LOG.info("MRAppMaster received a signal. Signaling RMCommunicator and "
+ "JobHistoryEventHandler.");
+
// Notify the JHEH and RMCommunicator that a SIGTERM has been received so
// that they don't take too long in shutting down
if(appMaster.containerAllocator instanceof ContainerAllocatorRouter) {
((ContainerAllocatorRouter) appMaster.containerAllocator)
.setSignalled(true);
+ ((ContainerAllocatorRouter) appMaster.containerAllocator)
+ .setShouldUnregister(appMaster.isLastAMRetry);
}
+
if(appMaster.jobHistoryEventHandler != null) {
- appMaster.jobHistoryEventHandler.setSignalled(true);
+ appMaster.jobHistoryEventHandler
+ .setForcejobCompletion(appMaster.isLastAMRetry);
}
appMaster.stop();
}
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Tue Oct 16 00:02:55 2012
@@ -82,8 +82,7 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -115,16 +114,15 @@ public class MRClientService extends Abs
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress address = new InetSocketAddress(0);
- ClientToAMSecretManager secretManager = null;
+ ClientToAMTokenSecretManager secretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
- secretManager = new ClientToAMSecretManager();
String secretKeyStr =
System
.getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
byte[] bytes = Base64.decodeBase64(secretKeyStr);
- ClientTokenIdentifier identifier = new ClientTokenIdentifier(
- this.appContext.getApplicationID());
- secretManager.setMasterKey(identifier, bytes);
+ secretManager =
+ new ClientToAMTokenSecretManager(this.appContext.getApplicationID(),
+ bytes);
}
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Oct 16 00:02:55 2012
@@ -582,17 +582,23 @@ public class JobImpl implements org.apac
String jobFile =
remoteJobConfFile == null ? "" : remoteJobConfFile.toString();
+ StringBuilder diagsb = new StringBuilder();
+ for (String s : getDiagnostics()) {
+ diagsb.append(s).append("\n");
+ }
+
if (getState() == JobState.NEW) {
return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
- cleanupProgress, jobFile, amInfos, isUber);
+ cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
}
computeProgress();
- return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
- appSubmitTime, startTime, finishTime, setupProgress,
+ JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username,
+ state, appSubmitTime, startTime, finishTime, setupProgress,
this.mapProgress, this.reduceProgress,
- cleanupProgress, jobFile, amInfos, isUber);
+ cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
+ return report;
} finally {
readLock.unlock();
}
@@ -759,7 +765,8 @@ public class JobImpl implements org.apac
job.getCommitter().commitJob(job.getJobContext());
} catch (IOException e) {
LOG.error("Could not do commit for Job", e);
- job.logJobHistoryFinishedEvent();
+ job.addDiagnostic("Job commit failed: " + e.getMessage());
+ job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
return job.finished(JobState.FAILED);
}
job.logJobHistoryFinishedEvent();
@@ -1199,7 +1206,7 @@ public class JobImpl implements org.apac
}
}
- private void abortJob(
+ protected void abortJob(
org.apache.hadoop.mapreduce.JobStatus.State finalState) {
try {
committer.abortJob(jobContext, finalState);
@@ -1501,7 +1508,7 @@ public class JobImpl implements org.apac
}
}
- private void addDiagnostic(String diag) {
+ protected void addDiagnostic(String diag) {
diagnostics.add(diag);
}
Modified: hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Oct 16 00:02:55 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@@ -71,6 +72,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
@@ -86,6 +88,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
@@ -120,6 +123,7 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
@@ -128,6 +132,8 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.RackResolver;
+import com.google.common.base.Preconditions;
+
/**
* Implementation of TaskAttempt interface.
*/
@@ -404,10 +410,10 @@ public abstract class TaskAttemptImpl im
TaskAttemptState.FAILED,
TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
new TooManyFetchFailureTransition())
- .addTransition(
- TaskAttemptState.SUCCEEDED, TaskAttemptState.KILLED,
- TaskAttemptEventType.TA_KILL,
- new KilledAfterSuccessTransition())
+ .addTransition(TaskAttemptState.SUCCEEDED,
+ EnumSet.of(TaskAttemptState.SUCCEEDED, TaskAttemptState.KILLED),
+ TaskAttemptEventType.TA_KILL,
+ new KilledAfterSuccessTransition())
.addTransition(
TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
@@ -605,10 +611,12 @@ public abstract class TaskAttemptImpl im
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
- localResources.put(
- MRJobConfig.JOB_JAR,
- createLocalResource(remoteFS, remoteJobJar,
- LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION));
+ LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+ LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+ String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+ JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+ rc.setPattern(pattern);
+ localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info("The job-jar file on the remote FS is "
+ remoteJobJar.toUri().toASCIIString());
} else {
@@ -639,14 +647,10 @@ public abstract class TaskAttemptImpl im
MRApps.setupDistributedCache(conf, localResources);
// Setup up task credentials buffer
- Credentials taskCredentials = new Credentials();
-
- if (UserGroupInformation.isSecurityEnabled()) {
- LOG.info("Adding #" + credentials.numberOfTokens()
- + " tokens and #" + credentials.numberOfSecretKeys()
- + " secret keys for NM use for launching container");
- taskCredentials.addAll(credentials);
- }
+ LOG.info("Adding #" + credentials.numberOfTokens()
+ + " tokens and #" + credentials.numberOfSecretKeys()
+ + " secret keys for NM use for launching container");
+ Credentials taskCredentials = new Credentials(credentials);
// LocalStorageToken is needed irrespective of whether security is enabled
// or not.
@@ -1483,6 +1487,9 @@ public abstract class TaskAttemptImpl im
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+ // too many fetch failure can only happen for map tasks
+ Preconditions
+ .checkArgument(taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP);
//add to diagnostic
taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
//set the finish time
@@ -1506,15 +1513,30 @@ public abstract class TaskAttemptImpl im
}
private static class KilledAfterSuccessTransition implements
- SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+ MultipleArcTransition<TaskAttemptImpl, TaskAttemptEvent, TaskAttemptState> {
@SuppressWarnings("unchecked")
@Override
- public void transition(TaskAttemptImpl taskAttempt,
+ public TaskAttemptState transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
- TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event;
- //add to diagnostic
- taskAttempt.addDiagnosticInfo(msgEvent.getMessage());
+ if(taskAttempt.getID().getTaskId().getTaskType() == TaskType.REDUCE) {
+ // after a reduce task has succeeded, its outputs are in safe in HDFS.
+ // logically such a task should not be killed. we only come here when
+ // there is a race condition in the event queue. E.g. some logic sends
+ // a kill request to this attempt when the successful completion event
+ // for this task is already in the event queue. so the kill event will
+ // get executed immediately after the attempt is marked successful and
+ // result in this transition being exercised.
+ // ignore this for reduce tasks
+ LOG.info("Ignoring killed event for successful reduce task attempt" +
+ taskAttempt.getID().toString());
+ return TaskAttemptState.SUCCEEDED;
+ }
+ if(event instanceof TaskAttemptKillEvent) {
+ TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event;
+ //add to diagnostic
+ taskAttempt.addDiagnosticInfo(msgEvent.getMessage());
+ }
// not setting a finish time since it was set on success
assert (taskAttempt.getFinishTime() != 0);
@@ -1528,6 +1550,7 @@ public abstract class TaskAttemptImpl im
.getTaskId().getJobId(), tauce));
taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED));
+ return TaskAttemptState.KILLED;
}
}