You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/12/05 19:00:50 UTC
svn commit: r1210547 [1/2] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/
hadoop-mapreduce-client/hadoop-mapreduce-clien...
Author: atm
Date: Mon Dec 5 18:00:44 2011
New Revision: 1210547
URL: http://svn.apache.org/viewvc?rev=1210547&view=rev
Log:
Merge trunk into HA branch.
Added:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java
- copied unchanged from r1210544, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileOutputFormat.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFileOutputFormat.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCounters.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLocalMRNotification.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLocalMRNotification.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskCommit.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/WordCount.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/WordCount.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/JobControlTestUtils.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestChainMapReduce.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultithreadedMapRunner.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestMultithreadedMapRunner.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestChild.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestChild.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestMapReduceChain.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestMapReduceChain.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestSingleElementChain.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/chain/TestSingleElementChain.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/map/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/map/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/map/TestMultithreadedMapper.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/map/TestMultithreadedMapper.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/
- copied from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
- copied unchanged from r1209643, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/
- copied from r1210544, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml
- copied unchanged from r1210544, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
- copied unchanged from r1210544, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
- copied unchanged from r1210544, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
Removed:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/conf/TestNoDefaultsJobConf.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestFileOutputFormat.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobCounters.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLazyOutput.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestLocalMRNotification.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/lib/TestChainMapReduce.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/lib/TestMultipleOutputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/lib/TestMultithreadedMapRunner.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestChild.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestMapReduceChain.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestSingleElementChain.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/input/TestMultipleInputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/map/TestMultithreadedMapper.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestJobOutputCommitter.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/output/TestMRMultipleOutputs.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java
Modified:
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/FileBench.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/io/TestSequenceFileMergeProgress.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/security/authorize/TestServiceLevelAuthorization.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/test/MapredTestDriver.java (props changed)
hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 5 18:00:44 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1209165
+/hadoop/common/trunk/hadoop-mapreduce-project:1152502-1210544
/hadoop/core/branches/branch-0.19/mapred:713112
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 5 18:00:44 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1209165
+/hadoop/common/trunk/hadoop-mapreduce-project/.gitignore:1161333-1210544
/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
/hadoop/core/trunk/.gitignore:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt Mon Dec 5 18:00:44 2011
@@ -35,11 +35,15 @@ Trunk (unreleased changes)
uri with no authority. (John George via jitendra)
MAPREDUCE-3169. Create a new MiniMRCluster equivalent which only provides
- client APIs cross MR1 and MR2. (Ahmed via tucu)
+ client APIs cross MR1 and MR2 (Ahmed via tucu)
MAPREDUCE-3415. improve MiniMRYarnCluster & DistributedShell JAR resolution.
(tucu)
+ HADOOP-7862 MR changes to work with HADOOP 7862:
+ Move the support for multiple protocols to lower layer so that Writable,
+ PB and Avro can all use it (Sanjay)
+
BUG FIXES
MAPREDUCE-3412. Fix 'ant docs'. (amarrk)
@@ -68,6 +72,8 @@ Trunk (unreleased changes)
MAPREDUCE-3477. Hadoop site documentation cannot be built anymore. (jeagles via tucu)
+ MAPREDUCE-3500. MRJobConfig creates an LD_LIBRARY_PATH using the platform ARCH. (tucu)
+
Release 0.23.1 - Unreleased
INCOMPATIBLE CHANGES
@@ -129,6 +135,12 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3448. TestCombineOutputCollector javac unchecked warning on mocked
generics (Jonathan Eagles via mahadev)
+ MAPREDUCE-3169 amendment. Deprecate MiniMRCluster. (Ahmed Radwan via
+ sseth)
+
+ MAPREDUCE-3369. Migrate MR1 tests to run on MR2 using the new interfaces
+ introduced in MAPREDUCE-3169. (Ahmed Radwan via tomwhite)
+
OPTIMIZATIONS
BUG FIXES
@@ -211,6 +223,26 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3452. fifoscheduler web ui page always shows 0% used for the queue.
(Jonathan Eagles via mahadev)
+ MAPREDUCE-3443. JobClient and Job should function in the context of the
+ UGI which created them. (Mahadev Konar via sseth)
+
+ MAPREDUCE-3460. MR AM can hang if containers are allocated on a node
+ blacklisted by the AM. (Hitesh Shah and Robert Joseph Evans via sseth)
+
+ MAPREDUCE-3453. RM web ui application details page shows RM cluster about
+ information. (Jonathan Eagles via sseth)
+
+ MAPREDUCE-3479. JobClient#getJob cannot find local jobs. (tomwhite)
+
+ MAPREDUCE-3456. $HADOOP_PREFIX/bin/yarn should set defaults for
+ $HADOOP_*_HOME (Eric Payne via mahadev)
+
+ MAPREDUCE-3458. Fix findbugs warnings in hadoop-examples. (Devaraj K
+ via mahadev)
+
+ MAPREDUCE-3485. DISKS_FAILED -101 error code should be defined in same location as
+ ABORTED_CONTAINER_EXIT_STATUS. (Ravi Gummadi via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 5 18:00:44 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1209165
+/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:1161333-1210544
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 5 18:00:44 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1209165
+/hadoop/common/trunk/hadoop-mapreduce-project/conf:1152502-1210544
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Mon Dec 5 18:00:44 2011
@@ -87,7 +87,7 @@ public class RMContainerAllocator extend
}
/*
- Vocabulory Used:
+ Vocabulary Used:
pending -> requests which are NOT yet sent to RM
scheduled -> requests which are sent to RM but not yet assigned
assigned -> requests which are assigned to a container
@@ -565,6 +565,7 @@ public class RMContainerAllocator extend
if (event.getEarlierAttemptFailed()) {
earlierFailedMaps.add(event.getAttemptID());
request = new ContainerRequest(event, PRIORITY_FAST_FAIL_MAP);
+ LOG.info("Added "+event.getAttemptID()+" to list of failed maps");
} else {
for (String host : event.getHosts()) {
LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
@@ -603,7 +604,9 @@ public class RMContainerAllocator extend
containersAllocated += allocatedContainers.size();
while (it.hasNext()) {
Container allocated = it.next();
- LOG.info("Assigning container " + allocated);
+ LOG.info("Assigning container " + allocated.getId() +
+ " with priority " + allocated.getPriority() +
+ " to NM " + allocated.getNodeId());
// check if allocated container meets memory requirements
// and whether we have any scheduled tasks that need
@@ -645,7 +648,8 @@ public class RMContainerAllocator extend
// we need to request for a new container
// and release the current one
LOG.info("Got allocated container on a blacklisted "
- + " host. Releasing container " + allocated);
+ + " host "+allocated.getNodeId().getHost()
+ +". Releasing container " + allocated);
// find the request matching this allocated container
// and replace it with a new one
@@ -727,10 +731,20 @@ public class RMContainerAllocator extend
}
private ContainerRequest getContainerReqToReplace(Container allocated) {
+ LOG.info("Finding containerReq for allocated container: " + allocated);
Priority priority = allocated.getPriority();
ContainerRequest toBeReplaced = null;
- if (PRIORITY_FAST_FAIL_MAP.equals(priority)
- || PRIORITY_MAP.equals(priority)) {
+ if (PRIORITY_FAST_FAIL_MAP.equals(priority)) {
+ LOG.info("Replacing FAST_FAIL_MAP container " + allocated.getId());
+ Iterator<TaskAttemptId> iter = earlierFailedMaps.iterator();
+ while (toBeReplaced == null && iter.hasNext()) {
+ toBeReplaced = maps.get(iter.next());
+ }
+ LOG.info("Found replacement: " + toBeReplaced);
+ return toBeReplaced;
+ }
+ else if (PRIORITY_MAP.equals(priority)) {
+ LOG.info("Replacing MAP container " + allocated.getId());
// allocated container was for a map
String host = allocated.getNodeId().getHost();
LinkedList<TaskAttemptId> list = mapsHostMapping.get(host);
@@ -749,6 +763,7 @@ public class RMContainerAllocator extend
TaskAttemptId tId = reduces.keySet().iterator().next();
toBeReplaced = reduces.remove(tId);
}
+ LOG.info("Found replacement: " + toBeReplaced);
return toBeReplaced;
}
@@ -758,7 +773,7 @@ public class RMContainerAllocator extend
//try to assign to earlierFailedMaps if present
ContainerRequest assigned = null;
while (assigned == null && earlierFailedMaps.size() > 0) {
- TaskAttemptId tId = earlierFailedMaps.removeFirst();
+ TaskAttemptId tId = earlierFailedMaps.removeFirst();
if (maps.containsKey(tId)) {
assigned = maps.remove(tId);
JobCounterUpdateEvent jce =
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java Mon Dec 5 18:00:44 2011
@@ -105,6 +105,13 @@ public abstract class RMContainerRequest
this.priority = priority;
}
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("AttemptId[").append(attemptID).append("]");
+ sb.append("Capability[").append(capability).append("]");
+ sb.append("Priority[").append(priority).append("]");
+ return sb.toString();
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java Mon Dec 5 18:00:44 2011
@@ -580,6 +580,135 @@ public class TestRMContainerAllocator {
}
}
+ @Test
+ public void testBlackListedNodesWithSchedulingToThatNode() throws Exception {
+ LOG.info("Running testBlackListedNodesWithSchedulingToThatNode");
+
+ Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, true);
+ conf.setInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 1);
+
+ MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // Submit the application
+ RMApp app = rm.submitApp(1024);
+ dispatcher.await();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+ dispatcher.await();
+
+ ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ dispatcher.await();
+
+ JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+ Job mockJob = mock(Job.class);
+ when(mockJob.getReport()).thenReturn(
+ MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
+ 0, 0, 0, 0, 0, 0, "jobfile", null));
+ MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
+ appAttemptId, mockJob);
+
+ // add resources to scheduler
+ MockNM nodeManager1 = rm.registerNode("h1:1234", 10240);
+ MockNM nodeManager3 = rm.registerNode("h3:1234", 10240);
+ dispatcher.await();
+
+ LOG.info("Requesting 1 Containers _1 on H1");
+ // create the container request
+ ContainerRequestEvent event1 = createReq(jobId, 1, 1024,
+ new String[] { "h1" });
+ allocator.sendRequest(event1);
+
+ LOG.info("RM Heartbeat (to send the container requests)");
+ // this tells the scheduler about the requests
+ // as nodes are not added, no allocations
+ List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
+ dispatcher.await();
+ Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
+
+ LOG.info("h1 Heartbeat (To actually schedule the containers)");
+ // update resources in scheduler
+ nodeManager1.nodeHeartbeat(true); // Node heartbeat
+ dispatcher.await();
+
+ LOG.info("RM Heartbeat (To process the scheduled containers)");
+ assigned = allocator.schedule();
+ dispatcher.await();
+ Assert.assertEquals("No of assignments must be 1", 1, assigned.size());
+
+ LOG.info("Failing container _1 on H1 (should blacklist the node)");
+ // Send events to blacklist nodes h1 and h2
+ ContainerFailedEvent f1 = createFailEvent(jobId, 1, "h1", false);
+ allocator.sendFailure(f1);
+
+ //At this stage, a request should be created for a fast fail map
+ //Create a FAST_FAIL request for a previously failed map.
+ ContainerRequestEvent event1f = createReq(jobId, 1, 1024,
+ new String[] { "h1" }, true, false);
+ allocator.sendRequest(event1f);
+
+ //Update the Scheduler with the new requests.
+ assigned = allocator.schedule();
+ dispatcher.await();
+ Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
+
+ // send another request with different resource and priority
+ ContainerRequestEvent event3 = createReq(jobId, 3, 1024,
+ new String[] { "h1", "h3" });
+ allocator.sendRequest(event3);
+
+ //Allocator is aware of prio:5 container, and prio:20 (h1+h3) container.
+ //RM is only aware of the prio:5 container
+
+ LOG.info("h1 Heartbeat (To actually schedule the containers)");
+ // update resources in scheduler
+ nodeManager1.nodeHeartbeat(true); // Node heartbeat
+ dispatcher.await();
+
+ LOG.info("RM Heartbeat (To process the scheduled containers)");
+ assigned = allocator.schedule();
+ dispatcher.await();
+ Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
+
+ //RMContainerAllocator gets assigned a p:5 on a blacklisted node.
+
+ //Send a release for the p:5 container + another request.
+ LOG.info("RM Heartbeat (To process the re-scheduled containers)");
+ assigned = allocator.schedule();
+ dispatcher.await();
+ Assert.assertEquals("No of assignments must be 0", 0, assigned.size());
+
+ //Hearbeat from H3 to schedule on this host.
+ LOG.info("h3 Heartbeat (To re-schedule the containers)");
+ nodeManager3.nodeHeartbeat(true); // Node heartbeat
+ dispatcher.await();
+
+ LOG.info("RM Heartbeat (To process the re-scheduled containers for H3)");
+ assigned = allocator.schedule();
+ dispatcher.await();
+
+ // For debugging
+ for (TaskAttemptContainerAssignedEvent assig : assigned) {
+ LOG.info(assig.getTaskAttemptID() +
+ " assgined to " + assig.getContainer().getId() +
+ " with priority " + assig.getContainer().getPriority());
+ }
+
+ Assert.assertEquals("No of assignments must be 2", 2, assigned.size());
+
+ // validate that all containers are assigned to h3
+ for (TaskAttemptContainerAssignedEvent assig : assigned) {
+ Assert.assertEquals("Assigned container " + assig.getContainer().getId()
+ + " host not correct", "h3", assig.getContainer().getNodeId().getHost());
+ }
+ }
+
private static class MyFifoScheduler extends FifoScheduler {
public MyFifoScheduler(RMContext rmContext) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Mon Dec 5 18:00:44 2011
@@ -21,6 +21,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URL;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,6 +43,7 @@ import org.apache.hadoop.mapreduce.util.
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
@@ -421,6 +423,11 @@ public class JobClient extends CLI {
}
Cluster cluster;
+ /**
+ * Ugi of the client. We store this ugi when the client is created and
+ * then make sure that the same ugi is used to run the various protocols.
+ */
+ UserGroupInformation clientUgi;
/**
* Create a job client.
@@ -458,6 +465,7 @@ public class JobClient extends CLI {
public void init(JobConf conf) throws IOException {
setConf(conf);
cluster = new Cluster(conf);
+ clientUgi = UserGroupInformation.getCurrentUser();
}
@InterfaceAudience.Private
@@ -487,8 +495,7 @@ public class JobClient extends CLI {
@Override
public boolean isManaged(Token<?> token) throws IOException {
return true;
- }
-
+ }
}
/**
@@ -500,6 +507,7 @@ public class JobClient extends CLI {
public JobClient(InetSocketAddress jobTrackAddr,
Configuration conf) throws IOException {
cluster = new Cluster(jobTrackAddr, conf);
+ clientUgi = UserGroupInformation.getCurrentUser();
}
/**
@@ -562,21 +570,38 @@ public class JobClient extends CLI {
* @throws FileNotFoundException
* @throws IOException
*/
- public RunningJob submitJob(JobConf conf) throws FileNotFoundException,
+ public RunningJob submitJob(final JobConf conf) throws FileNotFoundException,
IOException {
try {
conf.setBooleanIfUnset("mapred.mapper.new-api", false);
conf.setBooleanIfUnset("mapred.reducer.new-api", false);
- Job job = Job.getInstance(conf);
- job.submit();
+ Job job = clientUgi.doAs(new PrivilegedExceptionAction<Job> () {
+ @Override
+ public Job run() throws IOException, ClassNotFoundException,
+ InterruptedException {
+ Job job = Job.getInstance(conf);
+ job.submit();
+ return job;
+ }
+ });
+ // update our Cluster instance with the one created by Job for submission
+ // (we can't pass our Cluster instance to Job, since Job wraps the config
+ // instance, and the two configs would then diverge)
+ cluster = job.getCluster();
return new NetworkedJob(job);
} catch (InterruptedException ie) {
throw new IOException("interrupted", ie);
- } catch (ClassNotFoundException cnfe) {
- throw new IOException("class not found", cnfe);
}
}
+ private Job getJobUsingCluster(final JobID jobid) throws IOException,
+ InterruptedException {
+ return clientUgi.doAs(new PrivilegedExceptionAction<Job>() {
+ public Job run() throws IOException, InterruptedException {
+ return cluster.getJob(jobid);
+ }
+ });
+ }
/**
* Get an {@link RunningJob} object to track an ongoing job. Returns
* null if the id does not correspond to any known job.
@@ -586,9 +611,10 @@ public class JobClient extends CLI {
* <code>jobid</code> doesn't correspond to any known job.
* @throws IOException
*/
- public RunningJob getJob(JobID jobid) throws IOException {
+ public RunningJob getJob(final JobID jobid) throws IOException {
try {
- Job job = cluster.getJob(jobid);
+
+ Job job = getJobUsingCluster(jobid);
if (job != null) {
JobStatus status = JobStatus.downgrade(job.getStatus());
if (status != null) {
@@ -621,9 +647,10 @@ public class JobClient extends CLI {
return getTaskReports(jobId, TaskType.MAP);
}
- private TaskReport[] getTaskReports(JobID jobId, TaskType type) throws IOException {
+ private TaskReport[] getTaskReports(final JobID jobId, TaskType type) throws
+ IOException {
try {
- Job j = cluster.getJob(jobId);
+ Job j = getJobUsingCluster(jobId);
if(j == null) {
return EMPTY_TASK_REPORTS;
}
@@ -688,10 +715,11 @@ public class JobClient extends CLI {
* @param state the state of the task
* (pending/running/completed/failed/killed)
*/
- public void displayTasks(JobID jobId, String type, String state)
+ public void displayTasks(final JobID jobId, String type, String state)
throws IOException {
try {
- super.displayTasks(cluster.getJob(jobId), type, state);
+ Job job = getJobUsingCluster(jobId);
+ super.displayTasks(job, type, state);
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -706,15 +734,20 @@ public class JobClient extends CLI {
*/
public ClusterStatus getClusterStatus() throws IOException {
try {
- ClusterMetrics metrics = cluster.getClusterStatus();
- return new ClusterStatus(metrics.getTaskTrackerCount(),
- metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
- metrics.getOccupiedMapSlots(),
- metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
- metrics.getReduceSlotCapacity(),
- cluster.getJobTrackerStatus(),
- metrics.getDecommissionedTaskTrackerCount());
- } catch (InterruptedException ie) {
+ return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
+ public ClusterStatus run() throws IOException, InterruptedException {
+ ClusterMetrics metrics = cluster.getClusterStatus();
+ return new ClusterStatus(metrics.getTaskTrackerCount(),
+ metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
+ metrics.getOccupiedMapSlots(),
+ metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
+ metrics.getReduceSlotCapacity(),
+ cluster.getJobTrackerStatus(),
+ metrics.getDecommissionedTaskTrackerCount());
+ }
+ });
+ }
+ catch (InterruptedException ie) {
throw new IOException(ie);
}
}
@@ -750,13 +783,17 @@ public class JobClient extends CLI {
*/
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
try {
- ClusterMetrics metrics = cluster.getClusterStatus();
- return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
- arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
- cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
- metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
- metrics.getReduceSlotCapacity(),
- cluster.getJobTrackerStatus());
+ return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
+ public ClusterStatus run() throws IOException, InterruptedException {
+ ClusterMetrics metrics = cluster.getClusterStatus();
+ return new ClusterStatus(arrayToStringList(cluster.getActiveTaskTrackers()),
+ arrayToBlackListInfo(cluster.getBlackListedTaskTrackers()),
+ cluster.getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
+ metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
+ metrics.getReduceSlotCapacity(),
+ cluster.getJobTrackerStatus());
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -787,7 +824,14 @@ public class JobClient extends CLI {
*/
public JobStatus[] getAllJobs() throws IOException {
try {
- org.apache.hadoop.mapreduce.JobStatus[] jobs = cluster.getAllJobStatuses();
+ org.apache.hadoop.mapreduce.JobStatus[] jobs =
+ clientUgi.doAs(new PrivilegedExceptionAction<
+ org.apache.hadoop.mapreduce.JobStatus[]> () {
+ public org.apache.hadoop.mapreduce.JobStatus[] run()
+ throws IOException, InterruptedException {
+ return cluster.getAllJobStatuses();
+ }
+ });
JobStatus[] stats = new JobStatus[jobs.length];
for (int i = 0; i < jobs.length; i++) {
stats[i] = JobStatus.downgrade(jobs[i]);
@@ -909,7 +953,12 @@ public class JobClient extends CLI {
*/
public int getDefaultMaps() throws IOException {
try {
- return cluster.getClusterStatus().getMapSlotCapacity();
+ return clientUgi.doAs(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws IOException, InterruptedException {
+ return cluster.getClusterStatus().getMapSlotCapacity();
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -923,7 +972,12 @@ public class JobClient extends CLI {
*/
public int getDefaultReduces() throws IOException {
try {
- return cluster.getClusterStatus().getReduceSlotCapacity();
+ return clientUgi.doAs(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws IOException, InterruptedException {
+ return cluster.getClusterStatus().getReduceSlotCapacity();
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -936,8 +990,13 @@ public class JobClient extends CLI {
*/
public Path getSystemDir() {
try {
- return cluster.getSystemDir();
- } catch (IOException ioe) {
+ return clientUgi.doAs(new PrivilegedExceptionAction<Path>() {
+ @Override
+ public Path run() throws IOException, InterruptedException {
+ return cluster.getSystemDir();
+ }
+ });
+ } catch (IOException ioe) {
return null;
} catch (InterruptedException ie) {
return null;
@@ -962,7 +1021,11 @@ public class JobClient extends CLI {
*/
public JobQueueInfo[] getRootQueues() throws IOException {
try {
- return getJobQueueInfoArray(cluster.getRootQueues());
+ return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() {
+ public JobQueueInfo[] run() throws IOException, InterruptedException {
+ return getJobQueueInfoArray(cluster.getRootQueues());
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -976,9 +1039,13 @@ public class JobClient extends CLI {
* @return the array of immediate children JobQueueInfo objects
* @throws IOException
*/
- public JobQueueInfo[] getChildQueues(String queueName) throws IOException {
+ public JobQueueInfo[] getChildQueues(final String queueName) throws IOException {
try {
- return getJobQueueInfoArray(cluster.getChildQueues(queueName));
+ return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() {
+ public JobQueueInfo[] run() throws IOException, InterruptedException {
+ return getJobQueueInfoArray(cluster.getChildQueues(queueName));
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -993,7 +1060,11 @@ public class JobClient extends CLI {
*/
public JobQueueInfo[] getQueues() throws IOException {
try {
- return getJobQueueInfoArray(cluster.getQueues());
+ return clientUgi.doAs(new PrivilegedExceptionAction<JobQueueInfo[]>() {
+ public JobQueueInfo[] run() throws IOException, InterruptedException {
+ return getJobQueueInfoArray(cluster.getQueues());
+ }
+ });
} catch (InterruptedException ie) {
throw new IOException(ie);
}
@@ -1007,9 +1078,14 @@ public class JobClient extends CLI {
* @throws IOException
*/
- public JobStatus[] getJobsFromQueue(String queueName) throws IOException {
+ public JobStatus[] getJobsFromQueue(final String queueName) throws IOException {
try {
- QueueInfo queue = cluster.getQueue(queueName);
+ QueueInfo queue = clientUgi.doAs(new PrivilegedExceptionAction<QueueInfo>() {
+ @Override
+ public QueueInfo run() throws IOException, InterruptedException {
+ return cluster.getQueue(queueName);
+ }
+ });
if (queue == null) {
return null;
}
@@ -1032,9 +1108,14 @@ public class JobClient extends CLI {
* @return Queue information associated to particular queue.
* @throws IOException
*/
- public JobQueueInfo getQueueInfo(String queueName) throws IOException {
+ public JobQueueInfo getQueueInfo(final String queueName) throws IOException {
try {
- QueueInfo queueInfo = cluster.getQueue(queueName);
+ QueueInfo queueInfo = clientUgi.doAs(new
+ PrivilegedExceptionAction<QueueInfo>() {
+ public QueueInfo run() throws IOException, InterruptedException {
+ return cluster.getQueue(queueName);
+ }
+ });
if (queueInfo != null) {
return new JobQueueInfo(queueInfo);
}
@@ -1052,7 +1133,14 @@ public class JobClient extends CLI {
public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException {
try {
org.apache.hadoop.mapreduce.QueueAclsInfo[] acls =
- cluster.getQueueAclsForCurrentUser();
+ clientUgi.doAs(new
+ PrivilegedExceptionAction
+ <org.apache.hadoop.mapreduce.QueueAclsInfo[]>() {
+ public org.apache.hadoop.mapreduce.QueueAclsInfo[] run()
+ throws IOException, InterruptedException {
+ return cluster.getQueueAclsForCurrentUser();
+ }
+ });
QueueAclsInfo[] ret = new QueueAclsInfo[acls.length];
for (int i = 0 ; i < acls.length; i++ ) {
ret[i] = QueueAclsInfo.downgrade(acls[i]);
@@ -1070,8 +1158,14 @@ public class JobClient extends CLI {
* @throws IOException
*/
public Token<DelegationTokenIdentifier>
- getDelegationToken(Text renewer) throws IOException, InterruptedException {
- return cluster.getDelegationToken(renewer);
+ getDelegationToken(final Text renewer) throws IOException, InterruptedException {
+ return clientUgi.doAs(new
+ PrivilegedExceptionAction<Token<DelegationTokenIdentifier>>() {
+ public Token<DelegationTokenIdentifier> run() throws IOException,
+ InterruptedException {
+ return cluster.getDelegationToken(renewer);
+ }
+ });
}
/**
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java Mon Dec 5 18:00:44 2011
@@ -30,6 +30,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -315,7 +316,12 @@ public class Job extends JobContextImpl
* @throws IOException
*/
synchronized void updateStatus() throws IOException, InterruptedException {
- this.status = cluster.getClient().getJobStatus(status.getJobID());
+ this.status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
+ @Override
+ public JobStatus run() throws IOException, InterruptedException {
+ return cluster.getClient().getJobStatus(status.getJobID());
+ }
+ });
if (this.status == null) {
throw new IOException("Job status not available ");
}
@@ -430,6 +436,11 @@ public class Job extends JobContextImpl
updateStatus();
return status.isRetired();
}
+
+ @Private
+ public Cluster getCluster() {
+ return cluster;
+ }
/** Only for mocks in unit tests. */
@Private
@@ -476,8 +487,16 @@ public class Job extends JobContextImpl
InterruptedException {
int failCount = 1;
TaskCompletionEvent lastEvent = null;
- for (TaskCompletionEvent event : cluster.getClient().getTaskCompletionEvents(
- status.getJobID(), 0, 10)) {
+ TaskCompletionEvent[] events = ugi.doAs(new
+ PrivilegedExceptionAction<TaskCompletionEvent[]>() {
+ @Override
+ public TaskCompletionEvent[] run() throws IOException,
+ InterruptedException {
+ return cluster.getClient().getTaskCompletionEvents(
+ status.getJobID(), 0, 10);
+ }
+ });
+ for (TaskCompletionEvent event : events) {
if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) {
failCount++;
lastEvent = event;
@@ -500,7 +519,12 @@ public class Job extends JobContextImpl
public TaskReport[] getTaskReports(TaskType type)
throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().getTaskReports(getJobID(), type);
+ final TaskType tmpType = type;
+ return ugi.doAs(new PrivilegedExceptionAction<TaskReport[]>() {
+ public TaskReport[] run() throws IOException, InterruptedException {
+ return cluster.getClient().getTaskReports(getJobID(), tmpType);
+ }
+ });
}
/**
@@ -603,7 +627,14 @@ public class Job extends JobContextImpl
org.apache.hadoop.mapred.JobPriority.valueOf(priority.name()));
} else {
ensureState(JobState.RUNNING);
- cluster.getClient().setJobPriority(getJobID(), priority.toString());
+ final JobPriority tmpPriority = priority;
+ ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws IOException, InterruptedException {
+ cluster.getClient().setJobPriority(getJobID(), tmpPriority.toString());
+ return null;
+ }
+ });
}
}
@@ -615,12 +646,17 @@ public class Job extends JobContextImpl
* @return an array of {@link TaskCompletionEvent}s
* @throws IOException
*/
- public TaskCompletionEvent[] getTaskCompletionEvents(int startFrom,
- int numEvents) throws IOException, InterruptedException {
+ public TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom,
+ final int numEvents) throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().getTaskCompletionEvents(getJobID(),
- startFrom, numEvents);
- }
+ return ugi.doAs(new PrivilegedExceptionAction<TaskCompletionEvent[]>() {
+ @Override
+ public TaskCompletionEvent[] run() throws IOException, InterruptedException {
+ return cluster.getClient().getTaskCompletionEvents(getJobID(),
+ startFrom, numEvents);
+ }
+ });
+ }
/**
* Kill indicated task attempt.
@@ -628,10 +664,14 @@ public class Job extends JobContextImpl
* @param taskId the id of the task to be terminated.
* @throws IOException
*/
- public boolean killTask(TaskAttemptID taskId)
+ public boolean killTask(final TaskAttemptID taskId)
throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().killTask(taskId, false);
+ return ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ public Boolean run() throws IOException, InterruptedException {
+ return cluster.getClient().killTask(taskId, false);
+ }
+ });
}
/**
@@ -640,10 +680,15 @@ public class Job extends JobContextImpl
* @param taskId the id of the task to be terminated.
* @throws IOException
*/
- public boolean failTask(TaskAttemptID taskId)
+ public boolean failTask(final TaskAttemptID taskId)
throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().killTask(taskId, true);
+ return ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
+ @Override
+ public Boolean run() throws IOException, InterruptedException {
+ return cluster.getClient().killTask(taskId, true);
+ }
+ });
}
/**
@@ -656,7 +701,12 @@ public class Job extends JobContextImpl
public Counters getCounters()
throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().getJobCounters(getJobID());
+ return ugi.doAs(new PrivilegedExceptionAction<Counters>() {
+ @Override
+ public Counters run() throws IOException, InterruptedException {
+ return cluster.getClient().getJobCounters(getJobID());
+ }
+ });
}
/**
@@ -665,10 +715,15 @@ public class Job extends JobContextImpl
* @return the list of diagnostic messages for the task
* @throws IOException
*/
- public String[] getTaskDiagnostics(TaskAttemptID taskid)
+ public String[] getTaskDiagnostics(final TaskAttemptID taskid)
throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
- return cluster.getClient().getTaskDiagnostics(taskid);
+ return ugi.doAs(new PrivilegedExceptionAction<String[]>() {
+ @Override
+ public String[] run() throws IOException, InterruptedException {
+ return cluster.getClient().getTaskDiagnostics(taskid);
+ }
+ });
}
/**
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Mon Dec 5 18:00:44 2011
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.mapreduce;
-import org.apache.hadoop.util.PlatformName;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -439,7 +438,7 @@ public interface MRJobConfig {
"mapreduce.admin.user.env";
public static final String DEFAULT_MAPRED_ADMIN_USER_ENV =
- "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native/" + PlatformName.getPlatformName();
+ "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native";
public static final String WORKDIR = "work";
Propchange: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Dec 5 18:00:44 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1209165
+/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:1166973-1210544
/hadoop/core/branches/branch-0.19/mapred/src/java/mapred-default.xml:713112
/hadoop/core/trunk/src/mapred/mapred-default.xml:776175-785643
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Mon Dec 5 18:00:44 2011
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.InetSocketAddress;
-import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
@@ -156,6 +156,8 @@ public class ClientServiceDelegate {
application = rm.getApplicationReport(appId);
continue;
}
+ UserGroupInformation newUgi = UserGroupInformation.createRemoteUser(
+ UserGroupInformation.getCurrentUser().getUserName());
serviceAddr = application.getHost() + ":" + application.getRpcPort();
if (UserGroupInformation.isSecurityEnabled()) {
String clientTokenEncoded = application.getClientToken();
@@ -167,11 +169,17 @@ public class ClientServiceDelegate {
.getHost(), application.getRpcPort());
clientToken.setService(new Text(addr.getAddress().getHostAddress()
+ ":" + addr.getPort()));
- UserGroupInformation.getCurrentUser().addToken(clientToken);
+ newUgi.addToken(clientToken);
}
LOG.info("The url to track the job: " + application.getTrackingUrl());
LOG.debug("Connecting to " + serviceAddr);
- realProxy = instantiateAMProxy(serviceAddr);
+ final String tempStr = serviceAddr;
+ realProxy = newUgi.doAs(new PrivilegedExceptionAction<MRClientProtocol>() {
+ @Override
+ public MRClientProtocol run() throws IOException {
+ return instantiateAMProxy(tempStr);
+ }
+ });
return realProxy;
} catch (IOException e) {
//possibly the AM has crashed
@@ -243,17 +251,11 @@ public class ClientServiceDelegate {
MRClientProtocol instantiateAMProxy(final String serviceAddr)
throws IOException {
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
- MRClientProtocol proxy = currentUser
- .doAs(new PrivilegedAction<MRClientProtocol>() {
- @Override
- public MRClientProtocol run() {
- YarnRPC rpc = YarnRPC.create(conf);
- return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
+ YarnRPC rpc = YarnRPC.create(conf);
+ MRClientProtocol proxy =
+ (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
NetUtils.createSocketAddr(serviceAddr), conf);
- }
- });
LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
return proxy;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java Mon Dec 5 18:00:44 2011
@@ -34,7 +34,11 @@ import org.apache.hadoop.security.UserGr
* Due to major differences between MR1 and MR2, a number of methods are either
* unimplemented/unsupported or were re-implemented to provide wrappers around
* MR2 functionality.
+ *
+ * @deprecated Use {@link org.apache.hadoop.mapred.MiniMRClientClusterFactory}
+ * instead
*/
+@Deprecated
public class MiniMRCluster {
private static final Log LOG = LogFactory.getLog(MiniMRCluster.class);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml Mon Dec 5 18:00:44 2011
@@ -27,6 +27,10 @@
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
+ <properties>
+ <mr.examples.basedir>${basedir}</mr.examples.basedir>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -58,6 +62,18 @@
</archive>
</configuration>
</plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${mr.examples.basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
</project>
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Join.java Mon Dec 5 18:00:44 2011
@@ -19,7 +19,9 @@
package org.apache.hadoop.examples;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -29,9 +31,14 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.ClusterStatus;
import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapreduce.*;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.hadoop.mapreduce.lib.join.*;
+import org.apache.hadoop.mapreduce.lib.join.CompositeInputFormat;
+import org.apache.hadoop.mapreduce.lib.join.TupleWritable;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.Tool;
@@ -52,7 +59,7 @@ import org.apache.hadoop.util.ToolRunner
* [<i>in-dir</i>]* <i>in-dir</i> <i>out-dir</i>
*/
public class Join extends Configured implements Tool {
- public static String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
+ public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host";
static int printUsage() {
System.out.println("join [-r <reduces>] " +
"[-inFormat <input format class>] " +
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Pentomino.java Mon Dec 5 18:00:44 2011
@@ -69,7 +69,7 @@ public class Pentomino {
}
public int[] getRotations() {
- return rotations;
+ return rotations.clone();
}
public boolean getFlippable() {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java Mon Dec 5 18:00:44 2011
@@ -70,7 +70,7 @@ public class TeraGen extends Configured
public static enum Counters {CHECKSUM}
- public static String NUM_ROWS = "mapreduce.terasort.num-rows";
+ public static final String NUM_ROWS = "mapreduce.terasort.num-rows";
/**
* An input format that assigns ranges of longs to each mapper.
*/
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraInputFormat.java Mon Dec 5 18:00:44 2011
@@ -156,10 +156,10 @@ public class TeraInputFormat extends Fil
* them and picks N-1 keys to generate N equally sized partitions.
* @param job the job to sample
* @param partFile where to write the output file to
- * @throws IOException if something goes wrong
+ * @throws Throwable if something goes wrong
*/
public static void writePartitionFile(final JobContext job,
- Path partFile) throws IOException, InterruptedException {
+ Path partFile) throws Throwable {
long t1 = System.currentTimeMillis();
Configuration conf = job.getConfiguration();
final TeraInputFormat inFormat = new TeraInputFormat();
@@ -174,11 +174,12 @@ public class TeraInputFormat extends Fil
final long recordsPerSample = sampleSize / samples;
final int sampleStep = splits.size() / samples;
Thread[] samplerReader = new Thread[samples];
+ SamplerThreadGroup threadGroup = new SamplerThreadGroup("Sampler Reader Thread Group");
// take N samples from different parts of the input
for(int i=0; i < samples; ++i) {
final int idx = i;
samplerReader[i] =
- new Thread ("Sampler Reader " + idx) {
+ new Thread (threadGroup,"Sampler Reader " + idx) {
{
setDaemon(true);
}
@@ -201,7 +202,7 @@ public class TeraInputFormat extends Fil
} catch (IOException ie){
System.err.println("Got an exception while reading splits " +
StringUtils.stringifyException(ie));
- System.exit(-1);
+ throw new RuntimeException(ie);
} catch (InterruptedException e) {
}
@@ -215,6 +216,9 @@ public class TeraInputFormat extends Fil
for (int i = 0; i < samples; i++) {
try {
samplerReader[i].join();
+ if(threadGroup.getThrowable() != null){
+ throw threadGroup.getThrowable();
+ }
} catch (InterruptedException e) {
}
}
@@ -225,6 +229,25 @@ public class TeraInputFormat extends Fil
long t3 = System.currentTimeMillis();
System.out.println("Computing parititions took " + (t3 - t2) + "ms");
}
+
+ static class SamplerThreadGroup extends ThreadGroup{
+
+ private Throwable throwable;
+
+ public SamplerThreadGroup(String s) {
+ super(s);
+ }
+
+ @Override
+ public void uncaughtException(Thread thread, Throwable throwable) {
+ this.throwable = throwable;
+ }
+
+ public Throwable getThrowable() {
+ return this.throwable;
+ }
+
+ }
static class TeraRecordReader extends RecordReader<Text,Text> {
private FSDataInputStream in;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Mon Dec 5 18:00:44 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configured
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -300,7 +299,12 @@ public class TeraSort extends Configured
TeraInputFormat.PARTITION_FILENAME);
URI partitionUri = new URI(partitionFile.toString() +
"#" + TeraInputFormat.PARTITION_FILENAME);
- TeraInputFormat.writePartitionFile(job, partitionFile);
+ try {
+ TeraInputFormat.writePartitionFile(job, partitionFile);
+ } catch (Throwable e) {
+ LOG.error(e.getMessage());
+ return -1;
+ }
job.addCacheFile(partitionUri);
job.createSymlink();
long end = System.currentTimeMillis();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn Mon Dec 5 18:00:44 2011
@@ -148,9 +148,13 @@ IFS=
# add hadoop-common libs to CLASSPATH
if [ ! -d "$HADOOP_COMMON_HOME" ]; then
- echo No HADOOP_COMMON_HOME set.
- echo Please specify it either in yarn-env.sh or in the environment.
- exit 1
+ if [ -d "$HADOOP_PREFIX" ]; then
+ export HADOOP_COMMON_HOME=$HADOOP_PREFIX
+ else
+ echo No HADOOP_COMMON_HOME set.
+ echo Please specify it either in yarn-env.sh or in the environment.
+ exit 1
+ fi
fi
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common'/*'
@@ -158,9 +162,13 @@ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HO
# add hadoop-hdfs libs to CLASSPATH
if [ ! -d "$HADOOP_HDFS_HOME" ]; then
- echo No HADOOP_HDFS_HOME set.
- echo Please specify it either in yarn-env.sh or in the environment.
- exit 1
+ if [ -d "$HADOOP_PREFIX" ]; then
+ export HADOOP_HDFS_HOME=$HADOOP_PREFIX
+ else
+ echo No HADOOP_HDFS_HOME set.
+ echo Please specify it either in yarn-env.sh or in the environment.
+ exit 1
+ fi
fi
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*'
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java?rev=1210547&r1=1210546&r2=1210547&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java Mon Dec 5 18:00:44 2011
@@ -73,6 +73,11 @@ public interface ContainerStatus {
* <p>Container killed by the framework, either due to being released by
* the application or being 'lost' due to node failures etc. have a special
* exit code of {@literal -100}.</p>
+ *
+ * <p>When threshold number of the nodemanager-local-directories or
+ * threshold number of the nodemanager-log-directories become bad, then
+ * container is not launched and is exited with exit status of
+ * {@literal -101}.</p>
*
* @return <em>exit status</em> for the container
*/