You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2012/10/11 08:14:48 UTC
svn commit: r1396918 [1/2] - in
/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project: ./ bin/ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-cli...
Author: todd
Date: Thu Oct 11 06:14:26 2012
New Revision: 1396918
URL: http://svn.apache.org/viewvc?rev=1396918&view=rev
Log:
Merge trunk into branch
Added:
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/mapred-env.sh
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/conf/mapred-env.sh
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUtils.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestUtils.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
- copied unchanged from r1396916, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
Removed:
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestAuditLogger.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIFile.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestReduceTask.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsTextInputFormat.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSequenceFileInputFilter.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestUtils.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java
Modified:
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/INSTALL
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred-config.sh
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestTotalOrderPartitioner.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/DBCountPageView.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/c++/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/block_forensics/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build-contrib.xml (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/build.xml (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/data_join/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/eclipse-plugin/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/index/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/contrib/vaidya/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/examples/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/java/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobQueueInformation.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestSetupAndCleanupFailure.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/src/webapps/job/ (props changed)
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt Thu Oct 11 06:14:26 2012
@@ -61,6 +61,8 @@ Trunk (Unreleased)
MAPREDUCE-4371. Check for cyclic dependencies in Jobcontrol job DAG
(madhukara phatak via bobby)
+ MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes (Chris Nauroth via harsh)
+
BUG FIXES
MAPREDUCE-4356. [Rumen] Provide access to the method
@@ -126,14 +128,33 @@ Trunk (Unreleased)
MAPREDUCE-3868. Make Raid Compile. (Weiyan Wang via schen)
+ MAPREDUCE-4685. DBCount should not use ACCESS. (Viji via harsh)
+
+ MAPREDUCE-3223. Remove MR1 configs from mapred-default.xml (tlipcon via harsh)
+
+ MAPREDUCE-4678. Running the Pentomino example with defaults throws
+ java.lang.NegativeArraySizeException (Chris McConnell via harsh)
+
+ MAPREDUCE-4695. Fix LocalRunner on trunk after MAPREDUCE-3223 broke it
+ (harsh)
+
+ MAPREDUCE-4574. Fix TotalOrderParitioner to work with
+ non-WritableComparable key types. (harsh)
+
Release 2.0.3-alpha - Unreleased
INCOMPATIBLE CHANGES
+ MAPREDUCE-4123. Remove the 'mapred groups' command, which is no longer
+ supported. (Devaraj K via sseth)
+
NEW FEATURES
IMPROVEMENTS
+ MAPREDUCE-3678. The Map tasks logs should have the value of input
+ split it processed. (harsh)
+
OPTIMIZATIONS
BUG FIXES
@@ -147,6 +168,13 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4674. Hadoop examples secondarysort has a typo
"secondarysrot" in the usage. (Robert Justice via eli)
+ MAPREDUCE-4681. Fix unit tests broken by HDFS-3910. (acmurthy)
+
+ MAPREDUCE-4712. mr-jobhistory-daemon.sh doesn't accept --config
+ (Vinod Kumar Vavilapalli via tgraves)
+
+ MAPREDUCE-4654. TestDistCp is ignored. (Sandy Ryza via tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -187,9 +215,6 @@ Release 2.0.2-alpha - 2012-09-07
MAPREDUCE-3921. MR AM should act on node health status changes.
(Bikas Saha via sseth)
- MAPREDUCE-4253. Tests for mapreduce-client-core are lying under
- mapreduce-client-jobclient (Tsuyoshi Ozawa via harsh)
-
MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in
mapred-default.xml (Rui Kubo via harsh)
@@ -339,6 +364,10 @@ Release 2.0.2-alpha - 2012-09-07
MAPREDUCE-4380. Empty Userlogs directory is getting created under logs
directory (Devaraj K via bobby)
+ MAPREDUCE-4649. Ensure MapReduce JobHistory Daemon doens't assume
+ HADOOP_YARN_HOME and HADOOP_MAPRED_HOME are the same. (vinodkv via
+ acmurthy)
+
Release 2.0.0-alpha - 05-23-2012
INCOMPATIBLE CHANGES
@@ -522,6 +551,25 @@ Release 2.0.0-alpha - 05-23-2012
MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is
bad (Jason Lowe via bobby)
+Release 0.23.5 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-4554. Job Credentials are not transmitted if security is turned
+ off (Benoy Antony via bobby)
+
+ MAPREDUCE-4705. Fix a bug in job history lookup, which makes older jobs
+ inaccessible despite the presence of a valid history file. (Jason Lowe
+ via sseth)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -542,7 +590,16 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
-Release 0.23.3 - UNRELEASED
+ MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
+ in it. (Robert Evans via tgraves)
+
+ MAPREDUCE-4691. Historyserver can report "Unknown job" after RM says job
+ has completed (Robert Joseph Evans via jlowe)
+
+ MAPREDUCE-4689. JobClient.getMapTaskReports on failed job results in NPE
+ (jlowe via bobby)
+
+Release 0.23.3
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/INSTALL
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/INSTALL?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/INSTALL (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/INSTALL Thu Oct 11 06:14:26 2012
@@ -19,7 +19,7 @@ mvn clean package -Pdist -Dtar -DskipTes
You can omit -Pnative it you don't want to build native packages.
Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different
-directory, say YARN_HOME.
+directory, say HADOOP_YARN_HOME.
Step 5)
Start hdfs
@@ -32,7 +32,7 @@ You probably want to export these in had
export HADOOP_MAPRED_HOME=<mapred loc>
export HADOOP_COMMON_HOME=<common loc>
export HADOOP_HDFS_HOME=<hdfs loc>
-export YARN_HOME=directory where you untarred yarn
+export HADOOP_YARN_HOME=directory where you untarred yarn
export HADOOP_CONF_DIR=<conf loc>
export YARN_CONF_DIR=$HADOOP_CONF_DIR
@@ -53,7 +53,7 @@ Step 8) Modify mapred-site.xml to use ya
<value>yarn</value>
</property>
-Step 9) cd $YARN_HOME
+Step 9) cd $HADOOP_YARN_HOME
Step 10) sbin/yarn-daemon.sh start resourcemanager
@@ -64,7 +64,7 @@ Step 12) sbin/mr-jobhistory-daemon.sh st
Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME
ant examples -Dresolvers=internal
-$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output
+$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-*.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $HADOOP_YARN_HOME/modules/hadoop-mapreduce-client-jobclient-*.jar output
The output on the command line should be almost similar to what you see in the JT/TT setup (Hadoop 0.20/0.21)
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred Thu Oct 11 06:14:26 2012
@@ -17,7 +17,7 @@
bin=`which $0`
bin=`dirname ${bin}`
-bin=`cd "$bin"; pwd`
+bin=`cd "$bin" > /dev/null; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
@@ -35,7 +35,6 @@ function print_usage(){
echo " queue get information regarding JobQueues"
echo " classpath prints the class path needed for running"
echo " mapreduce subcommands"
- echo " groups get the groups which users belong to"
echo " historyserver run job history servers as a standalone daemon"
echo " distcp <srcurl> <desturl> copy file or directories recursively"
echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive"
@@ -63,9 +62,6 @@ elif [ "$COMMAND" = "sampler" ] ; then
HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "classpath" ] ; then
echo -n
-elif [ "$COMMAND" = "groups" ] ; then
- CLASS=org.apache.hadoop.mapred.tools.GetGroups
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
elif [ "$COMMAND" = "historyserver" ] ; then
CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS"
@@ -74,7 +70,8 @@ elif [ "$COMMAND" = "historyserver" ] ;
fi
elif [ "$COMMAND" = "mradmin" ] \
|| [ "$COMMAND" = "jobtracker" ] \
- || [ "$COMMAND" = "tasktracker" ] ; then
+ || [ "$COMMAND" = "tasktracker" ] \
+ || [ "$COMMAND" = "groups" ] ; then
echo "Sorry, the $COMMAND command is no longer supported."
echo "You may find similar functionality with the \"yarn\" shell command."
print_usage
@@ -115,6 +112,11 @@ for f in $HADOOP_MAPRED_HOME/${MAPRED_DI
CLASSPATH=${CLASSPATH}:$f;
done
+# Need YARN jars also
+for f in $HADOOP_YARN_HOME/${YARN_DIR}/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+done
+
# add libs to CLASSPATH
for f in $HADOOP_MAPRED_HOME/${MAPRED_LIB_JARS_DIR}/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred-config.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred-config.sh?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred-config.sh (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mapred-config.sh Thu Oct 11 06:14:26 2012
@@ -38,3 +38,15 @@ else
echo "Hadoop common not found."
exit
fi
+
+# Only set locally to use in HADOOP_OPTS. No need to export.
+# The following defaults are useful when somebody directly invokes bin/mapred.
+HADOOP_MAPRED_LOG_DIR=${HADOOP_MAPRED_LOG_DIR:-${HADOOP_MAPRED_HOME}/logs}
+HADOOP_MAPRED_LOGFILE=${HADOOP_MAPRED_LOGFILE:-hadoop.log}
+HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_MAPRED_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_MAPRED_LOGFILE"
+export HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_MAPRED_ROOT_LOGGER}"
+
+
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/bin/mr-jobhistory-daemon.sh Thu Oct 11 06:14:26 2012
@@ -16,22 +16,16 @@
# limitations under the License.
-# Runs a yarn command as a daemon.
#
# Environment Variables
#
-# HADOOP_LOGFILE Hadoop log file.
-# HADOOP_ROOT_LOGGER Hadoop root logger.
# HADOOP_JHS_LOGGER Hadoop JobSummary logger.
-# YARN_CONF_DIR Alternate conf dir. Default is ${YARN_HOME}/conf.
-# YARN_LOG_DIR Where log files are stored. PWD by default.
-# YARN_MASTER host:path where hadoop code should be rsync'd from
-# YARN_PID_DIR The pid files are stored. /tmp by default.
-# YARN_IDENT_STRING A string representing this instance of hadoop. $USER by default
-# YARN_NICENESS The scheduling priority for daemons. Defaults to 0.
+# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_MAPRED_HOME}/conf.
+# HADOOP_MAPRED_PID_DIR The pid files are stored. /tmp by default.
+# HADOOP_MAPRED_NICENESS The scheduling priority for daemons. Defaults to 0.
##
-usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <mapred-command> "
+usage="Usage: mr-jobhistory-daemon.sh [--config <conf-dir>] (start|stop) <mapred-command> "
# if no args specified, show usage
if [ $# -le 1 ]; then
@@ -44,7 +38,9 @@ bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
-. $HADOOP_LIBEXEC_DIR/yarn-config.sh
+if [ -e ${HADOOP_LIBEXEC_DIR}/mapred-config.sh ]; then
+ . $HADOOP_LIBEXEC_DIR/mapred-config.sh
+fi
# get arguments
startStop=$1
@@ -69,43 +65,43 @@ hadoop_rotate_log ()
fi
}
-if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then
- . "${YARN_CONF_DIR}/yarn-env.sh"
+if [ "$HADOOP_MAPRED_IDENT_STRING" = "" ]; then
+ export HADOOP_MAPRED_IDENT_STRING="$USER"
fi
-if [ "$YARN_IDENT_STRING" = "" ]; then
- export YARN_IDENT_STRING="$USER"
-fi
+export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_PREFIX}}
+export HADOOP_MAPRED_LOGFILE=mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.log
+export HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,RFA}
+export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA}
-# get log directory
-if [ "$YARN_LOG_DIR" = "" ]; then
- export YARN_LOG_DIR="$YARN_HOME/logs"
+if [ -f "${HADOOP_CONF_DIR}/mapred-env.sh" ]; then
+ . "${HADOOP_CONF_DIR}/mapred-env.sh"
fi
-mkdir -p "$YARN_LOG_DIR"
-chown $YARN_IDENT_STRING $YARN_LOG_DIR
-if [ "$YARN_PID_DIR" = "" ]; then
- YARN_PID_DIR=/tmp
+mkdir -p "$HADOOP_MAPRED_LOG_DIR"
+chown $HADOOP_MAPRED_IDENT_STRING $HADOOP_MAPRED_LOG_DIR
+
+if [ "$HADOOP_MAPRED_PID_DIR" = "" ]; then
+ HADOOP_MAPRED_PID_DIR=/tmp
fi
-# some variables
-export HADOOP_LOGFILE=yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.log
-export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,RFA}
-export HADOOP_JHS_LOGGER=${HADOOP_JHS_LOGGER:-INFO,JSA}
-log=$YARN_LOG_DIR/yarn-$YARN_IDENT_STRING-$command-$HOSTNAME.out
-pid=$YARN_PID_DIR/yarn-$YARN_IDENT_STRING-$command.pid
-YARN_STOP_TIMEOUT=${YARN_STOP_TIMEOUT:-5}
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_MAPRED_IDENT_STRING"
+
+log=$HADOOP_MAPRED_LOG_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command-$HOSTNAME.out
+pid=$HADOOP_MAPRED_PID_DIR/mapred-$HADOOP_MAPRED_IDENT_STRING-$command.pid
+
+HADOOP_MAPRED_STOP_TIMEOUT=${HADOOP_MAPRED_STOP_TIMEOUT:-5}
# Set default scheduling priority
-if [ "$YARN_NICENESS" = "" ]; then
- export YARN_NICENESS=0
+if [ "$HADOOP_MAPRED_NICENESS" = "" ]; then
+ export HADOOP_MAPRED_NICENESS=0
fi
case $startStop in
(start)
- mkdir -p "$YARN_PID_DIR"
+ mkdir -p "$HADOOP_MAPRED_PID_DIR"
if [ -f $pid ]; then
if kill -0 `cat $pid` > /dev/null 2>&1; then
@@ -114,15 +110,10 @@ case $startStop in
fi
fi
- if [ "$YARN_MASTER" != "" ]; then
- echo rsync from $YARN_MASTER
- rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' $YARN_MASTER/ "$YARN_HOME"
- fi
-
hadoop_rotate_log $log
echo starting $command, logging to $log
- cd "$YARN_HOME"
- nohup nice -n $YARN_NICENESS "$YARN_HOME"/bin/mapred --config $YARN_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
+ cd "$HADOOP_MAPRED_HOME"
+ nohup nice -n $HADOOP_MAPRED_NICENESS "$HADOOP_MAPRED_HOME"/bin/mapred --config $HADOOP_CONF_DIR $command "$@" > "$log" 2>&1 < /dev/null &
echo $! > $pid
sleep 1; head "$log"
;;
@@ -134,9 +125,9 @@ case $startStop in
if kill -0 $TARGET_PID > /dev/null 2>&1; then
echo stopping $command
kill $TARGET_PID
- sleep $YARN_STOP_TIMEOUT
+ sleep $HADOOP_MAPRED_STOP_TIMEOUT
if kill -0 $TARGET_PID > /dev/null 2>&1; then
- echo "$command did not stop gracefully after $YARN_STOP_TIMEOUT seconds: killing with kill -9"
+ echo "$command did not stop gracefully after $HADOOP_MAPRED_STOP_TIMEOUT seconds: killing with kill -9"
kill -9 $TARGET_PID
fi
else
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Thu Oct 11 06:14:26 2012
@@ -490,19 +490,17 @@ public class MRAppMaster extends Composi
try {
this.currentUser = UserGroupInformation.getCurrentUser();
- if (UserGroupInformation.isSecurityEnabled()) {
- // Read the file-system tokens from the localized tokens-file.
- Path jobSubmitDir =
- FileContext.getLocalFSFileContext().makeQualified(
- new Path(new File(MRJobConfig.JOB_SUBMIT_DIR)
- .getAbsolutePath()));
- Path jobTokenFile =
- new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
- fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
- LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
- + jobTokenFile);
- currentUser.addCredentials(fsTokens); // For use by AppMaster itself.
- }
+ // Read the file-system tokens from the localized tokens-file.
+ Path jobSubmitDir =
+ FileContext.getLocalFSFileContext().makeQualified(
+ new Path(new File(MRJobConfig.JOB_SUBMIT_DIR)
+ .getAbsolutePath()));
+ Path jobTokenFile =
+ new Path(jobSubmitDir, MRJobConfig.APPLICATION_TOKENS_FILE);
+ fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf));
+ LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile="
+ + jobTokenFile);
+ currentUser.addCredentials(fsTokens); // For use by AppMaster itself.
} catch (IOException e) {
throw new YarnException(e);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Thu Oct 11 06:14:26 2012
@@ -82,8 +82,7 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -115,16 +114,15 @@ public class MRClientService extends Abs
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress address = new InetSocketAddress(0);
- ClientToAMSecretManager secretManager = null;
+ ClientToAMTokenSecretManager secretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
- secretManager = new ClientToAMSecretManager();
String secretKeyStr =
System
.getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
byte[] bytes = Base64.decodeBase64(secretKeyStr);
- ClientTokenIdentifier identifier = new ClientTokenIdentifier(
- this.appContext.getApplicationID());
- secretManager.setMasterKey(identifier, bytes);
+ secretManager =
+ new ClientToAMTokenSecretManager(this.appContext.getApplicationID(),
+ bytes);
}
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Thu Oct 11 06:14:26 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@@ -610,10 +611,12 @@ public abstract class TaskAttemptImpl im
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
- localResources.put(
- MRJobConfig.JOB_JAR,
- createLocalResource(remoteFS, remoteJobJar,
- LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION));
+ LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+ LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+ String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+ JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+ rc.setPattern(pattern);
+ localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info("The job-jar file on the remote FS is "
+ remoteJobJar.toUri().toASCIIString());
} else {
@@ -644,14 +647,10 @@ public abstract class TaskAttemptImpl im
MRApps.setupDistributedCache(conf, localResources);
// Setup up task credentials buffer
- Credentials taskCredentials = new Credentials();
-
- if (UserGroupInformation.isSecurityEnabled()) {
- LOG.info("Adding #" + credentials.numberOfTokens()
- + " tokens and #" + credentials.numberOfSecretKeys()
- + " secret keys for NM use for launching container");
- taskCredentials.addAll(credentials);
- }
+ LOG.info("Adding #" + credentials.numberOfTokens()
+ + " tokens and #" + credentials.numberOfSecretKeys()
+ + " secret keys for NM use for launching container");
+ Credentials taskCredentials = new Credentials(credentials);
// LocalStorageToken is needed irrespective of whether security is enabled
// or not.
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java Thu Oct 11 06:14:26 2012
@@ -179,6 +179,10 @@ import org.junit.Test;
public Configuration getConfig() {
return conf;
}
+
+ @Override
+ protected void downloadTokensAndSetupUGI(Configuration conf) {
+ }
}
private final class MRAppTestCleanup extends MRApp {
@@ -266,4 +270,4 @@ import org.junit.Test;
Assert.assertTrue("Staging directory not cleaned before notifying RM",
app.cleanedBeforeContainerAllocatorStopped);
}
- }
\ No newline at end of file
+ }
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Thu Oct 11 06:14:26 2012
@@ -152,6 +152,10 @@ class LocalDistributedCacheManager {
localArchives.add(pathString);
} else if (resource.getType() == LocalResourceType.FILE) {
localFiles.add(pathString);
+ } else if (resource.getType() == LocalResourceType.PATTERN) {
+ //PATTERN is not currently used in local mode
+ throw new IllegalArgumentException("Resource type PATTERN is not " +
+ "implemented yet. " + resource.getResource());
}
Path resourcePath;
try {
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Thu Oct 11 06:14:26 2012
@@ -210,7 +210,7 @@ public class MRApps extends Apps {
Apps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
- MRJobConfig.JOB_JAR + Path.SEPARATOR);
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR);
Apps.addToEnvironment(
environment,
Environment.CLASSPATH.name(),
@@ -281,7 +281,7 @@ public class MRApps extends Apps {
}
private static String getResourceDescription(LocalResourceType type) {
- if(type == LocalResourceType.ARCHIVE) {
+ if(type == LocalResourceType.ARCHIVE || type == LocalResourceType.PATTERN) {
return "cache archive (" + MRJobConfig.CACHE_ARCHIVES + ") ";
}
return "cache file (" + MRJobConfig.CACHE_FILES + ") ";
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java Thu Oct 11 06:14:26 2012
@@ -166,7 +166,7 @@ public class TestMRApps {
}
String env_str = env.get("CLASSPATH");
assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
- env_str.indexOf("$PWD:job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
+ env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0);
}
@Test public void testSetClasspathWithNoUserPrecendence() {
@@ -180,7 +180,7 @@ public class TestMRApps {
}
String env_str = env.get("CLASSPATH");
int index =
- env_str.indexOf("job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*");
+ env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*");
assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not"
+ " in the classpath!", index, -1);
assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml Thu Oct 11 06:14:26 2012
@@ -68,6 +68,24 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>target/generated-sources/avro</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Thu Oct 11 06:14:26 2012
@@ -424,6 +424,7 @@ class MapTask extends Task {
job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart());
job.setLong(JobContext.MAP_INPUT_PATH, fileSplit.getLength());
}
+ LOG.info("Processing split: " + inputSplit);
}
static class NewTrackingRecordReader<K,V>
@@ -694,6 +695,7 @@ class MapTask extends Task {
org.apache.hadoop.mapreduce.InputSplit split = null;
split = getSplitDetails(new Path(splitIndex.getSplitLocation()),
splitIndex.getStartOffset());
+ LOG.info("Processing split: " + split);
org.apache.hadoop.mapreduce.RecordReader<INKEY,INVALUE> input =
new NewTrackingRecordReader<INKEY,INVALUE>
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TotalOrderPartitioner.java Thu Oct 11 06:14:26 2012
@@ -31,7 +31,7 @@ import org.apache.hadoop.mapred.Partitio
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class TotalOrderPartitioner<K extends WritableComparable<?>,V>
+public class TotalOrderPartitioner<K ,V>
extends org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner<K, V>
implements Partitioner<K,V> {
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/TotalOrderPartitioner.java Thu Oct 11 06:14:26 2012
@@ -47,7 +47,7 @@ import org.apache.hadoop.util.Reflection
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class TotalOrderPartitioner<K extends WritableComparable<?>,V>
+public class TotalOrderPartitioner<K,V>
extends Partitioner<K,V> implements Configurable {
private Node partitions;
@@ -298,12 +298,13 @@ public class TotalOrderPartitioner<K ext
@SuppressWarnings("unchecked") // map output key class
private K[] readPartitions(FileSystem fs, Path p, Class<K> keyClass,
Configuration conf) throws IOException {
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, p, conf);
+ SequenceFile.Reader reader = new SequenceFile.Reader(
+ conf,
+ SequenceFile.Reader.file(p));
ArrayList<K> parts = new ArrayList<K>();
K key = ReflectionUtils.newInstance(keyClass, conf);
- NullWritable value = NullWritable.get();
try {
- while (reader.next(key, value)) {
+ while ((key = (K) reader.next(key)) != null) {
parts.add(key);
key = ReflectionUtils.newInstance(keyClass, conf);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Thu Oct 11 06:14:26 2012
@@ -24,47 +24,6 @@
<configuration>
<property>
- <name>mapreduce.jobtracker.jobhistory.location</name>
- <value></value>
- <description> If job tracker is static the history files are stored
- in this single well known place. If No value is set here, by default,
- it is in the local file system at ${hadoop.log.dir}/history.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.jobhistory.task.numberprogresssplits</name>
- <value>12</value>
- <description> Every task attempt progresses from 0.0 to 1.0 [unless
- it fails or is killed]. We record, for each task attempt, certain
- statistics over each twelfth of the progress range. You can change
- the number of intervals we divide the entire range of progress into
- by setting this property. Higher values give more precision to the
- recorded data, but costs more memory in the job tracker at runtime.
- Each increment in this attribute costs 16 bytes per running task.
- </description>
-</property>
-
-<property>
- <name>mapreduce.job.userhistorylocation</name>
- <value></value>
- <description> User can specify a location to store the history files of
- a particular job. If nothing is specified, the logs are stored in
- output directory. The files are stored in "_logs/history/" in the directory.
- User can stop logging by giving the value "none".
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.jobhistory.completed.location</name>
- <value></value>
- <description> The completed job history files are stored at this single well
- known location. If nothing is specified, the files are stored at
- ${mapreduce.jobtracker.jobhistory.location}/done.
- </description>
-</property>
-
-<property>
<name>mapreduce.job.committer.setup.cleanup.needed</name>
<value>true</value>
<description> true, if job needs job-setup and job-cleanup.
@@ -99,15 +58,6 @@
</property>
<property>
- <name>mapreduce.jobtracker.address</name>
- <value>local</value>
- <description>The host and port that the MapReduce job tracker runs
- at. If "local", then jobs are run in-process as a single map
- and reduce task.
- </description>
-</property>
-
-<property>
<name>mapreduce.local.clientfactory.class.name</name>
<value>org.apache.hadoop.mapred.LocalClientFactory</value>
<description>This the client factory that is responsible for
@@ -115,138 +65,10 @@
</property>
<property>
- <name>mapreduce.jobtracker.http.address</name>
- <value>0.0.0.0:50030</value>
- <description>
- The job tracker http server address and port the server will listen on.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.handler.count</name>
- <value>10</value>
- <description>
- The number of server threads for the JobTracker. This should be roughly
- 4% of the number of tasktracker nodes.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.report.address</name>
- <value>127.0.0.1:0</value>
- <description>The interface and port that task tracker server listens on.
- Since it is only connected to by the tasks, it uses the local interface.
- EXPERT ONLY. Should only be changed if your host does not have the loopback
- interface.</description>
-</property>
-
-<property>
- <name>mapreduce.cluster.local.dir</name>
- <value>${hadoop.tmp.dir}/mapred/local</value>
- <description>The local directory where MapReduce stores intermediate
- data files. May be a comma-separated list of
- directories on different devices in order to spread disk i/o.
- Directories that do not exist are ignored.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.system.dir</name>
- <value>${hadoop.tmp.dir}/mapred/system</value>
- <description>The directory where MapReduce stores control files.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.staging.root.dir</name>
- <value>${hadoop.tmp.dir}/mapred/staging</value>
- <description>The root of the staging area for users' job files
- In practice, this should be the directory where users' home
- directories are located (usually /user)
- </description>
-</property>
-
-<property>
- <name>mapreduce.cluster.temp.dir</name>
- <value>${hadoop.tmp.dir}/mapred/temp</value>
- <description>A shared directory for temporary files.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.local.dir.minspacestart</name>
- <value>0</value>
- <description>If the space in mapreduce.cluster.local.dir drops under this,
- do not ask for more tasks.
- Value in bytes.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.local.dir.minspacekill</name>
- <value>0</value>
- <description>If the space in mapreduce.cluster.local.dir drops under this,
- do not ask more tasks until all the current ones have finished and
- cleaned up. Also, to save the rest of the tasks we have running,
- kill one of them, to clean up some space. Start with the reduce tasks,
- then go with the ones that have finished the least.
- Value in bytes.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.expire.trackers.interval</name>
- <value>600000</value>
- <description>Expert: The time-interval, in miliseconds, after which
- a tasktracker is declared 'lost' if it doesn't send heartbeats.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.instrumentation</name>
- <value>org.apache.hadoop.mapred.TaskTrackerMetricsInst</value>
- <description>Expert: The instrumentation class to associate with each TaskTracker.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.resourcecalculatorplugin</name>
- <value></value>
- <description>
- Name of the class whose instance will be used to query resource information
- on the tasktracker.
-
- The class must be an instance of
- org.apache.hadoop.util.ResourceCalculatorPlugin. If the value is null, the
- tasktracker attempts to use a class appropriate to the platform.
- Currently, the only platform supported is Linux.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.taskmemorymanager.monitoringinterval</name>
- <value>5000</value>
- <description>The interval, in milliseconds, for which the tasktracker waits
- between two cycles of monitoring its tasks' memory usage. Used only if
- tasks' memory management is enabled via mapred.tasktracker.tasks.maxmemory.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.tasks.sleeptimebeforesigkill</name>
- <value>5000</value>
- <description>The time, in milliseconds, the tasktracker waits for sending a
- SIGKILL to a task, after it has been sent a SIGTERM. This is currently
- not used on WINDOWS where tasks are just sent a SIGTERM.
- </description>
-</property>
-
-<property>
<name>mapreduce.job.maps</name>
<value>2</value>
<description>The default number of map tasks per job.
- Ignored when mapreduce.jobtracker.address is "local".
+ Ignored when mapreduce.framework.name is "local".
</description>
</property>
@@ -256,54 +78,21 @@
<description>The default number of reduce tasks per job. Typically set to 99%
of the cluster's reduce capacity, so that if a node fails the reduces can
still be executed in a single wave.
- Ignored when mapreduce.jobtracker.address is "local".
+ Ignored when mapreduce.framework.name is "local".
</description>
</property>
<property>
- <name>mapreduce.jobtracker.restart.recover</name>
- <value>false</value>
- <description>"true" to enable (job) recovery upon restart,
- "false" to start afresh
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.jobhistory.block.size</name>
- <value>3145728</value>
- <description>The block size of the job history file. Since the job recovery
- uses job history, its important to dump job history to disk as
- soon as possible. Note that this is an expert level parameter.
- The default value is set to 3 MB.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.taskscheduler</name>
- <value>org.apache.hadoop.mapred.JobQueueTaskScheduler</value>
- <description>The class responsible for scheduling the tasks.</description>
-</property>
-
-
-<property>
<name>mapreduce.job.split.metainfo.maxsize</name>
<value>10000000</value>
<description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value.
+ The MapReduce ApplicationMaster won't attempt to read submitted split metainfo
+ files bigger than this configured value.
No limits if set to -1.
</description>
</property>
<property>
- <name>mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob</name>
- <value></value>
- <description>The maximum number of running tasks for a job before
- it gets preempted. No limits if undefined.
- </description>
-</property>
-
-<property>
<name>mapreduce.map.maxattempts</name>
<value>4</value>
<description>Expert: The maximum number of attempts per map task.
@@ -333,7 +122,7 @@
<name>mapreduce.reduce.shuffle.connect.timeout</name>
<value>180000</value>
<description>Expert: The maximum amount of time (in milli seconds) reduce
- task spends in trying to connect to a tasktracker for getting map output.
+ task spends in trying to connect to a remote node for getting map output.
</description>
</property>
@@ -355,51 +144,6 @@
</description>
</property>
-<property>
- <name>mapreduce.tasktracker.map.tasks.maximum</name>
- <value>2</value>
- <description>The maximum number of map tasks that will be run
- simultaneously by a task tracker.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.reduce.tasks.maximum</name>
- <value>2</value>
- <description>The maximum number of reduce tasks that will be run
- simultaneously by a task tracker.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.retiredjobs.cache.size</name>
- <value>1000</value>
- <description>The number of retired job status to keep in the cache.
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.outofband.heartbeat</name>
- <value>false</value>
- <description>Expert: Set this to true to let the tasktracker send an
- out-of-band heartbeat on task-completion for better latency.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.jobhistory.lru.cache.size</name>
- <value>5</value>
- <description>The number of job history files loaded in memory. The jobs are
- loaded when they are first accessed. The cache is cleared based on LRU.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.instrumentation</name>
- <value>org.apache.hadoop.mapred.JobTrackerMetricsInst</value>
- <description>Expert: The instrumentation class to associate with each JobTracker.
- </description>
-</property>
<property>
<name>mapred.child.java.opts</name>
@@ -424,7 +168,7 @@
<description>User added environment variables for the task tracker child
processes. Example :
1) A=foo This will set the env variable A to foo
- 2) B=$B:c This is inherit tasktracker's B env variable.
+ 2) B=$B:c This is inherit nodemanager's B env variable.
</description>
</property>
@@ -568,17 +312,9 @@
<name>mapreduce.job.speculative.slownodethreshold</name>
<value>1.0</value>
<description>The number of standard deviations by which a Task
- Tracker's ave map and reduce progress-rates (finishTime-dispatchTime)
+ Tracker's average map and reduce progress-rates (finishTime-dispatchTime)
must be lower than the average of all successful map/reduce task's for
- the TT to be considered too slow to give a speculative task to.
- </description>
-</property>
-
-<property>
- <name>mapreduce.job.jvm.numtasks</name>
- <value>1</value>
- <description>How many tasks to run per jvm. If set to -1, there is
- no limit.
+ the NodeManager to be considered too slow to give a speculative task to.
</description>
</property>
@@ -630,12 +366,6 @@
take priority over this setting.</description>
</property>
-<property>
- <name>mapreduce.jobtracker.maxtasks.perjob</name>
- <value>-1</value>
- <description>The maximum number of tasks for a single job.
- A value of -1 indicates that there is no maximum. </description>
-</property>
<property>
<name>mapreduce.client.submit.file.replication</name>
@@ -645,41 +375,6 @@
</description>
</property>
-
-<property>
- <name>mapreduce.tasktracker.dns.interface</name>
- <value>default</value>
- <description>The name of the Network Interface from which a task
- tracker should report its IP address.
- </description>
- </property>
-
-<property>
- <name>mapreduce.tasktracker.dns.nameserver</name>
- <value>default</value>
- <description>The host name or IP address of the name server (DNS)
- which a TaskTracker should use to determine the host name used by
- the JobTracker for communication and display purposes.
- </description>
- </property>
-
-<property>
- <name>mapreduce.tasktracker.http.threads</name>
- <value>40</value>
- <description>The number of worker threads that for the http server. This is
- used for map output fetching
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.http.address</name>
- <value>0.0.0.0:50060</value>
- <description>
- The task tracker http server address and port.
- If the port is 0 then the server will start on a free port.
- </description>
-</property>
-
<property>
<name>mapreduce.task.files.preserve.failedtasks</name>
<value>false</value>
@@ -752,53 +447,9 @@
</property>
<property>
- <name>mapreduce.job.userlog.retain.hours</name>
- <value>24</value>
- <description>The maximum time, in hours, for which the user-logs are to be
- retained after the job completion.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.hosts.filename</name>
- <value></value>
- <description>Names a file that contains the list of nodes that may
- connect to the jobtracker. If the value is empty, all hosts are
- permitted.</description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.hosts.exclude.filename</name>
- <value></value>
- <description>Names a file that contains the list of hosts that
- should be excluded by the jobtracker. If the value is empty, no
- hosts are excluded.</description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.heartbeats.in.second</name>
- <value>100</value>
- <description>Expert: Approximate number of heart-beats that could arrive
- at JobTracker in a second. Assuming each RPC can be processed
- in 10msec, the default value is made 100 RPCs in a second.
- </description>
-</property>
-
-<property>
- <name>mapreduce.jobtracker.tasktracker.maxblacklists</name>
- <value>4</value>
- <description>The number of blacklists for a taskTracker by various jobs
- after which the task tracker could be blacklisted across
- all jobs. The tracker will be given a tasks later
- (after a day). The tracker will become a healthy
- tracker after a restart.
- </description>
-</property>
-
-<property>
<name>mapreduce.job.maxtaskfailures.per.tracker</name>
<value>3</value>
- <description>The number of task-failures on a tasktracker of a given job
+ <description>The number of task-failures on a node manager of a given job
after which new tasks of that job aren't assigned to it. It
MUST be less than mapreduce.map.maxattempts and
mapreduce.reduce.maxattempts otherwise the failed task will
@@ -820,8 +471,8 @@
<name>mapreduce.client.completion.pollinterval</name>
<value>5000</value>
<description>The interval (in milliseconds) between which the JobClient
- polls the JobTracker for updates about job status. You may want to set this
- to a lower value to make tests run faster on a single node system. Adjusting
+ polls the MapReduce ApplicationMaster for updates about job status. You may want to
+ set this to a lower value to make tests run faster on a single node system. Adjusting
this value in production may lead to unwanted client-server traffic.
</description>
</property>
@@ -836,32 +487,6 @@
</description>
</property>
- <property>
- <name>mapreduce.jobtracker.persist.jobstatus.active</name>
- <value>true</value>
- <description>Indicates if persistency of job status information is
- active or not.
- </description>
- </property>
-
- <property>
- <name>mapreduce.jobtracker.persist.jobstatus.hours</name>
- <value>1</value>
- <description>The number of hours job status information is persisted in DFS.
- The job status information will be available after it drops of the memory
- queue and between jobtracker restarts. With a zero value the job status
- information is not persisted at all in DFS.
- </description>
-</property>
-
- <property>
- <name>mapreduce.jobtracker.persist.jobstatus.dir</name>
- <value>/jobtracker/jobsInfo</value>
- <description>The directory where the job status information is persisted
- in a file system to be available after it drops of the memory queue and
- between jobtracker restarts.
- </description>
- </property>
<property>
<name>mapreduce.task.profile</name>
@@ -894,8 +519,8 @@
<description> The number of Task attempts AFTER which skip mode
will be kicked off. When skip mode is kicked off, the
tasks reports the range of records which it will process
- next, to the TaskTracker. So that on failures, TT knows which
- ones are possibly the bad records. On further executions,
+ next, to the MR ApplicationMaster. So that on failures, the MR AM
+ knows which ones are possibly the bad records. On further executions,
those are skipped.
</description>
</property>
@@ -1006,15 +631,6 @@
calls</description>
</property>
-<!-- Proxy Configuration -->
-<property>
- <name>mapreduce.jobtracker.taskcache.levels</name>
- <value>2</value>
- <description> This is the max level of the task cache. For example, if
- the level is 2, the tasks cached are at the host level and at the rack
- level.
- </description>
-</property>
<property>
<name>mapreduce.job.queuename</name>
@@ -1028,18 +644,29 @@
</property>
<property>
+ <name>mapreduce.cluster.local.dir</name>
+ <value>${hadoop.tmp.dir}/mapred/local</value>
+ <description>
+ The local directory where MapReduce stores intermediate
+ data files. May be a comma-separated list of
+ directories on different devices in order to spread disk i/o.
+ Directories that do not exist are ignored.
+ </description>
+</property>
+
+<property>
<name>mapreduce.cluster.acls.enabled</name>
<value>false</value>
<description> Specifies whether ACLs should be checked
for authorization of users for doing various queue and job level operations.
ACLs are disabled by default. If enabled, access control checks are made by
- JobTracker and TaskTracker when requests are made by users for queue
+ MapReduce ApplicationMaster when requests are made by users for queue
operations like submit job to a queue and kill a job in the queue and job
operations like viewing the job-details (See mapreduce.job.acl-view-job)
or for modifying the job (See mapreduce.job.acl-modify-job) using
Map/Reduce APIs, RPCs or via the console and web user interfaces.
- For enabling this flag(mapreduce.cluster.acls.enabled), this is to be set
- to true in mapred-site.xml on JobTracker node and on all TaskTracker nodes.
+ For enabling this flag, set to true in mapred-site.xml file of all
+ MapReduce clients (MR job submitting nodes).
</description>
</property>
@@ -1092,8 +719,8 @@
o job-level counters
o task-level counters
o tasks' diagnostic information
- o task-logs displayed on the TaskTracker web-UI and
- o job.xml showed by the JobTracker's web-UI
+ o task-logs displayed on the HistoryServer's web-UI and
+ o job.xml showed by the HistoryServer's web-UI
Every other piece of information of jobs is still accessible by any other
user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc.
@@ -1111,18 +738,10 @@
</property>
<property>
- <name>mapreduce.tasktracker.indexcache.mb</name>
- <value>10</value>
- <description> The maximum memory that a task tracker allows for the
- index cache that is used when serving map outputs to reducers.
- </description>
-</property>
-
-<property>
<name>mapreduce.task.merge.progress.records</name>
<value>10000</value>
<description> The number of records to process during merge before
- sending a progress notification to the TaskTracker.
+ sending a progress notification to the MR ApplicationMaster.
</description>
</property>
@@ -1143,22 +762,6 @@
</property>
<property>
- <name>mapreduce.tasktracker.taskcontroller</name>
- <value>org.apache.hadoop.mapred.DefaultTaskController</value>
- <description>TaskController which is used to launch and manage task execution
- </description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.group</name>
- <value></value>
- <description>Expert: Group to which TaskTracker belongs. If
- LinuxTaskController is configured via mapreduce.tasktracker.taskcontroller,
- the group owner of the task-controller binary should be same as this group.
- </description>
-</property>
-
-<property>
<name>mapreduce.shuffle.port</name>
<value>8080</value>
<description>Default port that the ShuffleHandler will run on. ShuffleHandler
@@ -1167,42 +770,6 @@
</description>
</property>
-<!-- Node health script variables -->
-
-<property>
- <name>mapreduce.tasktracker.healthchecker.script.path</name>
- <value></value>
- <description>Absolute path to the script which is
- periodicallyrun by the node health monitoring service to determine if
- the node is healthy or not. If the value of this key is empty or the
- file does not exist in the location configured here, the node health
- monitoring service is not started.</description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.healthchecker.interval</name>
- <value>60000</value>
- <description>Frequency of the node health script to be run,
- in milliseconds</description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.healthchecker.script.timeout</name>
- <value>600000</value>
- <description>Time after node health script should be killed if
- unresponsive and considered that the script has failed.</description>
-</property>
-
-<property>
- <name>mapreduce.tasktracker.healthchecker.script.args</name>
- <value></value>
- <description>List of arguments which are to be passed to
- node health script when it is being launched comma seperated.
- </description>
-</property>
-
-<!-- end of node health script variables -->
-
<!-- MR YARN Application properties -->
<property>
Propchange: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1390199-1396916
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTask.java Thu Oct 11 06:14:26 2012
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.util.Recor
public class CompletedTask implements Task {
+ private static final Counters EMPTY_COUNTERS = new Counters();
+
private final TaskId taskId;
private final TaskInfo taskInfo;
private TaskReport report;
@@ -124,7 +126,11 @@ public class CompletedTask implements Ta
report.setFinishTime(taskInfo.getFinishTime());
report.setTaskState(getState());
report.setProgress(getProgress());
- report.setCounters(TypeConverter.toYarn(getCounters()));
+ Counters counters = getCounters();
+ if (counters == null) {
+ counters = EMPTY_COUNTERS;
+ }
+ report.setCounters(TypeConverter.toYarn(counters));
if (successfulAttempt != null) {
report.setSuccessfulAttempt(successfulAttempt);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Thu Oct 11 06:14:26 2012
@@ -23,14 +23,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
@@ -62,6 +62,7 @@ import org.apache.hadoop.mapreduce.v2.jo
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.service.AbstractService;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -77,7 +78,7 @@ public class HistoryFileManager extends
private static enum HistoryInfoState {
IN_INTERMEDIATE, IN_DONE, DELETED, MOVE_FAILED
};
-
+
private static String DONE_BEFORE_SERIAL_TAIL = JobHistoryUtils
.doneSubdirsBeforeSerialTail();
@@ -130,7 +131,7 @@ public class HistoryFileManager extends
}
}
- private static class JobListCache {
+ static class JobListCache {
private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
private int maxSize;
private long maxAge;
@@ -199,6 +200,29 @@ public class HistoryFileManager extends
}
}
+ /**
+ * This class represents a user dir in the intermediate done directory. This
+ * is mostly for locking purposes.
+ */
+ private class UserLogDir {
+ long modTime = 0;
+
+ public synchronized void scanIfNeeded(FileStatus fs) {
+ long newModTime = fs.getModificationTime();
+ if (modTime != newModTime) {
+ Path p = fs.getPath();
+ try {
+ scanIntermediateDirectory(p);
+ //If scanning fails, we will scan again. We assume the failure is
+ // temporary.
+ modTime = newModTime;
+ } catch (IOException e) {
+ LOG.error("Error while trying to scan the directory " + p, e);
+ }
+ }
+ }
+ }
+
public class HistoryFileInfo {
private Path historyFile;
private Path confFile;
@@ -216,12 +240,14 @@ public class HistoryFileManager extends
: HistoryInfoState.IN_INTERMEDIATE;
}
- private synchronized boolean isMovePending() {
+ @VisibleForTesting
+ synchronized boolean isMovePending() {
return state == HistoryInfoState.IN_INTERMEDIATE
|| state == HistoryInfoState.MOVE_FAILED;
}
- private synchronized boolean didMoveFail() {
+ @VisibleForTesting
+ synchronized boolean didMoveFail() {
return state == HistoryInfoState.MOVE_FAILED;
}
@@ -342,7 +368,7 @@ public class HistoryFileManager extends
}
private SerialNumberIndex serialNumberIndex = null;
- private JobListCache jobListCache = null;
+ protected JobListCache jobListCache = null;
// Maintains a list of known done subdirectories.
private final Set<Path> existingDoneSubdirs = Collections
@@ -352,7 +378,8 @@ public class HistoryFileManager extends
* Maintains a mapping between intermediate user directories and the last
* known modification time.
*/
- private Map<String, Long> userDirModificationTimeMap = new HashMap<String, Long>();
+ private ConcurrentMap<String, UserLogDir> userDirModificationTimeMap =
+ new ConcurrentHashMap<String, UserLogDir>();
private JobACLsManager aclsMgr;
@@ -584,23 +611,15 @@ public class HistoryFileManager extends
for (FileStatus userDir : userDirList) {
String name = userDir.getPath().getName();
- long newModificationTime = userDir.getModificationTime();
- boolean shouldScan = false;
- synchronized (userDirModificationTimeMap) {
- if (!userDirModificationTimeMap.containsKey(name)
- || newModificationTime > userDirModificationTimeMap.get(name)) {
- shouldScan = true;
- userDirModificationTimeMap.put(name, newModificationTime);
- }
- }
- if (shouldScan) {
- try {
- scanIntermediateDirectory(userDir.getPath());
- } catch (IOException e) {
- LOG.error("Error while trying to scan the directory "
- + userDir.getPath(), e);
+ UserLogDir dir = userDirModificationTimeMap.get(name);
+ if(dir == null) {
+ dir = new UserLogDir();
+ UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
+ if(old != null) {
+ dir = old;
}
}
+ dir.scanIfNeeded(userDir);
}
}
@@ -691,8 +710,8 @@ public class HistoryFileManager extends
* @throws IOException
*/
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
- int jobSerialNumber = JobHistoryUtils.jobSerialNumber(jobId);
- String boxedSerialNumber = String.valueOf(jobSerialNumber);
+ String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
+ jobId, serialNumberFormat);
Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
if (dateStringSet == null) {
return null;