You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by su...@apache.org on 2012/11/30 20:58:54 UTC
svn commit: r1415815 [1/3] - in
/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./ conf/
dev-support/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/h...
Author: suresh
Date: Fri Nov 30 19:58:09 2012
New Revision: 1415815
URL: http://svn.apache.org/viewvc?rev=1415815&view=rev
Log:
Merging trunk into branch-trunk-win
Added:
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/
- copied from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
- copied unchanged from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/
- copied from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/
- copied from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
- copied unchanged from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/
- copied from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/css/
- copied from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/css/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/css/site.css
- copied unchanged from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/resources/css/site.css
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/site.xml
- copied unchanged from r1415786, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/site.xml
Removed:
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/build-utils.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/build.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/ivy/
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/ivy.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/src/
Modified:
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppView.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestKill.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTaskStatus.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/NLineInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (contents, props changed)
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAboutPage.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsNavBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksBlock.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTasksPage.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/HistoryInfo.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestNLineInputFormat.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java
hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1407223-1415786
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt Fri Nov 30 19:58:09 2012
@@ -166,6 +166,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8911. CRLF characters in source and text files.
(Raja Aluri via suresh)
+ MAPREDUCE-4723. Fix warnings found by findbugs 2. (Sandy Ryza via eli)
+
OPTIMIZATIONS
BUG FIXES
@@ -197,6 +199,9 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4777. In TestIFile, testIFileReaderWithCodec relies on
testIFileWriterWithCodec. (Sandy Ryza via tomwhite)
+ MAPREDUCE-4800. Cleanup o.a.h.mapred.MapTaskStatus - remove unused
+ code. (kkambatl via tucu)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -573,6 +578,32 @@ Release 2.0.0-alpha - 05-23-2012
MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is
bad (Jason Lowe via bobby)
+Release 0.23.6 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ MAPREDUCE-4811. JobHistoryServer should show when it was started in WebUI
+ About page (Ravi Prakash via jlowe)
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-4802. Takes a long time to load the task list on the AM for
+ large jobs (Ravi Prakash via bobby)
+
+ MAPREDUCE-4764. repair TestBinaryTokenFile (Ivan A. Veselovsky via bobby)
+
+ MAPREDUCE-4825. JobImpl.finished doesn't expect ERROR as a final job state
+ (jlowe via bobby)
+
+ MAPREDUCE-4817. Hardcoded task ping timeout kills tasks localizing large
+ amounts of data (tgraves)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -587,8 +618,15 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert
Evans via tgraves)
+ MAPREDUCE-4266. remove Ant remnants from MR (tgraves via bobby)
+
+ MAPREDUCE-4666. JVM metrics for history server (jlowe via jeagles)
+
OPTIMIZATIONS
+ MAPREDUCE-4720. Browser thinks History Server main page JS is taking too
+ long (Ravi Prakash via bobby)
+
BUG FIXES
MAPREDUCE-4554. Job Credentials are not transmitted if security is turned
@@ -640,8 +678,33 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4772. Fetch failures can take way too long for a map to be
restarted (bobby)
+
+ MAPREDUCE-4782. NLineInputFormat skips first line of last InputSplit
+ (Mark Fuhs via bobby)
+
+ MAPREDUCE-4774. JobImpl does not handle asynchronous task events in FAILED
+ state (jlowe via bobby)
+
+ MAPREDUCE-4751. AM stuck in KILL_WAIT for days (vinodkv via bobby)
+
+ MAPREDUCE-4787. TestJobMonitorAndPrint is broken (Rob Parker via bobby)
+
+ MAPREDUCE-4425. Speculation + Fetch failures can lead to a hung job (jlowe
+ via bobby)
+
+ MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
+ default (Ravi Prakash via bobby)
+
+ MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat
+ (Jason Lowe via tgraves)
+
+ MAPREDUCE-4797. LocalContainerAllocator can loop forever trying to contact
+ the RM (jlowe via bobby)
+
+ MAPREDUCE-4801. ShuffleHandler can generate large logs due to prematurely
+ closed channels (jlowe via bobby)
-Release 0.23.4 - UNRELEASED
+Release 0.23.4
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1407223-1415786
Propchange: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1407223-1415786
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml Fri Nov 30 19:58:09 2012
@@ -479,4 +479,28 @@
<Field name="sslFileBufferSize" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+ <Method name="sendSignal" />
+ <Bug pattern="NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+ <Method name="isSetsidSupported" />
+ <Bug pattern="NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+ <Method name="isSetsidSupported" />
+ <Bug pattern="NP_NULL_ON_SOME_PATH_EXCEPTION" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage$1" />
+ <Bug pattern="SE_BAD_FIELD_INNER_CLASS" />
+ </Match>
+
</FindBugsFilter>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Fri Nov 30 19:58:09 2012
@@ -89,11 +89,6 @@
<phase>test-compile</phase>
</execution>
</executions>
- <configuration>
- <excludes>
- <exclude>mrapp-generated-classpath</exclude>
- </excludes>
- </configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java Fri Nov 30 19:58:09 2012
@@ -359,9 +359,8 @@ public class LocalContainerLauncher exte
+ StringUtils.stringifyException(e));
}
// Report back any failures, for diagnostic purposes
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- exception.printStackTrace(new PrintStream(baos));
- umbilical.reportDiagnosticInfo(classicAttemptID, baos.toString());
+ umbilical.reportDiagnosticInfo(classicAttemptID,
+ StringUtils.stringifyException(exception));
throw new RuntimeException();
} catch (Throwable throwable) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Fri Nov 30 19:58:09 2012
@@ -274,7 +274,6 @@ public class TaskAttemptListenerImpl ext
@Override
public boolean ping(TaskAttemptID taskAttemptID) throws IOException {
LOG.info("Ping from " + taskAttemptID.toString());
- taskHeartbeatHandler.pinged(TypeConverter.toYarn(taskAttemptID));
return true;
}
@@ -315,8 +314,6 @@ public class TaskAttemptListenerImpl ext
+ taskStatus.getProgress());
// Task sends the updated state-string to the TT.
taskAttemptStatus.stateString = taskStatus.getStateString();
- // Set the output-size when map-task finishes. Set by the task itself.
- taskAttemptStatus.outputSize = taskStatus.getOutputSize();
// Task sends the updated phase to the TT.
taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
// Counters are updated by the task. Convert counters into new format as
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Fri Nov 30 19:58:09 2012
@@ -184,10 +184,8 @@ class YarnChild {
LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
}
// Report back any failures, for diagnostic purposes
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- exception.printStackTrace(new PrintStream(baos));
if (taskid != null) {
- umbilical.fatalError(taskid, baos.toString());
+ umbilical.fatalError(taskid, StringUtils.stringifyException(exception));
}
} catch (Throwable throwable) {
LOG.fatal("Error running child : "
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Fri Nov 30 19:58:09 2012
@@ -600,6 +600,8 @@ public class JobHistoryEventHandler exte
summary.setJobFinishTime(juce.getFinishTime());
setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters());
break;
+ default:
+ break;
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java Fri Nov 30 19:58:09 2012
@@ -53,7 +53,7 @@ public class JobEndNotifier implements C
protected String userUrl;
protected String proxyConf;
protected int numTries; //Number of tries to attempt notification
- protected int waitInterval; //Time to wait between retrying notification
+ protected int waitInterval; //Time (ms) to wait between retrying notification
protected URL urlToNotify; //URL to notify read from the config
protected Proxy proxyToUse = Proxy.NO_PROXY; //Proxy to use for notification
@@ -71,10 +71,10 @@ public class JobEndNotifier implements C
, conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS, 1)
);
waitInterval = Math.min(
- conf.getInt(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, 5)
- , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, 5)
+ conf.getInt(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, 5000)
+ , conf.getInt(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, 5000)
);
- waitInterval = (waitInterval < 0) ? 5 : waitInterval;
+ waitInterval = (waitInterval < 0) ? 5000 : waitInterval;
userUrl = conf.get(MRJobConfig.MR_JOB_END_NOTIFICATION_URL);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java Fri Nov 30 19:58:09 2012
@@ -46,33 +46,22 @@ import org.apache.hadoop.yarn.service.Ab
public class TaskHeartbeatHandler extends AbstractService {
private static class ReportTime {
- private long lastPing;
private long lastProgress;
public ReportTime(long time) {
setLastProgress(time);
}
- public synchronized void setLastPing(long time) {
- lastPing = time;
- }
-
public synchronized void setLastProgress(long time) {
lastProgress = time;
- lastPing = time;
- }
-
- public synchronized long getLastPing() {
- return lastPing;
}
-
+
public synchronized long getLastProgress() {
return lastProgress;
}
}
private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
- private static final int PING_TIMEOUT = 5 * 60 * 1000;
//thread which runs periodically to see the last time since a heartbeat is
//received from a task.
@@ -127,14 +116,6 @@ public class TaskHeartbeatHandler extend
}
}
- public void pinged(TaskAttemptId attemptID) {
- //only put for the registered attempts
- //TODO throw an exception if the task isn't registered.
- ReportTime time = runningAttempts.get(attemptID);
- if(time != null) {
- time.setLastPing(clock.getTime());
- }
- }
public void register(TaskAttemptId attemptID) {
runningAttempts.put(attemptID, new ReportTime(clock.getTime()));
@@ -159,10 +140,8 @@ public class TaskHeartbeatHandler extend
Map.Entry<TaskAttemptId, ReportTime> entry = iterator.next();
boolean taskTimedOut = (taskTimeOut > 0) &&
(currentTime > (entry.getValue().getLastProgress() + taskTimeOut));
- boolean pingTimedOut =
- (currentTime > (entry.getValue().getLastPing() + PING_TIMEOUT));
-
- if(taskTimedOut || pingTimedOut) {
+
+ if(taskTimedOut) {
// task is lost, remove from the list and raise lost event
iterator.remove();
eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(entry
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java Fri Nov 30 19:58:09 2012
@@ -49,7 +49,6 @@ public class TaskAttemptStatusUpdateEven
public Counters counters;
public String stateString;
public Phase phase;
- public long outputSize;
public List<TaskAttemptId> fetchFailedMaps;
public long mapFinishTime;
public long shuffleFinishTime;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Fri Nov 30 19:58:09 2012
@@ -348,6 +348,9 @@ public class JobImpl implements org.apac
.addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED,
EnumSet.of(JobEventType.JOB_KILL,
JobEventType.JOB_UPDATED_NODES,
+ JobEventType.JOB_TASK_COMPLETED,
+ JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
+ JobEventType.JOB_MAP_TASK_RESCHEDULED,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
// Transitions from KILLED state
@@ -709,7 +712,10 @@ public class JobImpl implements org.apac
* The only entry point to change the Job.
*/
public void handle(JobEvent event) {
- LOG.debug("Processing " + event.getJobId() + " of type " + event.getType());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing " + event.getJobId() + " of type "
+ + event.getType());
+ }
try {
writeLock.lock();
JobStateInternal oldState = getInternalState();
@@ -822,11 +828,15 @@ public class JobImpl implements org.apac
case KILLED:
metrics.killedJob(this);
break;
+ case ERROR:
case FAILED:
metrics.failedJob(this);
break;
case SUCCEEDED:
metrics.completedJob(this);
+ break;
+ default:
+ throw new IllegalArgumentException("Illegal job state: " + finalState);
}
return finalState;
}
@@ -1305,6 +1315,9 @@ public class JobImpl implements org.apac
case REDUCE:
this.finalReduceCounters.incrAllCounters(counters);
break;
+ default:
+ throw new IllegalStateException("Task type neither map nor reduce: " +
+ t.getType());
}
this.fullCounters.incrAllCounters(counters);
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Fri Nov 30 19:58:09 2012
@@ -1335,6 +1335,8 @@ public abstract class TaskAttemptImpl im
taskAttempt.attemptId,
TaskEventType.T_ATTEMPT_KILLED));
break;
+ default:
+ LOG.error("Task final state is not FAILED or KILLED: " + finalState);
}
if (taskAttempt.getLaunchTime() != 0) {
TaskAttemptUnsuccessfulCompletionEvent tauce =
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Fri Nov 30 19:58:09 2012
@@ -22,9 +22,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -118,9 +120,18 @@ public abstract class TaskImpl implement
protected Credentials credentials;
protected Token<JobTokenIdentifier> jobToken;
+ //should be set to one which comes first
+ //saying COMMIT_PENDING
+ private TaskAttemptId commitAttempt;
+
+ private TaskAttemptId successfulAttempt;
+
+ private final Set<TaskAttemptId> failedAttempts;
+ // Track the finished attempts - successful, failed and killed
+ private final Set<TaskAttemptId> finishedAttempts;
// counts the number of attempts that are either running or in a state where
// they will come to be running when they get a Container
- private int numberUncompletedAttempts = 0;
+ private final Set<TaskAttemptId> inProgressAttempts;
private boolean historyTaskStartGenerated = false;
@@ -182,6 +193,14 @@ public abstract class TaskImpl implement
EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED),
TaskEventType.T_ATTEMPT_KILLED,
new KillWaitAttemptKilledTransition())
+ .addTransition(TaskStateInternal.KILL_WAIT,
+ EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED),
+ TaskEventType.T_ATTEMPT_SUCCEEDED,
+ new KillWaitAttemptSucceededTransition())
+ .addTransition(TaskStateInternal.KILL_WAIT,
+ EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED),
+ TaskEventType.T_ATTEMPT_FAILED,
+ new KillWaitAttemptFailedTransition())
// Ignore-able transitions.
.addTransition(
TaskStateInternal.KILL_WAIT,
@@ -189,8 +208,6 @@ public abstract class TaskImpl implement
EnumSet.of(TaskEventType.T_KILL,
TaskEventType.T_ATTEMPT_LAUNCHED,
TaskEventType.T_ATTEMPT_COMMIT_PENDING,
- TaskEventType.T_ATTEMPT_FAILED,
- TaskEventType.T_ATTEMPT_SUCCEEDED,
TaskEventType.T_ADD_SPEC_ATTEMPT))
// Transitions from SUCCEEDED state
@@ -200,13 +217,15 @@ public abstract class TaskImpl implement
.addTransition(TaskStateInternal.SUCCEEDED,
EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED),
TaskEventType.T_ATTEMPT_KILLED, new RetroactiveKilledTransition())
+ .addTransition(TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED,
+ TaskEventType.T_ATTEMPT_SUCCEEDED,
+ new AttemptSucceededAtSucceededTransition())
// Ignore-able transitions.
.addTransition(
TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED,
EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT,
TaskEventType.T_ATTEMPT_COMMIT_PENDING,
TaskEventType.T_ATTEMPT_LAUNCHED,
- TaskEventType.T_ATTEMPT_SUCCEEDED,
TaskEventType.T_KILL))
// Transitions from FAILED state
@@ -242,15 +261,6 @@ public abstract class TaskImpl implement
private static final RecoverdAttemptsComparator RECOVERED_ATTEMPTS_COMPARATOR =
new RecoverdAttemptsComparator();
- //should be set to one which comes first
- //saying COMMIT_PENDING
- private TaskAttemptId commitAttempt;
-
- private TaskAttemptId successfulAttempt;
-
- private int failedAttempts;
- private int finishedAttempts;//finish are total of success, failed and killed
-
@Override
public TaskState getState() {
readLock.lock();
@@ -275,6 +285,9 @@ public abstract class TaskImpl implement
readLock = readWriteLock.readLock();
writeLock = readWriteLock.writeLock();
this.attempts = Collections.emptyMap();
+ this.finishedAttempts = new HashSet<TaskAttemptId>(2);
+ this.failedAttempts = new HashSet<TaskAttemptId>(2);
+ this.inProgressAttempts = new HashSet<TaskAttemptId>(2);
// This overridable method call is okay in a constructor because we
// have a convention that none of the overrides depends on any
// fields that need initialization.
@@ -611,9 +624,9 @@ public abstract class TaskImpl implement
taskAttemptsFromPreviousGeneration.remove(0).getAttemptId().getId();
}
- ++numberUncompletedAttempts;
+ inProgressAttempts.add(attempt.getID());
//schedule the nextAttemptNumber
- if (failedAttempts > 0) {
+ if (failedAttempts.size() > 0) {
eventHandler.handle(new TaskAttemptEvent(attempt.getID(),
TaskAttemptEventType.TA_RESCHEDULE));
} else {
@@ -788,12 +801,14 @@ public abstract class TaskImpl implement
implements SingleArcTransition<TaskImpl, TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
+ TaskTAttemptEvent taskTAttemptEvent = (TaskTAttemptEvent) event;
+ TaskAttemptId taskAttemptId = taskTAttemptEvent.getTaskAttemptID();
task.handleTaskAttemptCompletion(
- ((TaskTAttemptEvent) event).getTaskAttemptID(),
+ taskAttemptId,
TaskAttemptCompletionEventStatus.SUCCEEDED);
- task.finishedAttempts++;
- --task.numberUncompletedAttempts;
- task.successfulAttempt = ((TaskTAttemptEvent) event).getTaskAttemptID();
+ task.finishedAttempts.add(taskAttemptId);
+ task.inProgressAttempts.remove(taskAttemptId);
+ task.successfulAttempt = taskAttemptId;
task.eventHandler.handle(new JobTaskEvent(
task.taskId, TaskState.SUCCEEDED));
LOG.info("Task succeeded with attempt " + task.successfulAttempt);
@@ -824,11 +839,13 @@ public abstract class TaskImpl implement
SingleArcTransition<TaskImpl, TaskEvent> {
@Override
public void transition(TaskImpl task, TaskEvent event) {
+ TaskAttemptId taskAttemptId =
+ ((TaskTAttemptEvent) event).getTaskAttemptID();
task.handleTaskAttemptCompletion(
- ((TaskTAttemptEvent) event).getTaskAttemptID(),
+ taskAttemptId,
TaskAttemptCompletionEventStatus.KILLED);
- task.finishedAttempts++;
- --task.numberUncompletedAttempts;
+ task.finishedAttempts.add(taskAttemptId);
+ task.inProgressAttempts.remove(taskAttemptId);
if (task.successfulAttempt == null) {
task.addAndScheduleAttempt();
}
@@ -840,15 +857,25 @@ public abstract class TaskImpl implement
MultipleArcTransition<TaskImpl, TaskEvent, TaskStateInternal> {
protected TaskStateInternal finalState = TaskStateInternal.KILLED;
+ protected final TaskAttemptCompletionEventStatus taCompletionEventStatus;
+
+ public KillWaitAttemptKilledTransition() {
+ this(TaskAttemptCompletionEventStatus.KILLED);
+ }
+
+ public KillWaitAttemptKilledTransition(
+ TaskAttemptCompletionEventStatus taCompletionEventStatus) {
+ this.taCompletionEventStatus = taCompletionEventStatus;
+ }
@Override
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
- task.handleTaskAttemptCompletion(
- ((TaskTAttemptEvent) event).getTaskAttemptID(),
- TaskAttemptCompletionEventStatus.KILLED);
- task.finishedAttempts++;
+ TaskAttemptId taskAttemptId =
+ ((TaskTAttemptEvent) event).getTaskAttemptID();
+ task.handleTaskAttemptCompletion(taskAttemptId, taCompletionEventStatus);
+ task.finishedAttempts.add(taskAttemptId);
// check whether all attempts are finished
- if (task.finishedAttempts == task.attempts.size()) {
+ if (task.finishedAttempts.size() == task.attempts.size()) {
if (task.historyTaskStartGenerated) {
TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
finalState, null); // TODO JH verify failedAttempt null
@@ -867,43 +894,57 @@ public abstract class TaskImpl implement
}
}
+ private static class KillWaitAttemptSucceededTransition extends
+ KillWaitAttemptKilledTransition {
+ public KillWaitAttemptSucceededTransition() {
+ super(TaskAttemptCompletionEventStatus.SUCCEEDED);
+ }
+ }
+
+ private static class KillWaitAttemptFailedTransition extends
+ KillWaitAttemptKilledTransition {
+ public KillWaitAttemptFailedTransition() {
+ super(TaskAttemptCompletionEventStatus.FAILED);
+ }
+ }
+
private static class AttemptFailedTransition implements
MultipleArcTransition<TaskImpl, TaskEvent, TaskStateInternal> {
@Override
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
- task.failedAttempts++;
TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
- if (castEvent.getTaskAttemptID().equals(task.commitAttempt)) {
+ TaskAttemptId taskAttemptId = castEvent.getTaskAttemptID();
+ task.failedAttempts.add(taskAttemptId);
+ if (taskAttemptId.equals(task.commitAttempt)) {
task.commitAttempt = null;
}
- TaskAttempt attempt = task.attempts.get(castEvent.getTaskAttemptID());
+ TaskAttempt attempt = task.attempts.get(taskAttemptId);
if (attempt.getAssignedContainerMgrAddress() != null) {
//container was assigned
task.eventHandler.handle(new ContainerFailedEvent(attempt.getID(),
attempt.getAssignedContainerMgrAddress()));
}
- task.finishedAttempts++;
- if (task.failedAttempts < task.maxAttempts) {
+ task.finishedAttempts.add(taskAttemptId);
+ if (task.failedAttempts.size() < task.maxAttempts) {
task.handleTaskAttemptCompletion(
- ((TaskTAttemptEvent) event).getTaskAttemptID(),
+ taskAttemptId,
TaskAttemptCompletionEventStatus.FAILED);
// we don't need a new event if we already have a spare
- if (--task.numberUncompletedAttempts == 0
+ task.inProgressAttempts.remove(taskAttemptId);
+ if (task.inProgressAttempts.size() == 0
&& task.successfulAttempt == null) {
task.addAndScheduleAttempt();
}
} else {
task.handleTaskAttemptCompletion(
- ((TaskTAttemptEvent) event).getTaskAttemptID(),
+ taskAttemptId,
TaskAttemptCompletionEventStatus.TIPFAILED);
- TaskTAttemptEvent ev = (TaskTAttemptEvent) event;
- TaskAttemptId taId = ev.getTaskAttemptID();
if (task.historyTaskStartGenerated) {
TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, attempt.getDiagnostics(),
- TaskStateInternal.FAILED, taId);
+ TaskStateInternal.FAILED, taskAttemptId);
task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
taskFailedEvent));
} else {
@@ -927,14 +968,14 @@ public abstract class TaskImpl implement
@Override
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
- if (event instanceof TaskTAttemptEvent) {
- TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
- if (task.getInternalState() == TaskStateInternal.SUCCEEDED &&
- !castEvent.getTaskAttemptID().equals(task.successfulAttempt)) {
- // don't allow a different task attempt to override a previous
- // succeeded state
- return TaskStateInternal.SUCCEEDED;
- }
+ TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
+ if (task.getInternalState() == TaskStateInternal.SUCCEEDED &&
+ !castEvent.getTaskAttemptID().equals(task.successfulAttempt)) {
+ // don't allow a different task attempt to override a previous
+ // succeeded state
+ task.finishedAttempts.add(castEvent.getTaskAttemptID());
+ task.inProgressAttempts.remove(castEvent.getTaskAttemptID());
+ return TaskStateInternal.SUCCEEDED;
}
// a successful REDUCE task should not be overridden
@@ -953,7 +994,7 @@ public abstract class TaskImpl implement
// believe that there's no redundancy.
unSucceed(task);
// fake increase in Uncomplete attempts for super.transition
- ++task.numberUncompletedAttempts;
+ task.inProgressAttempts.add(castEvent.getTaskAttemptID());
return super.transition(task, event);
}
@@ -976,6 +1017,8 @@ public abstract class TaskImpl implement
!attemptId.equals(task.successfulAttempt)) {
// don't allow a different task attempt to override a previous
// succeeded state
+ task.finishedAttempts.add(castEvent.getTaskAttemptID());
+ task.inProgressAttempts.remove(castEvent.getTaskAttemptID());
return TaskStateInternal.SUCCEEDED;
}
}
@@ -1006,6 +1049,16 @@ public abstract class TaskImpl implement
}
}
+ private static class AttemptSucceededAtSucceededTransition
+ implements SingleArcTransition<TaskImpl, TaskEvent> {
+ @Override
+ public void transition(TaskImpl task, TaskEvent event) {
+ TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
+ task.finishedAttempts.add(castEvent.getTaskAttemptID());
+ task.inProgressAttempts.remove(castEvent.getTaskAttemptID());
+ }
+ }
+
private static class KillNewTransition
implements SingleArcTransition<TaskImpl, TaskEvent> {
@Override
@@ -1045,7 +1098,7 @@ public abstract class TaskImpl implement
(attempt, "Task KILL is received. Killing attempt!");
}
- task.numberUncompletedAttempts = 0;
+ task.inProgressAttempts.clear();
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java Fri Nov 30 19:58:09 2012
@@ -19,7 +19,6 @@
package org.apache.hadoop.mapreduce.v2.app.local;
import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -62,7 +61,6 @@ public class LocalContainerAllocator ext
@SuppressWarnings("rawtypes")
private final EventHandler eventHandler;
- private AtomicInteger containerCount = new AtomicInteger();
private long retryInterval;
private long retrystartTime;
private String nmHost;
@@ -102,9 +100,9 @@ public class LocalContainerAllocator ext
this.applicationAttemptId, this.lastResponseID, super
.getApplicationProgress(), new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>());
- AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
AMResponse response;
try {
+ AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
response = allocateResponse.getAMResponse();
// Reset retry count if no exception occurred.
retrystartTime = System.currentTimeMillis();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Fri Nov 30 19:58:09 2012
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -145,6 +146,8 @@ public class RMContainerAllocator extend
BlockingQueue<ContainerAllocatorEvent> eventQueue
= new LinkedBlockingQueue<ContainerAllocatorEvent>();
+ private ScheduleStats scheduleStats = new ScheduleStats();
+
public RMContainerAllocator(ClientService clientService, AppContext context) {
super(clientService, context);
this.stopped = new AtomicBoolean(false);
@@ -208,13 +211,10 @@ public class RMContainerAllocator extend
@Override
protected synchronized void heartbeat() throws Exception {
- LOG.info("Before Scheduling: " + getStat());
+ scheduleStats.updateAndLogIfChanged("Before Scheduling: ");
List<Container> allocatedContainers = getResources();
- LOG.info("After Scheduling: " + getStat());
if (allocatedContainers.size() > 0) {
- LOG.info("Before Assign: " + getStat());
scheduledRequests.assign(allocatedContainers);
- LOG.info("After Assign: " + getStat());
}
int completedMaps = getJob().getCompletedMaps();
@@ -235,6 +235,8 @@ public class RMContainerAllocator extend
maxReduceRampupLimit, reduceSlowStart);
recalculateReduceSchedule = false;
}
+
+ scheduleStats.updateAndLogIfChanged("After Scheduling: ");
}
@Override
@@ -245,7 +247,7 @@ public class RMContainerAllocator extend
}
eventHandlingThread.interrupt();
super.stop();
- LOG.info("Final Stats: " + getStat());
+ scheduleStats.log("Final Stats: ");
}
public boolean getIsReduceStarted() {
@@ -427,7 +429,9 @@ public class RMContainerAllocator extend
return;
}
- LOG.info("Recalculating schedule...");
+ int headRoom = getAvailableResources() != null ?
+ getAvailableResources().getMemory() : 0;
+ LOG.info("Recalculating schedule, headroom=" + headRoom);
//check for slow start
if (!getIsReduceStarted()) {//not set yet
@@ -536,24 +540,6 @@ public class RMContainerAllocator extend
}
}
- /**
- * Synchronized to avoid findbugs warnings
- */
- private synchronized String getStat() {
- return "PendingReduces:" + pendingReduces.size() +
- " ScheduledMaps:" + scheduledRequests.maps.size() +
- " ScheduledReduces:" + scheduledRequests.reduces.size() +
- " AssignedMaps:" + assignedRequests.maps.size() +
- " AssignedReduces:" + assignedRequests.reduces.size() +
- " completedMaps:" + getJob().getCompletedMaps() +
- " completedReduces:" + getJob().getCompletedReduces() +
- " containersAllocated:" + containersAllocated +
- " containersReleased:" + containersReleased +
- " hostLocalAssigned:" + hostLocalAssigned +
- " rackLocalAssigned:" + rackLocalAssigned +
- " availableResources(headroom):" + getAvailableResources();
- }
-
@SuppressWarnings("unchecked")
private List<Container> getResources() throws Exception {
int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
@@ -595,6 +581,9 @@ public class RMContainerAllocator extend
if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
//something changed
recalculateReduceSchedule = true;
+ if (LOG.isDebugEnabled() && headRoom != newHeadRoom) {
+ LOG.debug("headroom=" + newHeadRoom);
+ }
}
if (LOG.isDebugEnabled()) {
@@ -1123,4 +1112,60 @@ public class RMContainerAllocator extend
}
}
}
+
+ private class ScheduleStats {
+ int numPendingReduces;
+ int numScheduledMaps;
+ int numScheduledReduces;
+ int numAssignedMaps;
+ int numAssignedReduces;
+ int numCompletedMaps;
+ int numCompletedReduces;
+ int numContainersAllocated;
+ int numContainersReleased;
+
+ public void updateAndLogIfChanged(String msgPrefix) {
+ boolean changed = false;
+
+ // synchronized to fix findbug warnings
+ synchronized (RMContainerAllocator.this) {
+ changed |= (numPendingReduces != pendingReduces.size());
+ numPendingReduces = pendingReduces.size();
+ changed |= (numScheduledMaps != scheduledRequests.maps.size());
+ numScheduledMaps = scheduledRequests.maps.size();
+ changed |= (numScheduledReduces != scheduledRequests.reduces.size());
+ numScheduledReduces = scheduledRequests.reduces.size();
+ changed |= (numAssignedMaps != assignedRequests.maps.size());
+ numAssignedMaps = assignedRequests.maps.size();
+ changed |= (numAssignedReduces != assignedRequests.reduces.size());
+ numAssignedReduces = assignedRequests.reduces.size();
+ changed |= (numCompletedMaps != getJob().getCompletedMaps());
+ numCompletedMaps = getJob().getCompletedMaps();
+ changed |= (numCompletedReduces != getJob().getCompletedReduces());
+ numCompletedReduces = getJob().getCompletedReduces();
+ changed |= (numContainersAllocated != containersAllocated);
+ numContainersAllocated = containersAllocated;
+ changed |= (numContainersReleased != containersReleased);
+ numContainersReleased = containersReleased;
+ }
+
+ if (changed) {
+ log(msgPrefix);
+ }
+ }
+
+ public void log(String msgPrefix) {
+ LOG.info(msgPrefix + "PendingReds:" + numPendingReduces +
+ " ScheduledMaps:" + numScheduledMaps +
+ " ScheduledReds:" + numScheduledReduces +
+ " AssignedMaps:" + numAssignedMaps +
+ " AssignedReds:" + numAssignedReduces +
+ " CompletedMaps:" + numCompletedMaps +
+ " CompletedReds:" + numCompletedReduces +
+ " ContAlloc:" + numContainersAllocated +
+ " ContRel:" + numContainersReleased +
+ " HostLocal:" + hostLocalAssigned +
+ " RackLocal:" + rackLocalAssigned);
+ }
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java Fri Nov 30 19:58:09 2012
@@ -210,7 +210,7 @@ public abstract class RMContainerRequest
return; //already blacklisted
}
Integer failures = nodeFailures.remove(hostName);
- failures = failures == null ? 0 : failures;
+ failures = failures == null ? Integer.valueOf(0) : failures;
failures++;
LOG.info(failures + " failures on node " + hostName);
if (failures >= maxTaskFailuresPerNode) {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java Fri Nov 30 19:58:09 2012
@@ -43,7 +43,6 @@ abstract class StartEndTimesBase impleme
static final int MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
= 1;
- protected Configuration conf = null;
protected AppContext context = null;
protected final Map<TaskAttemptId, Long> startTimes
@@ -82,7 +81,6 @@ abstract class StartEndTimesBase impleme
@Override
public void contextualize(Configuration conf, AppContext context) {
- this.conf = conf;
this.context = context;
Map<JobId, Job> allJobs = context.getAllJobs();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppView.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppView.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppView.java Fri Nov 30 19:58:09 2012
@@ -35,7 +35,6 @@ public class AppView extends TwoColumnLa
protected void commonPreHead(Page.HTML<_> html) {
set(ACCORDION_ID, "nav");
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:1}");
- set(THEMESWITCHER_ID, "themeswitcher");
}
@Override
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/NavBlock.java Fri Nov 30 19:58:09 2012
@@ -83,7 +83,6 @@ public class NavBlock extends HtmlBlock
li().a("/conf", "Configuration")._().
li().a("/logs", "Local logs")._().
li().a("/stacks", "Server stacks")._().
- li().a("/metrics", "Server metrics")._()._()._().
- div("#themeswitcher")._();
+ li().a("/metrics", "Server metrics")._()._()._();
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java Fri Nov 30 19:58:09 2012
@@ -21,15 +21,13 @@ package org.apache.hadoop.mapreduce.v2.a
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.percent;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@@ -66,6 +64,8 @@ public class TasksBlock extends HtmlBloc
th("Finish Time").
th("Elapsed Time")._()._().
tbody();
+ StringBuilder tasksTableData = new StringBuilder("[\n");
+
for (Task task : app.getJob().getTasks().values()) {
if (type != null && task.getType() != type) {
continue;
@@ -73,31 +73,28 @@ public class TasksBlock extends HtmlBloc
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
String pct = percent(info.getProgress() / 100);
- long startTime = info.getStartTime();
- long finishTime = info.getFinishTime();
- long elapsed = info.getElapsedTime();
- tbody.
- tr().
- td().
- br().$title(String.valueOf(info.getTaskNum()))._(). // sorting
- a(url("task", tid), tid)._().
- td().
- br().$title(pct)._().
- div(_PROGRESSBAR).
- $title(join(pct, '%')). // tooltip
- div(_PROGRESSBAR_VALUE).
- $style(join("width:", pct, '%'))._()._()._().
- td(info.getState()).
- td().
- br().$title(String.valueOf(startTime))._().
- _(Times.format(startTime))._().
- td().
- br().$title(String.valueOf(finishTime))._().
- _(Times.format(finishTime))._().
- td().
- br().$title(String.valueOf(elapsed))._().
- _(StringUtils.formatTime(elapsed))._()._();
+ tasksTableData.append("[\"<a href='").append(url("task", tid))
+ .append("'>").append(tid).append("</a>\",\"")
+ //Progress bar
+ .append("<br title='").append(pct)
+ .append("'> <div class='").append(C_PROGRESSBAR).append("' title='")
+ .append(join(pct, '%')).append("'> ").append("<div class='")
+ .append(C_PROGRESSBAR_VALUE).append("' style='")
+ .append(join("width:", pct, '%')).append("'> </div> </div>\",\"")
+
+ .append(info.getState()).append("\",\"")
+ .append(info.getStartTime()).append("\",\"")
+ .append(info.getFinishTime()).append("\",\"")
+ .append(info.getElapsedTime()).append("\"],\n");
}
+ //Remove the last comma and close off the array of arrays
+ if(tasksTableData.charAt(tasksTableData.length() - 2) == ',') {
+ tasksTableData.delete(tasksTableData.length()-2, tasksTableData.length()-1);
+ }
+ tasksTableData.append("]");
+ html.script().$type("text/javascript").
+ _("var tasksTableData=" + tasksTableData)._();
+
tbody._()._();
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksPage.java Fri Nov 30 19:58:09 2012
@@ -37,11 +37,26 @@ public class TasksPage extends AppView {
}
private String tasksTableInit() {
- return tableInit().
- // Sort by id upon page load
- append(", aaSorting: [[0, 'asc']]").
- append(",aoColumns:[{sType:'title-numeric'},{sType:'title-numeric',").
- append("bSearchable:false},null,{sType:'title-numeric'},").
- append("{sType:'title-numeric'},{sType:'title-numeric'}]}").toString();
+ return tableInit()
+ .append(", 'aaData': tasksTableData")
+ .append(", bDeferRender: true")
+ .append(", bProcessing: true")
+
+ .append("\n, aoColumnDefs: [\n")
+ .append("{'sType':'numeric', 'aTargets': [0]")
+ .append(", 'mRender': parseHadoopID }")
+
+ .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets': [1]")
+ .append(", 'mRender': parseHadoopProgress }")
+
+
+ .append("\n, {'sType':'numeric', 'aTargets': [3, 4]")
+ .append(", 'mRender': renderHadoopDate }")
+
+ .append("\n, {'sType':'numeric', 'aTargets': [5]")
+ .append(", 'mRender': renderHadoopElapsedTime }]")
+
+ // Sort by id upon page load
+ .append(", aaSorting: [[0, 'asc']] }").toString();
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java Fri Nov 30 19:58:09 2012
@@ -285,6 +285,8 @@ public class JobInfo {
case SCHEDULED:
++this.mapsPending;
break;
+ default:
+ break;
}
break;
case REDUCE:
@@ -296,8 +298,13 @@ public class JobInfo {
case SCHEDULED:
++this.reducesPending;
break;
+ default:
+ break;
}
break;
+ default:
+ throw new IllegalStateException(
+ "Task type is neither map nor reduce: " + task.getType());
}
// Attempts counts
Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
@@ -337,6 +344,9 @@ public class JobInfo {
this.failedReduceAttempts += failed;
this.killedReduceAttempts += killed;
break;
+ default:
+ throw new IllegalStateException("Task type neither map nor reduce: " +
+ task.getType());
}
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Fri Nov 30 19:58:09 2012
@@ -54,6 +54,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
+import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
@@ -63,6 +64,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
+import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskImpl;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
@@ -243,6 +245,39 @@ public class MRApp extends MRAppMaster {
return job;
}
+ public void waitForInternalState(JobImpl job,
+ JobStateInternal finalState) throws Exception {
+ int timeoutSecs = 0;
+ JobStateInternal iState = job.getInternalState();
+ while (!finalState.equals(iState) && timeoutSecs++ < 20) {
+ System.out.println("Job Internal State is : " + iState
+ + " Waiting for Internal state : " + finalState);
+ Thread.sleep(500);
+ iState = job.getInternalState();
+ }
+ System.out.println("Task Internal State is : " + iState);
+ Assert.assertEquals("Task Internal state is not correct (timedout)",
+ finalState, iState);
+ }
+
+ public void waitForInternalState(TaskImpl task,
+ TaskStateInternal finalState) throws Exception {
+ int timeoutSecs = 0;
+ TaskReport report = task.getReport();
+ TaskStateInternal iState = task.getInternalState();
+ while (!finalState.equals(iState) && timeoutSecs++ < 20) {
+ System.out.println("Task Internal State is : " + iState
+ + " Waiting for Internal state : " + finalState + " progress : "
+ + report.getProgress());
+ Thread.sleep(500);
+ report = task.getReport();
+ iState = task.getInternalState();
+ }
+ System.out.println("Task Internal State is : " + iState);
+ Assert.assertEquals("Task Internal state is not correct (timedout)",
+ finalState, iState);
+ }
+
public void waitForInternalState(TaskAttemptImpl attempt,
TaskAttemptStateInternal finalState) throws Exception {
int timeoutSecs = 0;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java Fri Nov 30 19:58:09 2012
@@ -412,7 +412,6 @@ public class TestFetchFailure {
status.fetchFailedMaps = new ArrayList<TaskAttemptId>();
status.id = attempt.getID();
status.mapFinishTime = 0;
- status.outputSize = 0;
status.phase = phase;
status.progress = 0.5f;
status.shuffleFinishTime = 0;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java Fri Nov 30 19:58:09 2012
@@ -55,22 +55,22 @@ public class TestJobEndNotifier extends
//Test maximum retry interval is capped by
//MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL
private void testWaitInterval(Configuration conf) {
- conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, "5");
- conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "1");
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, "5000");
+ conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "1000");
setConf(conf);
- Assert.assertTrue("Expected waitInterval to be 1, but was " + waitInterval,
- waitInterval == 1);
+ Assert.assertTrue("Expected waitInterval to be 1000, but was "
+ + waitInterval, waitInterval == 1000);
- conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "10");
+ conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "10000");
setConf(conf);
- Assert.assertTrue("Expected waitInterval to be 5, but was " + waitInterval,
- waitInterval == 5);
+ Assert.assertTrue("Expected waitInterval to be 5000, but was "
+ + waitInterval, waitInterval == 5000);
//Test negative numbers are set to default
conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "-10");
setConf(conf);
- Assert.assertTrue("Expected waitInterval to be 5, but was " + waitInterval,
- waitInterval == 5);
+ Assert.assertTrue("Expected waitInterval to be 5000, but was "
+ + waitInterval, waitInterval == 5000);
}
private void testProxyConfiguration(Configuration conf) {
@@ -125,17 +125,28 @@ public class TestJobEndNotifier extends
public void testNotifyRetries() throws InterruptedException {
Configuration conf = new Configuration();
conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_URL, "http://nonexistent");
+ JobReport jobReport = Mockito.mock(JobReport.class);
+
+ long startTime = System.currentTimeMillis();
+ this.notificationCount = 0;
+ this.setConf(conf);
+ this.notify(jobReport);
+ long endTime = System.currentTimeMillis();
+ Assert.assertEquals("Only 1 try was expected but was : "
+ + this.notificationCount, this.notificationCount, 1);
+ Assert.assertTrue("Should have taken more than 5 seconds it took "
+ + (endTime - startTime), endTime - startTime > 5000);
+
conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS, "3");
conf.set(MRJobConfig.MR_JOB_END_RETRY_ATTEMPTS, "3");
conf.set(MRJobConfig.MR_JOB_END_RETRY_INTERVAL, "3000");
conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL, "3000");
- JobReport jobReport = Mockito.mock(JobReport.class);
- long startTime = System.currentTimeMillis();
+ startTime = System.currentTimeMillis();
this.notificationCount = 0;
this.setConf(conf);
this.notify(jobReport);
- long endTime = System.currentTimeMillis();
+ endTime = System.currentTimeMillis();
Assert.assertEquals("Only 3 retries were expected but was : "
+ this.notificationCount, this.notificationCount, 3);
Assert.assertTrue("Should have taken more than 9 seconds it took "