You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/04/15 22:39:36 UTC
svn commit: r1092803 - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
mr-client/had...
Author: mahadev
Date: Fri Apr 15 20:39:35 2011
New Revision: 1092803
URL: http://svn.apache.org/viewvc?rev=1092803&view=rev
Log:
MAPREDUCE-2424. Polish uber-AppMaster: add uber-AM counters and GUI indicators. (Greg Roelofs via mahadev)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Apr 15 20:39:35 2011
@@ -16,6 +16,13 @@ Trunk (unreleased changes)
Fix YarnRemoteException to give more details. (siddharth seth via mahadev)
+ MAPREDUCE-2405. Implement uber-AppMaster (in-cluster LocalJobRunner for
+ MRv2). (Greg Roelofs and Sharad Agarwal via mahadev)
+
+ MAPREDUCE-2424. Polish uber-AppMaster: add uber-AM counters and GUI
+ indicators. (Greg Roelofs via mahadev)
+
+
INCOMPATIBLE CHANGES
MAPREDUCE-1866. Removes deprecated class
@@ -431,7 +438,7 @@ Release 0.21.0 - 2010-08-13
NEW FEATURES
MAPREDUCE-1774. Large-scale Automated Framework (Sharad Agarwal, Sreekanth
- Ramakrishnan, Konstantin Boudnik, at all via cos)
+ Ramakrishnan, Konstantin Boudnik, et al. via cos)
MAPREDUCE-706. Support for FIFO pools in the fair scheduler.
(Matei Zaharia)
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java Fri Apr 15 20:39:35 2011
@@ -138,7 +138,7 @@ public class LocalContainerLauncher exte
* - creates "remoteTask" for us == real Task
* - sends CONTAINER_REMOTE_LAUNCH
* - TA: UNASSIGNED -> ASSIGNED
- * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us!)
+ * - CONTAINER_REMOTE_LAUNCH handled by LocalContainerLauncher (us)
* - sucks "remoteTask" out of TaskAttemptImpl via getRemoteTask()
* - sends TA_CONTAINER_LAUNCHED
* [[ elsewhere...
@@ -251,6 +251,13 @@ public class LocalContainerLauncher exte
try {
JobConf conf = new JobConf(getConfig());
+ // mark this as an uberized subtask so it can set task counter
+ // (longer-term/FIXME: could redefine as job counter and send
+ // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
+ // will need new Job state-machine transition and JobImpl jobCounters
+ // map to handle)
+ conf.setBoolean("mapreduce.task.uberized", true);
+
// META-FIXME: do we want the extra sanity-checking (doneWithMaps,
// etc.), or just assume/hope the state machine(s) and uber-AM work
// as expected?
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Fri Apr 15 20:39:35 2011
@@ -122,7 +122,8 @@ public class JobImpl implements org.apac
// Maximum no. of fetch-failure notifications after which map task is failed
private static final int MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
- private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
//final fields
private final Clock clock;
@@ -131,14 +132,25 @@ public class JobImpl implements org.apac
private final JobId jobId;
private final org.apache.hadoop.mapreduce.JobID oldJobId;
private final TaskAttemptListener taskAttemptListener;
- private boolean lazyTasksCopyNeeded = false;
private final Object tasksSyncHandle = new Object();
- private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
private final Set<TaskId> mapTasks = new LinkedHashSet<TaskId>();
private final Set<TaskId> reduceTasks = new LinkedHashSet<TaskId>();
private final EventHandler eventHandler;
+
+ private boolean lazyTasksCopyNeeded = false;
+ private volatile Map<TaskId, Task> tasks = new LinkedHashMap<TaskId, Task>();
+ private Counters jobCounters = newCounters();
+ // FIXME: support job-level counters
+ //
+ // Presumably want to define new event type that job-related entities
+ // (e.g., MRAppMaster or LocalContainerLauncher) can emit with some sort
+ // of payload (maybe just Counters?); then define new Job state-machine
+ // transition to handle the event and update jobCounters with payload data.
+ // Can then replace task-level uber counters (MR-2424) with job-level ones
+ // sent from LocalContainerLauncher, and eventually including a count of
+ // of uber-AM attempts (probably sent from MRAppMaster).
public Configuration conf;
-
+
//fields initialized in init
private FileSystem fs;
private Path remoteJobSubmitDir;
@@ -404,9 +416,10 @@ public class JobImpl implements org.apac
@Override
public Counters getCounters() {
Counters counters = newCounters();
- // TODO: compute job level counters
+ // TODO: compute job-level counters
readLock.lock();
try {
+ incrAllCounters(counters, jobCounters);
return incrTaskCounters(counters, tasks.values());
} finally {
readLock.unlock();
@@ -414,7 +427,8 @@ public class JobImpl implements org.apac
}
public static Counters newCounters() {
- Counters counters = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Counters.class);
+ Counters counters = RecordFactoryProvider.getRecordFactory(null)
+ .newRecordInstance(Counters.class);
// counters.groups = new HashMap<String, CounterGroup>();
return counters;
}
@@ -432,7 +446,8 @@ public class JobImpl implements org.apac
for (CounterGroup otherGroup: other.getAllCounterGroups().values()) {
CounterGroup group = counters.getCounterGroup(otherGroup.getName());
if (group == null) {
- group = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(CounterGroup.class);
+ group = RecordFactoryProvider.getRecordFactory(null)
+ .newRecordInstance(CounterGroup.class);
// group.counters = new HashMap<CharSequence, Counter>();
group.setName(otherGroup.getName());
counters.setCounterGroup(group.getName(), group);
@@ -441,7 +456,8 @@ public class JobImpl implements org.apac
for (Counter otherCounter : otherGroup.getAllCounters().values()) {
Counter counter = group.getCounter(otherCounter.getName());
if (counter == null) {
- counter = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Counter.class);
+ counter = RecordFactoryProvider.getRecordFactory(null)
+ .newRecordInstance(Counter.class);
counter.setName(otherCounter.getName());
group.setCounter(counter.getName(), counter);
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java Fri Apr 15 20:39:35 2011
@@ -72,6 +72,7 @@ public class JobBlock extends HtmlBlock
info("Job Overview").
_("Job Name:", job.getName()).
_("State:", job.getState()).
+ _("Uberized:", job.isUber()).
_("Started:", new Date(jobReport.getStartTime())).
_("Elapsed:", StringUtils.formatTime(System.currentTimeMillis()
- jobReport.getStartTime()));
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java Fri Apr 15 20:39:35 2011
@@ -325,6 +325,11 @@ class MapTask extends Task {
boolean useNewApi = job.getUseNewMapper();
initialize(job, getJobID(), reporter, useNewApi);
+ if (job.getBoolean("mapreduce.task.uberized", false)) {
+ reporter.getCounter(TaskCounter.NUM_UBER_AM_MAPS).increment(1);
+ // or getCounters().findCounter( ... ).increment(1);
+ }
+
// check if it is a cleanupJobTask
if (jobCleanup) {
runJobCleanupTask(umbilical, reporter);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Fri Apr 15 20:39:35 2011
@@ -367,6 +367,11 @@ public class ReduceTask extends Task {
boolean useNewApi = job.getUseNewReducer();
initialize(job, getJobID(), reporter, useNewApi);
+ if (job.getBoolean("mapreduce.task.uberized", false)) {
+ reporter.getCounter(TaskCounter.NUM_UBER_AM_REDUCES).increment(1);
+ // or getCounters().findCounter( ... ).increment(1);
+ }
+
// check if it is a cleanupJobTask
if (jobCleanup) {
runJobCleanupTask(umbilical, reporter);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.java Fri Apr 15 20:39:35 2011
@@ -42,6 +42,8 @@ public enum TaskCounter {
SHUFFLED_MAPS,
FAILED_SHUFFLE,
MERGED_MAP_OUTPUTS,
+ NUM_UBER_AM_MAPS,
+ NUM_UBER_AM_REDUCES,
GC_TIME_MILLIS,
CPU_MILLISECONDS,
PHYSICAL_MEMORY_BYTES,
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCounter.properties Fri Apr 15 20:39:35 2011
@@ -31,6 +31,8 @@ SPILLED_RECORDS.name= Spilled R
SHUFFLED_MAPS.name= Shuffled Maps
FAILED_SHUFFLE.name= Failed Shuffles
MERGED_MAP_OUTPUTS.name= Merged Map outputs
+NUM_UBER_AM_MAPS.name= Uberized sub-MapTasks
+NUM_UBER_AM_REDUCES.name= Uberized sub-ReduceTasks
GC_TIME_MILLIS.name= GC time elapsed (ms)
CPU_MILLISECONDS.name= CPU time spent (ms)
PHYSICAL_MEMORY_BYTES.name= Physical memory (bytes) snapshot
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1092803&r1=1092802&r2=1092803&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Fri Apr 15 20:39:35 2011
@@ -265,7 +265,7 @@ public class YARNRunner implements Clien
appContext.setApplicationId(applicationId);
Resource capability = recordFactory.newRecordInstance(Resource.class);
capability.setMemory(conf.getInt(YARN_AM_VMEM_MB, DEFAULT_YARN_AM_VMEM_MB));
- LOG.info("Master capability = " + capability);
+ LOG.info("AppMaster capability = " + capability);
appContext.setMasterCapability(capability);
FileContext defaultFS = FileContext.getFileContext(conf);