You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by br...@apache.org on 2013/07/25 03:32:27 UTC
git commit: Changed JT to ignore unhealthy TaskTrackers.
Updated Branches:
refs/heads/master 58e6c9074 -> 5fca4b109
Changed JT to ignore unhealthy TaskTrackers.
Review: https://reviews.apache.org/r/11117
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5fca4b10
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5fca4b10
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5fca4b10
Branch: refs/heads/master
Commit: 5fca4b109b6dd20002f657b0984e3daa77f4b8ed
Parents: 58e6c90
Author: Brenden Matthews <br...@airbnb.com>
Authored: Mon May 13 16:08:13 2013 -0700
Committer: Brenden Matthews <br...@airbnb.com>
Committed: Wed Jul 24 18:32:21 2013 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapred/MesosScheduler.java | 35 ++++++++++++--------
1 file changed, 22 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5fca4b10/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
index c0d0fa3..d9a9dbb 100644
--- a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
+++ b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
@@ -305,8 +305,8 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
public void resourceOffers(SchedulerDriver schedulerDriver,
List<Offer> offers) {
// Before synchronizing, we pull all needed information from the JobTracker.
- final HttpHost jobTrackerAddress = new HttpHost(jobTracker.getHostname(),
- jobTracker.getTrackerPort());
+ final HttpHost jobTrackerAddress =
+ new HttpHost(jobTracker.getHostname(), jobTracker.getTrackerPort());
final Collection<TaskTrackerStatus> taskTrackers = jobTracker.taskTrackers();
@@ -329,7 +329,15 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
// Mark active (heartbeated) TaskTrackers and compute idle slots.
int idleMapSlots = 0;
int idleReduceSlots = 0;
+ int unhealthyTrackers = 0;
+
for (TaskTrackerStatus status : taskTrackers) {
+ if (!status.getHealthStatus().isNodeHealthy()) {
+ // Skip this node if it's unhealthy.
+ ++unhealthyTrackers;
+ continue;
+ }
+
HttpHost host = new HttpHost(status.getHost(), status.getHttpPort());
if (mesosTrackers.containsKey(host)) {
mesosTrackers.get(host).active = true;
@@ -354,17 +362,18 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
int neededReduceSlots = Math.max(0, pendingReduces - idleReduceSlots);
LOG.info(join("\n", Arrays.asList(
- "JobTracker Status",
- " Pending Map Tasks: " + pendingMaps,
- " Pending Reduce Tasks: " + pendingReduces,
- " Idle Map Slots: " + idleMapSlots,
- " Idle Reduce Slots: " + idleReduceSlots,
- " Inactive Map Slots: " + inactiveMapSlots
- + " (launched but no hearbeat yet)",
- " Inactive Reduce Slots: " + inactiveReduceSlots
- + " (launched but no hearbeat yet)",
- " Needed Map Slots: " + neededMapSlots,
- " Needed Reduce Slots: " + neededReduceSlots)));
+ "JobTracker Status",
+ " Pending Map Tasks: " + pendingMaps,
+ " Pending Reduce Tasks: " + pendingReduces,
+ " Idle Map Slots: " + idleMapSlots,
+ " Idle Reduce Slots: " + idleReduceSlots,
+ " Inactive Map Slots: " + inactiveMapSlots
+ + " (launched but no hearbeat yet)",
+ " Inactive Reduce Slots: " + inactiveReduceSlots
+ + " (launched but no hearbeat yet)",
+ " Needed Map Slots: " + neededMapSlots,
+ " Needed Reduce Slots: " + neededReduceSlots,
+ " Unhealthy Trackers: " + unhealthyTrackers)));
// Launch TaskTrackers to satisfy the slot requirements.
// TODO(bmahler): Consider slotting intelligently.