You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2013/06/17 01:03:51 UTC
git commit: TEZ-245. YARN-787 build break fix (bikas)
Updated Branches:
refs/heads/master f8b47f19a -> e4fd8c312
TEZ-245. YARN-787 build break fix (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e4fd8c31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e4fd8c31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e4fd8c31
Branch: refs/heads/master
Commit: e4fd8c3124806d8cc60dd35926bc8e5cd905a2f9
Parents: f8b47f1
Author: Bikas Saha <bi...@apache.org>
Authored: Sun Jun 16 16:00:07 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Sun Jun 16 16:00:07 2013 -0700
----------------------------------------------------------------------
.../tez/dag/app/dag/impl/TaskAttemptImpl.java | 50 ++++++++++----------
.../apache/tez/dag/app/rm/TaskScheduler.java | 3 --
.../dag/app/rm/TaskSchedulerEventHandler.java | 31 +-----------
.../tez/dag/app/rm/TestTaskScheduler.java | 6 +--
4 files changed, 27 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 0eddbfb..b7ba7b8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -625,8 +625,8 @@ public class TaskAttemptImpl implements TaskAttempt,
ta.getDAGID()
);
- long slotMillis = computeSlotMillis(ta);
- jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillis);
+// long slotMillis = computeSlotMillis(ta);
+// jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillis);
return jce;
}
@@ -637,38 +637,38 @@ public class TaskAttemptImpl implements TaskAttempt,
new DAGEventCounterUpdate(
taskAttempt.getDAGID());
- long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
if (taState == TaskAttemptStateInternal.FAILED) {
jce.addCounterUpdate(DAGCounter.NUM_FAILED_TASKS, 1);
} else if (taState == TaskAttemptStateInternal.KILLED) {
jce.addCounterUpdate(DAGCounter.NUM_KILLED_TASKS, 1);
}
- if (!taskAlreadyCompleted) {
- // dont double count the elapsed time
- jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillisIncrement);
- }
+
+// long slotMillisIncrement = computeSlotMillis(taskAttempt);
+// if (!taskAlreadyCompleted) {
+// // dont double count the elapsed time
+// jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillisIncrement);
+// }
return jce;
}
- private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
- int slotMemoryReq =
- taskAttempt.taskResource.getMemory();
-
- int minSlotMemSize =
- taskAttempt.appContext.getClusterInfo().getMinContainerCapability()
- .getMemory();
-
- int simSlotsRequired =
- minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
- / minSlotMemSize);
-
- long slotMillisIncrement =
- simSlotsRequired
- * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
- return slotMillisIncrement;
- }
+// private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
+// int slotMemoryReq =
+// taskAttempt.taskResource.getMemory();
+//
+// int minSlotMemSize =
+// taskAttempt.appContext.getClusterInfo().getMinContainerCapability()
+// .getMemory();
+//
+// int simSlotsRequired =
+// minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
+// / minSlotMemSize);
+//
+// long slotMillisIncrement =
+// simSlotsRequired
+// * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
+// return slotMillisIncrement;
+// }
// TODO: JobHistory
// TODO Change to return a JobHistoryEvent.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 95120c4..6068919 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.client.AMRMClientAsync;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.RackResolver;
-import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
import com.google.common.annotations.VisibleForTesting;
@@ -90,7 +89,6 @@ public class TaskScheduler extends AbstractService
public void nodesUpdated(List<NodeReport> updatedNodes);
public void appShutdownRequested();
public void setApplicationRegistrationData(
- Resource minContainerCapability,
Resource maxContainerCapability,
Map<ApplicationAccessType, String> appAcls
);
@@ -191,7 +189,6 @@ public class TaskScheduler extends AbstractService
}
// upcall to app outside locks
appClient.setApplicationRegistrationData(
- response.getMinimumResourceCapability(),
response.getMaximumResourceCapability(),
response.getApplicationACLs());
} catch (YarnException e) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index fe128b6..e4553dd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -269,33 +269,6 @@ public class TaskSchedulerEventHandler extends AbstractService
}
}
- /*
- @SuppressWarnings("unchecked")
- private int maybeComputeNormalizedRequestForType(
- AMSchedulerTALaunchRequestEvent event, int prevComputedSize) {
- if (prevComputedSize == 0) {
- int supportedMaxContainerCapability = appContext.getClusterInfo()
- .getMaxContainerCapability().getMemory();
- prevComputedSize = event.getCapability().getMemory();
- int minSlotMemSize = appContext.getClusterInfo()
- .getMinContainerCapability().getMemory();
- prevComputedSize = (int) Math.ceil((float) prevComputedSize
- / minSlotMemSize)
- * minSlotMemSize;
- if (prevComputedSize > supportedMaxContainerCapability) {
- String diagMsg = " capability required is more than the supported "
- + "max container capability in the cluster. Killing the Job. "
- + "ResourceReqt: " + prevComputedSize
- + " maxContainerCapability:" + supportedMaxContainerCapability;
- LOG.info(diagMsg);
- eventHandler.handle(new DAGDiagnosticsUpdateEvent(job.getID(), diagMsg));
- eventHandler.handle(new DAGEvent(job.getID(), DAGEventType.JOB_KILL));
- }
- }
- return prevComputedSize;
- }
- */
-
private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
/**
// Add to queue of pending tasks.
@@ -490,11 +463,9 @@ public class TaskSchedulerEventHandler extends AbstractService
}
@Override
- public synchronized void setApplicationRegistrationData(Resource minContainerCapability,
+ public synchronized void setApplicationRegistrationData(
Resource maxContainerCapability,
Map<ApplicationAccessType, String> appAcls) {
- this.appContext.getClusterInfo().setMinContainerCapability(
- minContainerCapability);
this.appContext.getClusterInfo().setMaxContainerCapability(
maxContainerCapability);
this.appAcls = appAcls;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 5f66cf2..ebc0916 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -82,11 +82,8 @@ public class TestTaskScheduler {
RegisterApplicationMasterResponse mockRegResponse =
mock(RegisterApplicationMasterResponse.class);
- Resource mockMinResource = mock(Resource.class);
Resource mockMaxResource = mock(Resource.class);
Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
- when(mockRegResponse.getMinimumResourceCapability()).
- thenReturn(mockMinResource);
when(mockRegResponse.getMaximumResourceCapability()).
thenReturn(mockMaxResource);
when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
@@ -96,8 +93,7 @@ public class TestTaskScheduler {
scheduler.start();
verify(mockRMClient).start();
verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl);
- verify(mockApp).setApplicationRegistrationData(mockMinResource,
- mockMaxResource,
+ verify(mockApp).setApplicationRegistrationData(mockMaxResource,
mockAcls);
when(mockRMClient.getClusterNodeCount()).thenReturn(5);