You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2016/10/07 16:32:43 UTC
tez git commit: TEZ-3368. NPE in DelayedContainerManager (Jason Lowe
via jeagles)
Repository: tez
Updated Branches:
refs/heads/master 149db1b48 -> 5bd5613fc
TEZ-3368. NPE in DelayedContainerManager (Jason Lowe via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5bd5613f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5bd5613f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5bd5613f
Branch: refs/heads/master
Commit: 5bd5613fca24bb521f0d8a9dd2cace4f80c5e90a
Parents: 149db1b
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Fri Oct 7 11:32:17 2016 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Fri Oct 7 11:32:17 2016 -0500
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../apache/tez/dag/app/rm/TezAMRMClientAsync.java | 17 +++++++----------
.../tez/dag/app/rm/YarnTaskSchedulerService.java | 9 +++++++++
3 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c85bc8e..4167747 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3368. NPE in DelayedContainerManager
TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs
TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly.
TEZ-3000. Fix TestContainerReuse.
@@ -115,6 +116,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3368. NPE in DelayedContainerManager
TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs
TEZ-3429. Set reconfigureDoneTime on VertexConfigurationDoneEvent properly.
TEZ-3000. Fix TestContainerReuse.
@@ -606,6 +608,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3368. NPE in DelayedContainerManager
TEZ-3440. Shuffling to memory can get out-of-sync when fetching multiple compressed map outputs
TEZ-3426. Second AM attempt launched for session mode and recovery disabled for certain cases
TEZ-2852. TestVertexImpl fails due to race in AsyncDispatcher.
http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
index 7b3df84..40bc3aa 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java
@@ -20,8 +20,8 @@ package org.apache.tez.dag.app.rm;
import java.util.Collection;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -74,12 +74,10 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs
}
public synchronized Priority getTopPriority() {
- Iterator<Priority> iter =
- knownRequestsByPriority.descendingKeySet().iterator();
- if (!iter.hasNext()) {
+ if (knownRequestsByPriority.isEmpty()) {
return null;
}
- return iter.next();
+ return knownRequestsByPriority.lastKey();
}
// Remove after YARN-1723 is fixed
@@ -132,13 +130,12 @@ public class TezAMRMClientAsync<T extends ContainerRequest> extends AMRMClientAs
String resourceName, Resource capability) {
// Sort based on reverse order. By default, Priority ordering is based on
// highest numeric value being considered to be lowest priority.
- Iterator<Priority> iter =
- knownRequestsByPriority.descendingKeySet().iterator();
- if (!iter.hasNext()) {
+ Map.Entry<Priority,LocalityRequestCounter> entry = knownRequestsByPriority.lastEntry();
+ if (entry == null || entry.getValue() == null) {
return Collections.emptyList();
}
- Priority p = iter.next();
- LocalityRequestCounter lrc = knownRequestsByPriority.get(p);
+ Priority p = entry.getKey();
+ LocalityRequestCounter lrc = entry.getValue();
if (lrc.localityRequests.get() == 0) {
// Fallback to ANY if there are no pending requests that require
// locality matching
http://git-wip-us.apache.org/repos/asf/tez/blob/5bd5613f/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
index 6dfc7b9..5087d0d 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java
@@ -47,6 +47,7 @@ import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.math3.random.RandomDataGenerator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -1958,6 +1959,14 @@ public class YarnTaskSchedulerService extends TaskScheduler
@Override
public void run() {
+ try {
+ mainLoop();
+ } catch (Throwable e) {
+ ExitUtil.terminate(1, e);
+ }
+ }
+
+ private void mainLoop() {
while(running) {
// Try assigning all containers if there's a request to do so.
if (tryAssigningAll) {