You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by jl...@apache.org on 2017/10/12 14:35:02 UTC
tez git commit: TEZ-3852. Optimize ContainerContext.isSuperSet to
speed container reuse decisions. Contributed by Jonathan Eagles
Repository: tez
Updated Branches:
refs/heads/master c82b2eade -> 0207281b3
TEZ-3852. Optimize ContainerContext.isSuperSet to speed container reuse decisions. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/0207281b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/0207281b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/0207281b
Branch: refs/heads/master
Commit: 0207281b3cea4f78b5a57fb81fef0cf14d8c24e1
Parents: c82b2ea
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 12 09:33:14 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 12 09:33:14 2017 -0500
----------------------------------------------------------------------
.../apache/tez/dag/app/ContainerContext.java | 52 ++++++++------------
.../tez/dag/app/rm/TestContainerReuse.java | 4 ++
2 files changed, 24 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
index f00b27b..d88daff 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java
@@ -129,26 +129,24 @@ public class ContainerContext {
// classpath modification
private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
Map<String, LocalResource> reqLRs) {
- Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
- for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
- LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
- if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
+ for (Entry<String, LocalResource> reqLREntry : reqLRs.entrySet()) {
+ LocalResource requestedLocalResource = srcLRs.get(reqLREntry.getKey());
+ if (requestedLocalResource == null) {
+ LocalResource lr = reqLREntry.getValue();
+ if (!LocalResourceType.FILE.equals(lr.getType())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot match container: Additional local resource needed is not of type FILE"
+ + ", resourceName: " + reqLREntry.getKey()
+ + ", resourceDetails: " + reqLREntry);
+ }
+ return false;
+ }
+ } else if(!reqLREntry.getValue().equals(requestedLocalResource)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Cannot match container: Attempting to use same target resource name: "
- + srcLREntry.getKey()
+ + reqLREntry.getKey()
+ ", but with different source resources. Already localized: "
- + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
- }
- return false;
- }
- }
- for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
- LocalResource lr = additionalLREntry.getValue();
- if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cannot match container: Additional local resource needed is not of type FILE"
- + ", resourceName: " + additionalLREntry.getKey()
- + ", resourceDetails: " + additionalLREntry);
+ + requestedLocalResource + ", requested: " + reqLREntry.getValue());
}
return false;
}
@@ -161,24 +159,14 @@ public class ContainerContext {
for (Entry<K, V> oEntry : matchMap.entrySet()) {
K oKey = oEntry.getKey();
V oVal = oEntry.getValue();
- if (srcMap.containsKey(oKey)) {
- if (!oVal.equals(srcMap.get(oKey))) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Incompatible container context"
+ V srcVal = srcMap.get(oKey);
+ if (!oVal.equals(srcVal)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Incompatible container context"
+ ", matchInfo=" + matchInfo
+ ", thisKey=" + oKey
- + ", thisVal=" + srcMap.get(oKey)
+ + ", thisVal=" + srcVal
+ ", otherVal=" + oVal);
- }
- return false;
- }
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Incompatible container context"
- + ", matchInfo=" + matchInfo
- + ", thisKey=" + oKey
- + ", thisVal=null"
- + ", otherVal=" + oVal);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/0207281b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 7e9e9ab..4e29dd5 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -38,6 +38,7 @@ import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.tez.common.TezUtils;
import org.apache.tez.serviceplugins.api.TaskScheduler;
import org.mockito.ArgumentCaptor;
@@ -1127,8 +1128,11 @@ public class TestContainerReuse {
String rsrc1 = "rsrc1";
String rsrc2 = "rsrc2";
LocalResource lr1 = mock(LocalResource.class);
+ doReturn(LocalResourceType.FILE).when(lr1).getType();
LocalResource lr2 = mock(LocalResource.class);
+ doReturn(LocalResourceType.FILE).when(lr2).getType();
LocalResource lr3 = mock(LocalResource.class);
+ doReturn(LocalResourceType.FILE).when(lr3).getType();
AMContainerEventAssignTA assignEvent = null;