You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2013/11/20 02:58:55 UTC
git commit: TEZ-221. Deadlock in DAGSchedulerMRR when setting slow
start to 0.5f on a small cluster (bikas)
Updated Branches:
refs/heads/master 207264823 -> 808be4d04
TEZ-221. Deadlock in DAGSchedulerMRR when setting slow start to 0.5f on a small cluster (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/808be4d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/808be4d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/808be4d0
Branch: refs/heads/master
Commit: 808be4d0476eea14114096953dd46b06a8d74233
Parents: 2072648
Author: Bikas Saha <bi...@apache.org>
Authored: Tue Nov 19 17:55:25 2013 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Tue Nov 19 17:55:25 2013 -0800
----------------------------------------------------------------------
.../apache/tez/dag/api/TezConfiguration.java | 4 --
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 43 +-------------------
.../dag/app/dag/impl/ShuffleVertexManager.java | 7 ----
.../tez/dag/app/dag/impl/TestDAGImpl.java | 1 +
4 files changed, 3 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 6392ded..c41a9ea 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -136,10 +136,6 @@ public class TezConfiguration extends Configuration {
public static final float
TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION_DEFAULT = 0.5f;
- public static final String TEZ_AM_AGGRESSIVE_SCHEDULING = TEZ_AM_PREFIX +
- "aggressive.scheduling";
- public static boolean TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT = false;
-
/**
* The complete path to the serialized dag plan file
* <code>TEZ_AM_PLAN_PB_BINARY</code>. Used to make the plan available to
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index ee6b832..9fb81e0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.client.DAGStatus;
@@ -881,46 +880,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
private void assignDAGScheduler(DAGImpl dag) {
- if (dag.conf.getBoolean(TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING,
- TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT)) {
- LOG.info("Using Natural order dag scheduler due to aggressive scheduling");
- dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
- } else {
- boolean isMRR = true;
- for (Vertex vertex : dag.vertices.values()) {
- Map<Vertex, Edge> outVertices = vertex.getOutputVertices();
- Map<Vertex, Edge> inVertices = vertex.getInputVertices();
- if (!(outVertices == null || outVertices.isEmpty() || (outVertices
- .size() == 1 && outVertices.values().iterator().next().getEdgeProperty()
- .getDataMovementType() == EdgeProperty.DataMovementType.SCATTER_GATHER))) {
- // more than 1 output OR single output is not bipartite
- isMRR = false;
- break;
- }
- if (!(inVertices == null || inVertices.isEmpty() || (inVertices
- .size() == 1 && inVertices.values().iterator().next().getEdgeProperty()
- .getDataMovementType() == EdgeProperty.DataMovementType.SCATTER_GATHER))) {
- // more than 1 output OR single output is not bipartite
- isMRR = false;
- break;
- }
- }
-
- if (isMRR) {
- LOG.info("Using MRR dag scheduler");
- dag.dagScheduler = new DAGSchedulerMRR(
- dag,
- dag.eventHandler,
- dag.appContext.getTaskScheduler(),
- dag.conf
- .getFloat(
- TezConfiguration.TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION,
- TezConfiguration.TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION_DEFAULT));
- } else {
- LOG.info("Using Natural order dag scheduler");
- dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
- }
- }
+ LOG.info("Using Natural order dag scheduler");
+ dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
}
private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
index 5143868..eda2dd5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
@@ -414,13 +414,6 @@ public class ShuffleVertexManager implements VertexScheduler {
"max cannot be < min.");
}
- if (conf.getBoolean(TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING,
- TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT)) {
- LOG.info("Setting min/max threshold to 0 due to aggressive scheduling");
- this.slowStartMinSrcCompletionFraction = 0;
- this.slowStartMaxSrcCompletionFraction = 0;
- }
-
enableAutoParallelism = conf
.getBoolean(
TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 55b846d..b0ad929 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -844,6 +844,7 @@ public class TestDAGImpl {
// FIXME need to implement
}
+ @Ignore
@SuppressWarnings("unchecked")
@Test
public void testOutofBandFailureForMRRScheduler() {