You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2014/02/05 22:31:50 UTC
git commit: TEZ-745. Rename TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE
in TezConfiguration to TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS (Jonathan
Eagles via bikas)
Updated Branches:
refs/heads/master bd5624179 -> 2c5343998
TEZ-745. Rename TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE in TezConfiguration to TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS (Jonathan Eagles via bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2c534399
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2c534399
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2c534399
Branch: refs/heads/master
Commit: 2c53439981f5d2d2b8ce8543a6f219fae987e00a
Parents: bd56241
Author: Bikas Saha <bi...@apache.org>
Authored: Wed Feb 5 13:31:42 2014 -0800
Committer: Bikas Saha <bi...@apache.org>
Committed: Wed Feb 5 13:31:42 2014 -0800
----------------------------------------------------------------------
.../apache/tez/dag/api/TezConfiguration.java | 6 +++---
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 20 ++++++++++----------
.../tez/dag/app/dag/impl/TestDAGImpl.java | 2 +-
3 files changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index ba80854..1616adc 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -52,9 +52,9 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX+"log.level";
public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
- public static final String TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE =
- TEZ_AM_PREFIX + "abort-all-outputs-on-dag-failure";
- public static final boolean TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE_DEFAULT = true;
+ public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS =
+ TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success";
+ public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;
/** Java options for the Tez AppMaster process. */
public static final String TEZ_AM_JAVA_OPTS = TEZ_AM_PREFIX
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 1c558d1..85f102c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -133,7 +133,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private volatile boolean committedOrAborted = false;
private volatile boolean allOutputsCommitted = false;
- boolean abortAllOutputsOnFailure = true;
+ boolean commitAllOutputsOnSuccess = true;
@VisibleForTesting
DAGScheduler dagScheduler;
@@ -662,7 +662,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
LOG.info("Calling DAG commit/abort for dag: " + getID());
this.committedOrAborted = true;
- boolean successfulOutputsAlreadyCommitted = !abortAllOutputsOnFailure;
+ boolean successfulOutputsAlreadyCommitted = !commitAllOutputsOnSuccess;
boolean failedWhileCommitting = false;
if (dagSucceeded && !successfulOutputsAlreadyCommitted) {
// commit all shared outputs
@@ -743,8 +743,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
for (Map.Entry<String, OutputCommitter> entry : outputCommitters
.entrySet()) {
final OutputCommitter committer = entry.getValue();
- if (abortAllOutputsOnFailure // abort all outputs on failure
- || vertex.getState() != VertexState.SUCCEEDED // always abort non-successful outputs
+ if (commitAllOutputsOnSuccess // commit all outputs on success
+ || vertex.getState() != VertexState.SUCCEEDED // never commit unsuccessful outputs
) {
LOG.info("Aborting output: " + entry.getKey() + " for vertex: "
+ vertex.getVertexId());
@@ -1067,9 +1067,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
//dag.metrics.preparingJob(dag);
dag.initTime = dag.clock.getTime();
- dag.abortAllOutputsOnFailure = dag.conf.getBoolean(
- TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE,
- TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE_DEFAULT);
+ dag.commitAllOutputsOnSuccess = dag.conf.getBoolean(
+ TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+ TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT);
// If we have no vertices, fail the dag
dag.numVertices = dag.getJobPlan().getVertexCount();
@@ -1179,7 +1179,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
vertexId, vertexPlan, vertexName, dag.conf,
dag.eventHandler, dag.taskAttemptListener,
dag.clock, dag.taskHeartbeatHandler,
- !dag.abortAllOutputsOnFailure, dag.appContext, vertexLocationHint,
+ !dag.commitAllOutputsOnSuccess, dag.appContext, vertexLocationHint,
dag.vertexGroups);
return v;
}
@@ -1402,7 +1402,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
private boolean vertexSucceeded(Vertex vertex) {
numSuccessfulVertices++;
boolean failedCommit = false;
- if (!abortAllOutputsOnFailure) {
+ if (!commitAllOutputsOnSuccess) {
// committing successful outputs immediately. check for shared outputs
List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
if (groupsList != null) {
@@ -1453,7 +1453,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
+ ", vertexName=" + vertex.getName()
+ ", vertexId=" + vertex.getVertexId());
- if (!abortAllOutputsOnFailure) {
+ if (!commitAllOutputsOnSuccess) {
// partial output may already have been committed. fail if so
List<VertexGroupInfo> groupList = vertexGroupInfo.get(vertex.getName());
if (groupList != null) {
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 1d8274b..4e02f59 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -896,7 +896,7 @@ public class TestDAGImpl {
@Test(timeout=5000)
public void testDAGErrorAbortNonSuccessfulOutputs() {
// vertex success -> vertex output commit. failed dag aborts only non-successful vertices
- conf.setBoolean(TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE, false);
+ conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
initDAG(mrrDag);
dispatcher.await();
startDAG(mrrDag);