You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/01/09 02:39:07 UTC

[1/2] git commit: TEZ-685. Add archive link to mail list page. (hitesh)

Updated Branches:
  refs/heads/master 25615ff32 -> bd23c9d4d


TEZ-685. Add archive link to mail list page. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/35a523b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/35a523b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/35a523b1

Branch: refs/heads/master
Commit: 35a523b177e0b96eabc9ca5643bbdc52ab0ebf4f
Parents: 25615ff
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Dec 26 10:38:56 2013 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Jan 8 17:35:50 2014 -0800

----------------------------------------------------------------------
 docs/pom.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/35a523b1/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index e568e64..1bea40f 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -54,7 +54,7 @@
             <unsubscribe>mailto:user-unsubscribe@tez.incubator.apache.org
             </unsubscribe>
             <post>mailto:user@tez.incubator.apache.org</post>
-            <archive></archive>
+            <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-user/</archive>
         </mailingList>
         <mailingList>
             <name>Development list</name>
@@ -63,7 +63,7 @@
             <unsubscribe>mailto:dev-unsubscribe@tez.incubator.apache.org
             </unsubscribe>
             <post>mailto:dev@tez.incubator.apache.org</post>
-            <archive></archive>
+            <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-dev/</archive>
         </mailingList>
         <mailingList>
             <name>Commit list</name>
@@ -72,7 +72,7 @@
             <unsubscribe>mailto:commits-unsubscribe@tez.incubator.apache.org
             </unsubscribe>
             <post>mailto:commits@tez.incubator.apache.org</post>
-            <archive></archive>
+            <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-commits/</archive>
         </mailingList>
         <mailingList>
             <name>JIRA Issues list</name>
@@ -81,7 +81,7 @@
             <unsubscribe>mailto:issues-unsubscribe@tez.incubator.apache.org
             </unsubscribe>
             <post>mailto:issues@tez.incubator.apache.org</post>
-            <archive></archive>
+            <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-issues/</archive>
         </mailingList>
     </mailingLists>
 


[2/2] git commit: TEZ-41. Get VertexCommitter from API and remove MRVertexOutputCommitter. (hitesh)

Posted by hi...@apache.org.
TEZ-41. Get VertexCommitter from API and remove MRVertexOutputCommitter. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/bd23c9d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/bd23c9d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/bd23c9d4

Branch: refs/heads/master
Commit: bd23c9d4d88ee3adf503503faf0ff1bb4305726c
Parents: 35a523b
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Jan 8 17:37:24 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Jan 8 17:37:24 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   4 +
 .../apache/tez/dag/api/RootInputLeafOutput.java |   3 +-
 .../java/org/apache/tez/dag/api/Vertex.java     |  15 +-
 .../apache/tez/runtime/api/OutputCommitter.java |  71 ++++
 .../tez/runtime/api/OutputCommitterContext.java |  68 ++++
 .../apache/hadoop/CustomOutputCommitter.java    |  85 -----
 .../hadoop/mapred/MRVertexOutputCommitter.java  | 174 ---------
 .../committer/NullVertexOutputCommitter.java    |  47 ---
 .../tez/dag/api/committer/VertexContext.java    |  60 ---
 .../api/committer/VertexOutputCommitter.java    |  70 ----
 .../dag/impl/OutputCommitterContextImpl.java    |  77 ++++
 .../apache/tez/dag/app/dag/impl/VertexImpl.java | 133 ++++---
 .../rm/container/ContainerSignatureMatcher.java |  17 +
 .../tez/dag/app/dag/impl/TestVertexImpl.java    | 361 ++++++++++++-------
 tez-mapreduce-examples/pom.xml                  |   7 +-
 .../mapreduce/examples/FilterLinesByWord.java   |   7 +-
 .../examples/FilterLinesByWordOneToOne.java     |   4 +-
 .../tez/mapreduce/examples/WordCount.java       |   3 +-
 .../mapreduce/committer/MROutputCommitter.java  | 180 +++++++++
 .../apache/tez/mapreduce/hadoop/MRHelpers.java  |   3 +-
 20 files changed, 749 insertions(+), 640 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 69f648b..6d5035b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,10 @@ Apache Tez Change Log
 
 Release 0.3.0 - Unreleased
 
+INCOMPATIBLE CHANGES
+  
+  TEZ-41. Get VertexCommitter from API and remove MRVertexOutputCommitter.
+
 Release 0.2.0 - 2013-11-30
 
   First version.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
index 1334c41..11929b9 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
@@ -26,7 +26,7 @@ class  RootInputLeafOutput <T extends TezEntityDescriptor> {
   private final String name;
   private final T descriptor;
   private final Class<?> initializerClazz;
-  
+
   RootInputLeafOutput(String name, T descriptor, Class<?> initializerClazz) {
     this.name = name;
     this.descriptor = descriptor;
@@ -44,4 +44,5 @@ class  RootInputLeafOutput <T extends TezEntityDescriptor> {
   public Class<?> getInitializerClass() {
     return this.initializerClazz;
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 17fa558..4037fe5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.runtime.api.LogicalIOProcessor;
+import org.apache.tez.runtime.api.OutputCommitter;
 import org.apache.tez.runtime.api.TezRootInputInitializer;
 import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
 
-public class Vertex { // FIXME rename to Task
+public class Vertex {
 
   private final String vertexName;
   private final ProcessorDescriptor processorDescriptor;
@@ -176,15 +177,21 @@ public class Vertex { // FIXME rename to Task
    *          the name of the output. This will be used when accessing the
    *          output in the {@link LogicalIOProcessor}
    * @param outputDescriptor
+   * @param outputCommitterClazz Class to be used for the OutputCommitter.
+   *                             Can be null.
    * @return
    */
-  // TODO : Add a processing component.
-  public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor) {
+  public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor,
+      Class<? extends OutputCommitter> outputCommitterClazz) {
     additionalOutputs.add(new RootInputLeafOutput<OutputDescriptor>(outputName,
-        outputDescriptor, null));
+        outputDescriptor, outputCommitterClazz));
     return this;
   }
 
+  public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor) {
+    return addOutput(outputName, outputDescriptor, null);
+  }
+
   public String getJavaOpts(){
 	  return javaOpts;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
new file mode 100644
index 0000000..f0cee4e
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import org.apache.tez.dag.api.client.VertexStatus;
+
+import java.io.IOException;
+
+/**
+ * OutputCommitter to "finalize" the output and make it user-visible if needed.
+ * The OutputCommitter is allowed only on a terminal Output.
+ */
+public abstract class OutputCommitter {
+
+  /**
+   * Setup up the Output committer.
+   *
+   * @param context Context of the output that is being acted upon
+   * @throws java.lang.Exception
+   */
+  public abstract void initialize(OutputCommitterContext context)
+      throws Exception;
+
+  /**
+   * For the framework to setup the output during initialization. This is
+   * called from the application master process for the vertex. This will be
+   * called multiple times, once per dag attempt.
+   *
+   * @throws java.lang.Exception if setup fails
+   */
+  public abstract void setupOutput() throws Exception;
+
+  /**
+   * For committing the output after a successful vertex completion.
+   * Note that this is invoked for the outputs of vertices with a successful
+   * final state. This is called from the application master process.
+   * This is guaranteed to only be called once.
+   * If it throws an exception the entire vertex will fail.
+   *
+   * @throws java.lang.Exception
+   */
+  public abstract void commitOutput() throws Exception;
+
+  /**
+   * For aborting an unsuccessful vertex's output. Note that this is invoked for
+   * vertices with a final failed state. This is called from the application
+   * master process. This may be called multiple times.
+   *
+   * @param finalState final run-state of the vertex
+   * @throws java.lang.Exception
+   */
+  public abstract void abortOutput(VertexStatus.State finalState)
+    throws Exception;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
new file mode 100644
index 0000000..2136073
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import java.util.List;
+
+/**
+ * Context through which the OutputCommitter can access all the relevant
+ * information that it needs.
+ */
+
+public interface OutputCommitterContext {
+
+  /**
+   * Get the {@link org.apache.hadoop.yarn.api.records.ApplicationId}
+   * @return the {@link org.apache.hadoop.yarn.api.records.ApplicationId}
+   */
+  public ApplicationId getApplicationId();
+
+  /**
+   * Get the current DAG Attempt Number
+   * @return DAG Attempt Number
+   */
+  public int getDAGAttemptNumber();
+
+  /**
+   * Get the name of the DAG
+   * @return the DAG name
+   */
+  public String getDAGName();
+
+  /**
+   * Get the name of the Vertex
+   * @return Vertex Name
+   */
+  public String getVertexName();
+
+  /**
+   * Get the name of the Output
+   * @return Output Name
+   */
+  public String getOutputName();
+
+  /**
+   * Get the User Payload for the Output
+   * @return User Payload
+   */
+  public byte[] getUserPayload();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
deleted file mode 100644
index f151ced..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.OutputCommitter;
-import org.apache.hadoop.mapred.TaskAttemptContext;
-
-public class CustomOutputCommitter extends OutputCommitter {
-
-  public static final String JOB_SETUP_FILE_NAME = "_job_setup";
-  public static final String JOB_COMMIT_FILE_NAME = "_job_commit";
-  public static final String JOB_ABORT_FILE_NAME = "_job_abort";
-  public static final String TASK_SETUP_FILE_NAME = "_task_setup";
-  public static final String TASK_ABORT_FILE_NAME = "_task_abort";
-  public static final String TASK_COMMIT_FILE_NAME = "_task_commit";
-
-  @Override
-  public void setupJob(JobContext jobContext) throws IOException {
-    writeFile(jobContext.getJobConf(), JOB_SETUP_FILE_NAME);
-  }
-
-  @Override
-  public void commitJob(JobContext jobContext) throws IOException {
-    super.commitJob(jobContext);
-    writeFile(jobContext.getJobConf(), JOB_COMMIT_FILE_NAME);
-  }
-
-  @Override
-  public void abortJob(JobContext jobContext, int status) 
-  throws IOException {
-    super.abortJob(jobContext, status);
-    writeFile(jobContext.getJobConf(), JOB_ABORT_FILE_NAME);
-  }
-  
-  @Override
-  public void setupTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
-  }
-
-  @Override
-  public boolean needsTaskCommit(TaskAttemptContext taskContext)
-      throws IOException {
-    return true;
-  }
-
-  @Override
-  public void commitTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
-  }
-
-  @Override
-  public void abortTask(TaskAttemptContext taskContext) throws IOException {
-    writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
-  }
-
-  private void writeFile(JobConf conf , String filename) throws IOException {
-    System.out.println("writing file ----" + filename);
-    Path outputPath = FileOutputFormat.getOutputPath(conf);
-    FileSystem fs = outputPath.getFileSystem(conf);
-    fs.create(new Path(outputPath, filename)).close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
deleted file mode 100644
index 14a0c52..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.JobStatus.State;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.utils.TezBuilderUtils;
-import org.apache.tez.mapreduce.hadoop.MRHelpers;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-
-public class MRVertexOutputCommitter extends VertexOutputCommitter {
-
-  private static final Log LOG = LogFactory.getLog(
-      MRVertexOutputCommitter.class);
-
-  private OutputCommitter committer = null;
-  private JobContext jobContext = null;
-  private volatile boolean initialized = false;
-  private JobConf jobConf = null;
-
-  public MRVertexOutputCommitter() {
-  }
-
-  @SuppressWarnings("rawtypes")
-  private OutputCommitter getOutputCommitter(VertexContext context) {
-
-    OutputCommitter committer = null;
-    boolean newApiCommitter = false;
-    if (jobConf.getBoolean("mapred.reducer.new-api", false)
-        || jobConf.getBoolean("mapred.mapper.new-api", false))  {
-      newApiCommitter = true;
-      LOG.info("Using mapred newApiCommitter.");
-    }
-
-    LOG.info("OutputCommitter set in config for vertex: "
-        + context.getVertexID() + " : "
-        + jobConf.get("mapred.output.committer.class"));
-
-    if (newApiCommitter) {
-      TezTaskID taskId = TezBuilderUtils.newTaskId(context.getDAGID(),
-          context.getVertexID().getId(), 0);
-      TezTaskAttemptID attemptID =
-          TezBuilderUtils.newTaskAttemptId(taskId, 0);
-      TaskAttemptContext taskContext = new TaskAttemptContextImpl(jobConf,
-          TezMRTypeConverter.fromTez(attemptID));
-      try {
-        OutputFormat outputFormat = ReflectionUtils.newInstance(taskContext
-            .getOutputFormatClass(), jobConf);
-        committer = outputFormat.getOutputCommitter(taskContext);
-      } catch (Exception e) {
-        throw new TezUncheckedException(e);
-      }
-    } else {
-      committer = ReflectionUtils.newInstance(jobConf.getClass(
-          "mapred.output.committer.class", FileOutputCommitter.class,
-          org.apache.hadoop.mapred.OutputCommitter.class), jobConf);
-    }
-    LOG.info("OutputCommitter is " + committer.getClass().getName());
-    return committer;
-  }
-
-  // FIXME we are using ApplicationId as DAG id
-  private JobContext getJobContextFromVertexContext(VertexContext context)
-      throws IOException {
-    JobID jobId = TypeConverter.fromYarn(
-        context.getDAGID().getApplicationId());
-    return new MRJobContextImpl(jobConf, jobId);
-  }
-
-  private State getJobStateFromVertexStatusState(VertexStatus.State state) {
-    switch(state) {
-    case INITED:
-      return JobStatus.State.PREP;
-    case RUNNING:
-      return JobStatus.State.RUNNING;
-    case SUCCEEDED:
-      return JobStatus.State.SUCCEEDED;
-    case KILLED:
-      return JobStatus.State.KILLED;
-    case FAILED:
-    case ERROR:
-      return JobStatus.State.FAILED;
-    default:
-      throw new TezUncheckedException("Unknown VertexStatus.State: " + state);
-    }
-  }
-
-  @Override
-  public void init(VertexContext context) throws IOException {
-    // TODO VertexContext not the best way to get ApplicationAttemptId. No
-    // alternates right now.
-
-    byte[] userPayload = context.getUserPayload();
-    if (userPayload == null) {
-      jobConf = new JobConf();
-    } else {
-      jobConf = new JobConf(
-          MRHelpers.createConfFromUserPayload(context.getUserPayload()));
-    }
-
-    jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
-        context.getApplicationAttemptId().getAttemptId());
-    committer = getOutputCommitter(context);
-    jobContext = getJobContextFromVertexContext(context);
-    initialized = true;
-  }
-
-  @Override
-  public void setupVertex() throws IOException {
-    if (!initialized) {
-      throw new RuntimeException("Committer not initialized");
-    }
-    committer.setupJob(jobContext);
-  }
-
-  @Override
-  public void commitVertex() throws IOException {
-    if (!initialized) {
-      throw new RuntimeException("Committer not initialized");
-    }
-    committer.commitJob(jobContext);
-  }
-
-  @Override
-  public void abortVertex(VertexStatus.State finalState) throws IOException {
-    if (!initialized) {
-      throw new RuntimeException("Committer not initialized");
-    }
-    State jobState = getJobStateFromVertexStatusState(finalState);
-    committer.abortJob(jobContext, jobState);
-  }
-
-  private static class MRJobContextImpl
-      extends org.apache.hadoop.mapred.JobContextImpl {
-
-    public MRJobContextImpl(JobConf jobConf, JobID jobId) {
-      super(jobConf, jobId);
-    }
-
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
deleted file mode 100644
index 08b2d4a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-import java.io.IOException;
-
-import org.apache.tez.dag.api.client.VertexStatus;
-
-public class NullVertexOutputCommitter extends VertexOutputCommitter {
-
-  @Override
-  public void init(VertexContext context) throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void setupVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void commitVertex() throws IOException {
-    // Nothing to do
-  }
-
-  @Override
-  public void abortVertex(VertexStatus.State finalState) throws IOException {
-    // Nothing to do
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
deleted file mode 100644
index 5c4430e..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public class VertexContext {
-
-  private final TezDAGID tezDAGID;
-  private final byte[] userPayload;
-  private final ApplicationAttemptId applicationAttemptId;
-  private final TezVertexID tezVertexID;
-
-  public VertexContext(TezDAGID tezDAGID, byte[] userPayload,
-      TezVertexID tezVertexID,
-      ApplicationAttemptId applicationAttemptId) {
-    this.tezDAGID = tezDAGID;
-    this.userPayload = userPayload;
-    this.tezVertexID = tezVertexID;
-    this.applicationAttemptId = applicationAttemptId;
-  }
-
-  public TezDAGID getDAGID() {
-    return tezDAGID;
-  }
-
-  public byte[] getUserPayload() {
-    return userPayload;
-  }
-
-  // TODO get rid of this as part of VertexContext cleanup
-  public ApplicationAttemptId getApplicationAttemptId() {
-    return applicationAttemptId;
-  }
-
-  public TezVertexID getVertexID() {
-    return tezVertexID;
-  }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
deleted file mode 100644
index 34c6e89..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.tez.dag.api.client.VertexStatus;
-
-@Public
-@Unstable
-public abstract class VertexOutputCommitter {
-
-  /**
-   * Setup up the vertex output committer.
-   *
-   * @param context Context of the vertex whose output is being written.
-   * @throws IOException
-   */
-  public abstract void init(VertexContext context) throws IOException;
-
-  /**
-   * For the framework to setup the vertex output during initialization. This is
-   * called from the application master process for the vertex. This will be
-   * called multiple times, once per dag attempt for each vertex.
-   *
-   * @throws IOException if setup fails
-   */
-  public abstract void setupVertex() throws IOException;
-
-  /**
-   * For committing vertex's output after successful vertex completion.
-   * Note that this is invoked for vertices with a successful final state.
-   * This is called from the application master process for the entire vertex.
-   * This is guaranteed to only be called once.
-   * If it throws an exception the entire vertex will fail.
-   *
-   * @throws IOException
-   */
-  public abstract void commitVertex() throws IOException;
-
-  /**
-   * For aborting an unsuccessful vertex's output. Note that this is invoked for
-   * vertices with a final failed state. This is called from the application
-   * master process for the entire vertex. This may be called multiple times.
-   *
-   * @param state final runstate of the vertex
-   * @throws IOException
-   */
-  public abstract void abortVertex(VertexStatus.State finalState)
-      throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
new file mode 100644
index 0000000..68c8835
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+
+public class OutputCommitterContextImpl implements OutputCommitterContext {
+
+  private final ApplicationId applicationId;
+  private final int dagAttemptNumber;
+  private final String dagName;
+  private final String vertexName;
+  private final String outputName;
+  private final byte[] userPayload;
+
+  public OutputCommitterContextImpl(ApplicationId applicationId,
+      int dagAttemptNumber,
+      String dagName,
+      String vertexName,
+      String outputName,
+      byte[] userPayload) {
+    this.applicationId = applicationId;
+    this.dagAttemptNumber = dagAttemptNumber;
+    this.dagName = dagName;
+    this.vertexName = vertexName;
+    this.outputName = outputName;
+    this.userPayload = userPayload;
+  }
+
+  @Override
+  public ApplicationId getApplicationId() {
+    return applicationId;
+  }
+
+  @Override
+  public int getDAGAttemptNumber() {
+    return dagAttemptNumber;
+  }
+
+  @Override
+  public String getDAGName() {
+    return dagName;
+  }
+
+  @Override
+  public String getVertexName() {
+    return vertexName;
+  }
+
+  @Override
+  public String getOutputName() {
+    return outputName;
+  }
+
+  @Override
+  public byte[] getUserPayload() {
+    return userPayload;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 2672609..f77521f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -37,11 +37,11 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.MRVertexOutputCommitter;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.StringInterner;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -65,9 +65,6 @@ import org.apache.tez.dag.api.client.ProgressBuilder;
 import org.apache.tez.dag.api.client.StatusGetOpts;
 import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.client.VertexStatusBuilder;
-import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto;
 import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
@@ -116,6 +113,9 @@ import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.RuntimeUtils;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
 import org.apache.tez.runtime.api.events.DataMovementEvent;
 import org.apache.tez.runtime.api.events.InputFailedEvent;
 import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
@@ -437,15 +437,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private final String vertexName;
   private final ProcessorDescriptor processorDescriptor;
 
-  // For committer
-  private final VertexContext vertexContext;
-
   @VisibleForTesting
   Map<Vertex, Edge> sourceVertices;
   private Map<Vertex, Edge> targetVertices;
 
   private Map<String, RootInputLeafOutputDescriptor<InputDescriptor>> additionalInputs;
   private Map<String, RootInputLeafOutputDescriptor<OutputDescriptor>> additionalOutputs;
+  private Map<String, OutputCommitter> outputCommitters;
   private final List<InputSpec> additionalInputSpecs = new ArrayList<InputSpec>();
   private final List<OutputSpec> additionalOutputSpecs = new ArrayList<OutputSpec>();
   private Set<String> inputsWithInitializers;
@@ -461,7 +459,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   private boolean parallelismSet = false;
   private TezVertexID originalOneToOneSplitSource = null;
 
-  private VertexOutputCommitter committer;
   private AtomicBoolean committed = new AtomicBoolean(false);
   private VertexLocationHint vertexLocationHint;
   private Map<String, LocalResource> localResources;
@@ -500,7 +497,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     this.writeLock = readWriteLock.writeLock();
 
     this.credentials = credentials;
-    this.committer = new NullVertexOutputCommitter();
     this.vertexLocationHint = vertexLocationHint;
     if (LOG.isDebugEnabled()) {
       logLocationHints(this.vertexLocationHint);
@@ -521,10 +517,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
         .getTaskConfig().getJavaOpts() : null;
 
-    this.vertexContext = new VertexContext(getDAGId(),
-        this.processorDescriptor.getUserPayload(), this.vertexId,
-        getApplicationAttemptId());
-
     this.containerContext = new ContainerContext(this.localResources,
         this.credentials, this.environment, this.javaOpts, this);
 
@@ -1053,7 +1045,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
             // commit only once
             LOG.info("Invoking committer commit for vertex, vertexId="
                 + vertex.logIdentifier);
-            vertex.committer.commitVertex();
+            if (vertex.outputCommitters != null) {
+              for (Entry<String, OutputCommitter> entry :
+                  vertex.outputCommitters.entrySet()) {
+                LOG.info("Invoking committer commit for output=" + entry.getKey()
+                    + ", vertexId=" + vertex.logIdentifier);
+                entry.getValue().commitOutput();
+              }
+            }
           }
         } catch (Exception e) {
           LOG.error("Failed to do commit on vertex, vertexId="
@@ -1167,27 +1166,51 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
     // TODO TEZ-41 make commmitter type configurable per vertex
 
     if (!this.additionalOutputSpecs.isEmpty()) {
-      committer = new MRVertexOutputCommitter();
-    }
-    try {
-      LOG.info("Invoking committer init for vertex, vertexId=" + logIdentifier);
-      committer.init(vertexContext);
-      LOG.info("Invoking committer setup for vertex, vertexId="
-          + logIdentifier);
-      committer.setupVertex();
-    } catch (Exception e) {
-      LOG.warn("Vertex init failed, vertexId=" + logIdentifier, e);
-      addDiagnostic("Vertex init failed : "
-          + StringUtils.stringifyException(e));
-      trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
-      abortVertex(VertexStatus.State.FAILED);
-      // TODO: Metrics
-      //job.metrics.endPreparingJob(vertex);
-      return finished(VertexState.FAILED);
+      try {
+        LOG.info("Invoking committer inits for vertex, vertexId=" + logIdentifier);
+        for (Entry<String, RootInputLeafOutputDescriptor<OutputDescriptor>> entry:
+          additionalOutputs.entrySet())  {
+          final String outputName = entry.getKey();
+          final RootInputLeafOutputDescriptor<OutputDescriptor> od = entry.getValue();
+          if (od.getInitializerClassName() == null
+            || od.getInitializerClassName().isEmpty()) {
+            LOG.info("Ignoring committer as none specified for output="
+                + outputName
+                + ", vertexId=" + logIdentifier);
+            continue;
+          }
+          LOG.info("Instantiating committer for output=" + outputName
+              + ", vertexId=" + logIdentifier
+              + ", committerClass=" + od.getInitializerClassName());
+
+          OutputCommitter outputCommitter = RuntimeUtils.createClazzInstance(
+              od.getInitializerClassName());
+          OutputCommitterContext outputCommitterContext =
+              new OutputCommitterContextImpl(appContext.getApplicationID(),
+                  appContext.getApplicationAttemptId().getAttemptId(),
+                  appContext.getCurrentDAG().getName(),
+                  vertexName,
+                  outputName,
+                  od.getDescriptor().getUserPayload());
+
+          LOG.info("Invoking committer init for output=" + outputName
+              + ", vertexId=" + logIdentifier);
+          outputCommitter.initialize(outputCommitterContext);
+          outputCommitters.put(outputName, outputCommitter);
+          LOG.info("Invoking committer setup for output=" + outputName
+              + ", vertexId=" + logIdentifier);
+          outputCommitter.setupOutput();
+        }
+      } catch (Exception e) {
+        LOG.warn("Vertex Committer init failed, vertexId=" + logIdentifier, e);
+        addDiagnostic("Vertex init failed : "
+            + StringUtils.stringifyException(e));
+        trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
+        abortVertex(VertexStatus.State.FAILED);
+        return finished(VertexState.FAILED);
+      }
     }
-
     // TODO: Metrics
-    //vertex.metrics.endPreparingJob(job);
     initedTime = clock.getTime();
 
     return VertexState.INITED;
@@ -1588,14 +1611,20 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   }
 
   private void abortVertex(VertexStatus.State finalState) {
-    try {
-      LOG.info("Invoking committer abort for vertex, vertexId="
-          + logIdentifier);
-      committer.abortVertex(finalState);
-    } catch (Exception e) {
-      LOG.warn("Could not abort vertex, vertexId=" + logIdentifier, e);
+    LOG.info("Invoking committer abort for vertex, vertexId="
+        + logIdentifier);
+    if (outputCommitters != null) {
+      for (Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
+        try {
+          LOG.info("Invoking committer abort for output=" + entry.getKey()
+              + ", vertexId=" + logIdentifier);
+          entry.getValue().abortOutput(finalState);
+        } catch (Exception e) {
+        LOG.warn("Could not abort committer for output=" + entry.getKey()
+            + ", vertexId=" + logIdentifier, e);
+        }
+      }
     }
-
     if (finishTime == 0) {
       setFinishTime();
     }
@@ -1826,7 +1855,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
     @Override
     public VertexState transition(VertexImpl vertex, VertexEvent event) {
-      if (vertex.committer instanceof NullVertexOutputCommitter) {
+      if (vertex.outputCommitters == null
+          || vertex.outputCommitters.isEmpty()) {
         LOG.info(vertex.getVertexId() + " back to running due to rescheduling "
             + ((VertexEventTaskReschedule)event).getTaskID());
         (new TaskRescheduledTransition()).transition(vertex, event);
@@ -2034,10 +2064,20 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
 
   }
 
+  @Private
+  @VisibleForTesting
+  public OutputCommitter getOutputCommitter(String outputName) {
+    if (this.outputCommitters != null) {
+      return outputCommitters.get(outputName);
+    }
+    return null;
+  }
+
   @Override
   public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) {
     LOG.info("setting additional outputs for vertex " + this.vertexName);
     this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size());
+    this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size());
     for (RootInputLeafOutputProto output : outputs) {
 
       OutputDescriptor od = DagTypeConverters
@@ -2185,17 +2225,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
   }
 
   @VisibleForTesting
-  VertexOutputCommitter getVertexOutputCommitter() {
-    return this.committer;
-  }
-
-  @VisibleForTesting
-  // Only to be used for testing
-  void setVertexOutputCommitter(VertexOutputCommitter committer) {
-    this.committer = committer;
-  }
-
-  @VisibleForTesting
   VertexManager getVertexManager() {
     return this.vertexManager;
   }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
index 4e3f204..4d65c23 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.tez.dag.app.rm.container;
 
 import java.util.Map;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index d0c26d4..40de46f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -23,16 +23,24 @@ import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import com.google.protobuf.ByteString;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.MRVertexOutputCommitter;
+import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -49,9 +57,6 @@ import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
 import org.apache.tez.dag.api.oldrecords.TaskState;
 import org.apache.tez.dag.api.records.DAGProtos;
 import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
@@ -98,6 +103,8 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
 import org.apache.tez.dag.records.TezTaskID;
 import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
 import org.apache.tez.runtime.api.events.RootInputConfigureVertexTasksEvent;
 import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
 import org.junit.After;
@@ -138,8 +145,8 @@ public class TestVertexImpl {
   private VertexEventDispatcher vertexEventDispatcher;
   private DagEventDispatcher dagEventDispatcher;
 
-  private class CountingVertexOutputCommitter extends
-      VertexOutputCommitter {
+  public static class CountingOutputCommitter extends
+      OutputCommitter {
 
     public int initCounter = 0;
     public int setupCounter = 0;
@@ -149,7 +156,7 @@ public class TestVertexImpl {
     private boolean throwErrorOnAbort;
     private boolean throwRuntimeException;
 
-    public CountingVertexOutputCommitter(boolean throwError,
+    public CountingOutputCommitter(boolean throwError,
         boolean throwOnAbort,
         boolean throwRuntimeException) {
       this.throwError = throwError;
@@ -157,22 +164,29 @@ public class TestVertexImpl {
       this.throwRuntimeException = throwRuntimeException;
     }
 
-    public CountingVertexOutputCommitter() {
+    public CountingOutputCommitter() {
       this(false, false, false);
     }
 
     @Override
-    public void init(VertexContext context) throws IOException {
+    public void initialize(OutputCommitterContext context) throws IOException {
+      if (context.getUserPayload() != null) {
+        CountingOutputCommitterConfig conf =
+            new CountingOutputCommitterConfig(context.getUserPayload());
+        this.throwError = conf.throwError;
+        this.throwErrorOnAbort = conf.throwErrorOnAbort;
+        this.throwRuntimeException = conf.throwRuntimeException;
+      }
       ++initCounter;
     }
 
     @Override
-    public void setupVertex() {
+    public void setupOutput() throws IOException {
       ++setupCounter;
     }
 
     @Override
-    public void commitVertex() throws IOException {
+    public void commitOutput() throws IOException {
       ++commitCounter;
       if (throwError) {
         if (!throwRuntimeException) {
@@ -184,7 +198,7 @@ public class TestVertexImpl {
     }
 
     @Override
-    public void abortVertex(VertexStatus.State finalState) throws IOException {
+    public void abortOutput(VertexStatus.State finalState) throws IOException {
       ++abortCounter;
       if (throwErrorOnAbort) {
         if (!throwRuntimeException) {
@@ -194,6 +208,51 @@ public class TestVertexImpl {
         }
       }
     }
+
+    public static class CountingOutputCommitterConfig implements Writable {
+
+      boolean throwError = false;
+      boolean throwErrorOnAbort = false;
+      boolean throwRuntimeException = false;
+
+      public CountingOutputCommitterConfig() {
+      }
+
+      public CountingOutputCommitterConfig(boolean throwError,
+          boolean throwErrorOnAbort, boolean throwRuntimeException) {
+        this.throwError = throwError;
+        this.throwErrorOnAbort = throwErrorOnAbort;
+        this.throwRuntimeException = throwRuntimeException;
+      }
+
+      public CountingOutputCommitterConfig(byte[] payload) throws IOException {
+        DataInput in = new DataInputStream(new ByteArrayInputStream(payload));
+        this.readFields(in);
+      }
+
+      @Override
+      public void write(DataOutput out) throws IOException {
+        out.writeBoolean(throwError);
+        out.writeBoolean(throwErrorOnAbort);
+        out.writeBoolean(throwRuntimeException);
+      }
+
+      @Override
+      public void readFields(DataInput in) throws IOException {
+        throwError = in.readBoolean();
+        throwErrorOnAbort = in.readBoolean();
+        throwRuntimeException = in.readBoolean();
+      }
+
+      public byte[] toUserPayload() throws IOException {
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        DataOutput out = new DataOutputStream(bos);
+        write(out);
+        return bos.toByteArray();
+      }
+
+    }
+
   }
 
   private class TaskAttemptEventDispatcher implements EventHandler<TaskAttemptEvent> {
@@ -605,106 +664,107 @@ public class TestVertexImpl {
             .build()
         )
         .addVertex(
-          VertexPlan.newBuilder()
-            .setName("vertex6")
-            .setType(PlanVertexType.NORMAL)
-            .addTaskLocationHint(
-              PlanTaskLocationHint.newBuilder()
-                .addHost("host6")
-                .addRack("rack6")
-                .build()
-            )
-            .addOutputs(
-                DAGProtos.RootInputLeafOutputProto.newBuilder()
-                    .setEntityDescriptor(
-                      TezEntityDescriptorProto.newBuilder().setClassName("output").build()
-                    )
-                    .setName("outputx")
-            )
-            .setTaskConfig(
-                PlanTaskConfiguration.newBuilder()
-                    .setNumTasks(2)
-                    .setVirtualCores(4)
-                    .setMemoryMb(1024)
-                    .setJavaOpts("")
-                    .setTaskModule("x6.y6")
-                    .build()
+            VertexPlan.newBuilder()
+                .setName("vertex6")
+                .setType(PlanVertexType.NORMAL)
+                .addTaskLocationHint(
+                    PlanTaskLocationHint.newBuilder()
+                        .addHost("host6")
+                        .addRack("rack6")
+                        .build()
+                )
+                .addOutputs(
+                    DAGProtos.RootInputLeafOutputProto.newBuilder()
+                        .setEntityDescriptor(
+                            TezEntityDescriptorProto.newBuilder().setClassName("output").build()
+                        )
+                        .setName("outputx")
+                        .setInitializerClassName(CountingOutputCommitter.class.getName())
+                )
+                .setTaskConfig(
+                    PlanTaskConfiguration.newBuilder()
+                        .setNumTasks(2)
+                        .setVirtualCores(4)
+                        .setMemoryMb(1024)
+                        .setJavaOpts("")
+                        .setTaskModule("x6.y6")
+                        .build()
                 )
                 .addInEdgeId("e5")
                 .addInEdgeId("e6")
                 .build()
-            )
+        )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1"))
-                .setInputVertexName("vertex1")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
-                .setOutputVertexName("vertex3")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e1")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1"))
+                    .setInputVertexName("vertex1")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
+                    .setOutputVertexName("vertex3")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e1")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2"))
-                .setInputVertexName("vertex2")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2"))
-                .setOutputVertexName("vertex3")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e2")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2"))
+                    .setInputVertexName("vertex2")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2"))
+                    .setOutputVertexName("vertex3")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e2")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3"))
-                .setInputVertexName("vertex3")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4"))
-                .setOutputVertexName("vertex4")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e3")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3"))
+                    .setInputVertexName("vertex3")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4"))
+                    .setOutputVertexName("vertex4")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e3")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3"))
-                .setInputVertexName("vertex3")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5"))
-                .setOutputVertexName("vertex5")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e4")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3"))
+                    .setInputVertexName("vertex3")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5"))
+                    .setOutputVertexName("vertex5")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e4")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4"))
-                .setInputVertexName("vertex4")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4"))
-                .setOutputVertexName("vertex6")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e5")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4"))
+                    .setInputVertexName("vertex4")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4"))
+                    .setOutputVertexName("vertex6")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e5")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .addEdge(
-              EdgePlan.newBuilder()
-                .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5"))
-                .setInputVertexName("vertex5")
-                .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5"))
-                .setOutputVertexName("vertex6")
-                .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
-                .setId("e6")
-                .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
-                .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
-                .build()
+                EdgePlan.newBuilder()
+                    .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5"))
+                    .setInputVertexName("vertex5")
+                    .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5"))
+                    .setOutputVertexName("vertex6")
+                    .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+                    .setId("e6")
+                    .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+                    .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+                    .build()
             )
             .build();
 
@@ -1382,12 +1442,11 @@ public class TestVertexImpl {
   public void testVertexCommitterInit() {
     initAllVertices(VertexState.INITED);
     VertexImpl v2 = vertices.get("vertex2");
-    Assert.assertTrue(v2.getVertexOutputCommitter()
-        instanceof NullVertexOutputCommitter);
+    Assert.assertNull(v2.getOutputCommitter("output"));
 
     VertexImpl v6 = vertices.get("vertex6");
-    Assert.assertTrue(v6.getVertexOutputCommitter()
-        instanceof MRVertexOutputCommitter);
+    Assert.assertTrue(v6.getOutputCommitter("outputx")
+        instanceof CountingOutputCommitter);
   }
 
   @Test(timeout = 5000)
@@ -1407,12 +1466,11 @@ public class TestVertexImpl {
   public void testVertexTaskFailure() {
     initAllVertices(VertexState.INITED);
 
-    VertexImpl v = vertices.get("vertex2");
+    VertexImpl v = vertices.get("vertex6");
 
-    CountingVertexOutputCommitter committer =
-        new CountingVertexOutputCommitter();
-    v.setVertexOutputCommitter(committer);
     startVertex(v);
+    CountingOutputCommitter committer =
+        (CountingOutputCommitter) v.getOutputCommitter("outputx");
 
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
     TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1543,10 +1601,6 @@ public class TestVertexImpl {
 
     VertexImpl v = vertices.get("vertex2");
 
-    CountingVertexOutputCommitter committer =
-        new CountingVertexOutputCommitter();
-    v.setVertexOutputCommitter(committer);
-
     startVertex(v);
 
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1562,13 +1616,11 @@ public class TestVertexImpl {
         new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.RUNNING, v.getState());
-    Assert.assertEquals(0, committer.commitCounter);
 
     dispatcher.getEventHandler().handle(
         new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
     dispatcher.await();
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
-    Assert.assertEquals(1, committer.commitCounter);
 
   }
   
@@ -1580,8 +1632,6 @@ public class TestVertexImpl {
 
     VertexImpl v = vertices.get("vertex2");
 
-    v.setVertexOutputCommitter(new NullVertexOutputCommitter());
-
     startVertex(v);
 
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1617,14 +1667,24 @@ public class TestVertexImpl {
   
   @SuppressWarnings("unchecked")
   @Test(timeout = 5000)
-  public void testVertexSuccessToFailedAfterTaskScheduler() {
+  public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
     // For downstream failures
-    initAllVertices(VertexState.INITED);
-
     VertexImpl v = vertices.get("vertex2");
 
-    v.setVertexOutputCommitter(new CountingVertexOutputCommitter());
+    List<RootInputLeafOutputProto> outputs =
+        new ArrayList<RootInputLeafOutputProto>();
+    outputs.add(RootInputLeafOutputProto.newBuilder()
+        .setInitializerClassName(CountingOutputCommitter.class.getName())
+        .setName("output_v2")
+        .setEntityDescriptor(
+            TezEntityDescriptorProto.newBuilder()
+                .setUserPayload(ByteString.copyFrom(
+                    new CountingOutputCommitter.CountingOutputCommitterConfig()
+                        .toUserPayload())).build())
+        .build());
+    v.setAdditionalOutputs(outputs);
 
+    initAllVertices(VertexState.INITED);
     startVertex(v);
 
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1655,13 +1715,11 @@ public class TestVertexImpl {
   public void testVertexCommit() {
     initAllVertices(VertexState.INITED);
 
-    VertexImpl v = vertices.get("vertex2");
-
-    CountingVertexOutputCommitter committer =
-        new CountingVertexOutputCommitter();
-    v.setVertexOutputCommitter(committer);
+    VertexImpl v = vertices.get("vertex6");
 
     startVertex(v);
+    CountingOutputCommitter committer =
+        (CountingOutputCommitter) v.getOutputCommitter("outputx");
 
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
     TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1682,8 +1740,8 @@ public class TestVertexImpl {
     Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
     Assert.assertEquals(1, committer.commitCounter);
     Assert.assertEquals(0, committer.abortCounter);
-    Assert.assertEquals(0, committer.initCounter); // already done in init
-    Assert.assertEquals(0, committer.setupCounter); // already done in init
+    Assert.assertEquals(1, committer.initCounter);
+    Assert.assertEquals(1, committer.setupCounter);
   }
 
   @Test(timeout = 5000)
@@ -1693,17 +1751,28 @@ public class TestVertexImpl {
 
   @SuppressWarnings("unchecked")
   @Test(timeout = 5000)
-  public void testBadCommitter() {
-    initAllVertices(VertexState.INITED);
-
+  public void testBadCommitter() throws Exception {
     VertexImpl v = vertices.get("vertex2");
 
-    CountingVertexOutputCommitter committer =
-        new CountingVertexOutputCommitter(true, true, false);
-    v.setVertexOutputCommitter(committer);
+    List<RootInputLeafOutputProto> outputs =
+        new ArrayList<RootInputLeafOutputProto>();
+    outputs.add(RootInputLeafOutputProto.newBuilder()
+        .setInitializerClassName(CountingOutputCommitter.class.getName())
+        .setName("output_v2")
+        .setEntityDescriptor(
+            TezEntityDescriptorProto.newBuilder()
+                .setUserPayload(ByteString.copyFrom(
+                    new CountingOutputCommitter.CountingOutputCommitterConfig(
+                        true, true, false).toUserPayload())).build())
+        .build());
+    v.setAdditionalOutputs(outputs);
 
+    initAllVertices(VertexState.INITED);
     startVertex(v);
 
+    CountingOutputCommitter committer =
+        (CountingOutputCommitter) v.getOutputCommitter("output_v2");
+
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
     TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
 
@@ -1714,27 +1783,39 @@ public class TestVertexImpl {
     dispatcher.await();
     Assert.assertEquals(VertexState.FAILED, v.getState());
     Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause());
+
     Assert.assertEquals(1, committer.commitCounter);
 
     // FIXME need to verify whether abort needs to be called if commit fails
     Assert.assertEquals(0, committer.abortCounter);
-    Assert.assertEquals(0, committer.initCounter); // already done in init
-    Assert.assertEquals(0, committer.setupCounter); // already done in init
+    Assert.assertEquals(1, committer.initCounter);
+    Assert.assertEquals(1, committer.setupCounter);
   }
 
   @SuppressWarnings("unchecked")
   @Test(timeout = 5000)
-  public void testBadCommitter2() {
-    initAllVertices(VertexState.INITED);
-
+  public void testBadCommitter2() throws Exception {
     VertexImpl v = vertices.get("vertex2");
 
-    CountingVertexOutputCommitter committer =
-      new CountingVertexOutputCommitter(true, true, true);
-    v.setVertexOutputCommitter(committer);
+    List<RootInputLeafOutputProto> outputs =
+        new ArrayList<RootInputLeafOutputProto>();
+    outputs.add(RootInputLeafOutputProto.newBuilder()
+        .setInitializerClassName(CountingOutputCommitter.class.getName())
+        .setName("output_v2")
+        .setEntityDescriptor(
+            TezEntityDescriptorProto.newBuilder()
+                .setUserPayload(ByteString.copyFrom(
+                    new CountingOutputCommitter.CountingOutputCommitterConfig(
+                        true, true, true).toUserPayload())).build())
+        .build());
+    v.setAdditionalOutputs(outputs);
 
+    initAllVertices(VertexState.INITED);
     startVertex(v);
 
+    CountingOutputCommitter committer =
+        (CountingOutputCommitter) v.getOutputCommitter("output_v2");
+
     TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
     TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
 
@@ -1749,8 +1830,8 @@ public class TestVertexImpl {
 
     // FIXME need to verify whether abort needs to be called if commit fails
     Assert.assertEquals(0, committer.abortCounter);
-    Assert.assertEquals(0, committer.initCounter); // already done in init
-    Assert.assertEquals(0, committer.setupCounter); // already done in init
+    Assert.assertEquals(1, committer.initCounter);
+    Assert.assertEquals(1, committer.setupCounter);
   }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/pom.xml b/tez-mapreduce-examples/pom.xml
index aa920d7..372cec8 100644
--- a/tez-mapreduce-examples/pom.xml
+++ b/tez-mapreduce-examples/pom.xml
@@ -63,7 +63,12 @@
      </dependency>
      <dependency>
        <groupId>org.apache.tez</groupId>
-       <artifactId>tez-dag</artifactId>
+       <artifactId>tez-api</artifactId>
+       <scope>provided</scope>
+     </dependency>
+     <dependency>
+       <groupId>org.apache.tez</groupId>
+       <artifactId>tez-mapreduce</artifactId>
        <scope>provided</scope>
      </dependency>
   </dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 8c37683..f90cc58 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -67,6 +67,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
 import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
 import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
 import org.apache.tez.mapreduce.hadoop.InputSplitInfo;
@@ -223,9 +224,9 @@ public class FilterLinesByWord {
     stage2Vertex.setTaskEnvironment(stage2Env);
 
     // Configure the Output for stage2
-    stage2Vertex.addOutput("MROutput",
-        new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
-            .createUserPayloadFromConf(stage2Conf)));
+    OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
+        .setUserPayload(MRHelpers.createUserPayloadFromConf(stage2Conf));
+    stage2Vertex.addOutput("MROutput", od, MROutputCommitter.class);
 
     DAG dag = new DAG("FilterLinesByWord");
     Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 9c05599..5d59d8c 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -61,6 +61,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
 import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
 import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
 import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
@@ -215,7 +216,8 @@ public class FilterLinesByWordOneToOne {
     // Configure the Output for stage2
     stage2Vertex.addOutput("MROutput",
         new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
-            .createUserPayloadFromConf(stage2Conf)));
+            .createUserPayloadFromConf(stage2Conf)),
+        MROutputCommitter.class);
 
     DAG dag = new DAG("FilterLinesByWord");
     Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
index bb48e3f..7646c65 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
@@ -64,6 +64,7 @@ import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
 import org.apache.tez.dag.api.client.DAGClient;
 import org.apache.tez.dag.api.client.DAGStatus;
 import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
 import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
 import org.apache.tez.mapreduce.hadoop.MRHelpers;
 import org.apache.tez.mapreduce.hadoop.MRJobConfig;
@@ -227,7 +228,7 @@ public class WordCount {
     summerVertex.setTaskEnvironment(reduceEnv);
     OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
         .setUserPayload(finalReducePayload);
-    summerVertex.addOutput("MROutput", od);
+    summerVertex.addOutput("MROutput", od, MROutputCommitter.class);
     
     DAG dag = new DAG("WordCount");
     dag.addVertex(tokenizerVertex)

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
new file mode 100644
index 0000000..7e474c1
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.committer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRHelpers;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+
+import java.io.IOException;
+
+public class MROutputCommitter extends OutputCommitter {
+
+  private static final Log LOG = LogFactory.getLog(MROutputCommitter.class);
+
+  private org.apache.hadoop.mapreduce.OutputCommitter committer = null;
+  private JobContext jobContext = null;
+  private volatile boolean initialized = false;
+  private JobConf jobConf = null;
+
+  @Override
+  public void initialize(OutputCommitterContext context) throws IOException {
+    byte[] userPayload = context.getUserPayload();
+    if (userPayload == null) {
+      jobConf = new JobConf();
+    } else {
+      jobConf = new JobConf(
+          MRHelpers.createConfFromUserPayload(context.getUserPayload()));
+    }
+
+    jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+        context.getDAGAttemptNumber());
+    committer = getOutputCommitter(context);
+    jobContext = getJobContextFromVertexContext(context);
+    initialized = true;
+  }
+
+  @Override
+  public void setupOutput() throws IOException {
+    if (!initialized) {
+      throw new RuntimeException("Committer not initialized");
+    }
+    committer.setupJob(jobContext);
+  }
+
+  @Override
+  public void commitOutput() throws IOException {
+    if (!initialized) {
+      throw new RuntimeException("Committer not initialized");
+    }
+    committer.commitJob(jobContext);
+  }
+
+  @Override
+  public void abortOutput(VertexStatus.State finalState) throws IOException {
+    if (!initialized) {
+      throw new RuntimeException("Committer not initialized");
+    }
+    JobStatus.State jobState = getJobStateFromVertexStatusState(finalState);
+    committer.abortJob(jobContext, jobState);
+  }
+
+  private org.apache.hadoop.mapreduce.OutputCommitter
+      getOutputCommitter(OutputCommitterContext context) {
+
+    org.apache.hadoop.mapreduce.OutputCommitter committer = null;
+    boolean newApiCommitter = false;
+    if (jobConf.getBoolean("mapred.reducer.new-api", false)
+        || jobConf.getBoolean("mapred.mapper.new-api", false))  {
+      newApiCommitter = true;
+      LOG.info("Using mapred newApiCommitter.");
+    }
+
+    LOG.info("OutputCommitter set in config for outputName="
+        + context.getOutputName()
+        + ", vertexName=" + context.getVertexName()
+        + ", outputCommitterClass="
+        + jobConf.get("mapred.output.committer.class"));
+
+    if (newApiCommitter) {
+      TaskAttemptID taskAttemptID = new TaskAttemptID(
+          Long.toString(context.getApplicationId().getClusterTimestamp()),
+          context.getApplicationId().getId(),
+          (jobConf.getBoolean(MRConfig.IS_MAP_PROCESSOR,
+              false) ? TaskType.MAP : TaskType.REDUCE),
+          0, context.getDAGAttemptNumber());
+
+      TaskAttemptContext taskContext = new TaskAttemptContextImpl(jobConf,
+          taskAttemptID);
+      try {
+        OutputFormat outputFormat = ReflectionUtils.newInstance(taskContext
+            .getOutputFormatClass(), jobConf);
+        committer = outputFormat.getOutputCommitter(taskContext);
+      } catch (Exception e) {
+        throw new TezUncheckedException(e);
+      }
+    } else {
+      committer = ReflectionUtils.newInstance(jobConf.getClass(
+          "mapred.output.committer.class", FileOutputCommitter.class,
+          org.apache.hadoop.mapred.OutputCommitter.class), jobConf);
+    }
+    LOG.info("OutputCommitter for outputName="
+        + context.getOutputName()
+        + ", vertexName=" + context.getVertexName()
+        + ", outputCommitterClass="
+        + committer.getClass().getName());
+    return committer;
+  }
+
+  // FIXME we are using ApplicationId as DAG id
+  private JobContext getJobContextFromVertexContext(OutputCommitterContext context)
+      throws IOException {
+    JobID jobId = TypeConverter.fromYarn(
+        context.getApplicationId());
+    return new MRJobContextImpl(jobConf, jobId);
+  }
+
+  private JobStatus.State getJobStateFromVertexStatusState(VertexStatus.State state) {
+    switch(state) {
+      case INITED:
+        return JobStatus.State.PREP;
+      case RUNNING:
+        return JobStatus.State.RUNNING;
+      case SUCCEEDED:
+        return JobStatus.State.SUCCEEDED;
+      case KILLED:
+        return JobStatus.State.KILLED;
+      case FAILED:
+      case ERROR:
+        return JobStatus.State.FAILED;
+      default:
+        throw new TezUncheckedException("Unknown VertexStatus.State: " + state);
+    }
+  }
+
+  private static class MRJobContextImpl
+      extends org.apache.hadoop.mapred.JobContextImpl {
+
+    public MRJobContextImpl(JobConf jobConf, JobID jobId) {
+      super(jobConf, jobId);
+    }
+
+  }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index af1f9e6..41cd7c9 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -71,6 +71,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.Vertex;
 import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
 import org.apache.tez.mapreduce.combine.MRCombiner;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
 import org.apache.tez.mapreduce.input.MRInputLegacy;
 import org.apache.tez.mapreduce.output.MROutput;
 import org.apache.tez.mapreduce.partition.MRPartitioner;
@@ -981,7 +982,7 @@ public class MRHelpers {
   public static void addMROutput(Vertex vertex, byte[] userPayload) {
     OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
         .setUserPayload(userPayload);
-    vertex.addOutput("MROutput", od);
+    vertex.addOutput("MROutput", od, MROutputCommitter.class);
   }
   
   @SuppressWarnings("unchecked")