You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/01/09 02:39:07 UTC
[1/2] git commit: TEZ-685. Add archive link to mail list page.
(hitesh)
Updated Branches:
refs/heads/master 25615ff32 -> bd23c9d4d
TEZ-685. Add archive link to mail list page. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/35a523b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/35a523b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/35a523b1
Branch: refs/heads/master
Commit: 35a523b177e0b96eabc9ca5643bbdc52ab0ebf4f
Parents: 25615ff
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Dec 26 10:38:56 2013 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Jan 8 17:35:50 2014 -0800
----------------------------------------------------------------------
docs/pom.xml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/35a523b1/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index e568e64..1bea40f 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -54,7 +54,7 @@
<unsubscribe>mailto:user-unsubscribe@tez.incubator.apache.org
</unsubscribe>
<post>mailto:user@tez.incubator.apache.org</post>
- <archive></archive>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-user/</archive>
</mailingList>
<mailingList>
<name>Development list</name>
@@ -63,7 +63,7 @@
<unsubscribe>mailto:dev-unsubscribe@tez.incubator.apache.org
</unsubscribe>
<post>mailto:dev@tez.incubator.apache.org</post>
- <archive></archive>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-dev/</archive>
</mailingList>
<mailingList>
<name>Commit list</name>
@@ -72,7 +72,7 @@
<unsubscribe>mailto:commits-unsubscribe@tez.incubator.apache.org
</unsubscribe>
<post>mailto:commits@tez.incubator.apache.org</post>
- <archive></archive>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-commits/</archive>
</mailingList>
<mailingList>
<name>JIRA Issues list</name>
@@ -81,7 +81,7 @@
<unsubscribe>mailto:issues-unsubscribe@tez.incubator.apache.org
</unsubscribe>
<post>mailto:issues@tez.incubator.apache.org</post>
- <archive></archive>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-tez-issues/</archive>
</mailingList>
</mailingLists>
[2/2] git commit: TEZ-41. Get VertexCommitter from API and remove
MRVertexOutputCommitter. (hitesh)
Posted by hi...@apache.org.
TEZ-41. Get VertexCommitter from API and remove MRVertexOutputCommitter. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/bd23c9d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/bd23c9d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/bd23c9d4
Branch: refs/heads/master
Commit: bd23c9d4d88ee3adf503503faf0ff1bb4305726c
Parents: 35a523b
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Jan 8 17:37:24 2014 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Wed Jan 8 17:37:24 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 4 +
.../apache/tez/dag/api/RootInputLeafOutput.java | 3 +-
.../java/org/apache/tez/dag/api/Vertex.java | 15 +-
.../apache/tez/runtime/api/OutputCommitter.java | 71 ++++
.../tez/runtime/api/OutputCommitterContext.java | 68 ++++
.../apache/hadoop/CustomOutputCommitter.java | 85 -----
.../hadoop/mapred/MRVertexOutputCommitter.java | 174 ---------
.../committer/NullVertexOutputCommitter.java | 47 ---
.../tez/dag/api/committer/VertexContext.java | 60 ---
.../api/committer/VertexOutputCommitter.java | 70 ----
.../dag/impl/OutputCommitterContextImpl.java | 77 ++++
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 133 ++++---
.../rm/container/ContainerSignatureMatcher.java | 17 +
.../tez/dag/app/dag/impl/TestVertexImpl.java | 361 ++++++++++++-------
tez-mapreduce-examples/pom.xml | 7 +-
.../mapreduce/examples/FilterLinesByWord.java | 7 +-
.../examples/FilterLinesByWordOneToOne.java | 4 +-
.../tez/mapreduce/examples/WordCount.java | 3 +-
.../mapreduce/committer/MROutputCommitter.java | 180 +++++++++
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 3 +-
20 files changed, 749 insertions(+), 640 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 69f648b..6d5035b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,10 @@ Apache Tez Change Log
Release 0.3.0 - Unreleased
+INCOMPATIBLE CHANGES
+
+ TEZ-41. Get VertexCommitter from API and remove MRVertexOutputCommitter.
+
Release 0.2.0 - 2013-11-30
First version.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
index 1334c41..11929b9 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java
@@ -26,7 +26,7 @@ class RootInputLeafOutput <T extends TezEntityDescriptor> {
private final String name;
private final T descriptor;
private final Class<?> initializerClazz;
-
+
RootInputLeafOutput(String name, T descriptor, Class<?> initializerClazz) {
this.name = name;
this.descriptor = descriptor;
@@ -44,4 +44,5 @@ class RootInputLeafOutput <T extends TezEntityDescriptor> {
public Class<?> getInitializerClass() {
return this.initializerClazz;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 17fa558..4037fe5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -27,10 +27,11 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.runtime.api.LogicalIOProcessor;
+import org.apache.tez.runtime.api.OutputCommitter;
import org.apache.tez.runtime.api.TezRootInputInitializer;
import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
-public class Vertex { // FIXME rename to Task
+public class Vertex {
private final String vertexName;
private final ProcessorDescriptor processorDescriptor;
@@ -176,15 +177,21 @@ public class Vertex { // FIXME rename to Task
* the name of the output. This will be used when accessing the
* output in the {@link LogicalIOProcessor}
* @param outputDescriptor
+ * @param outputCommitterClazz Class to be used for the OutputCommitter.
+ * Can be null.
* @return
*/
- // TODO : Add a processing component.
- public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor) {
+ public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor,
+ Class<? extends OutputCommitter> outputCommitterClazz) {
additionalOutputs.add(new RootInputLeafOutput<OutputDescriptor>(outputName,
- outputDescriptor, null));
+ outputDescriptor, outputCommitterClazz));
return this;
}
+ public Vertex addOutput(String outputName, OutputDescriptor outputDescriptor) {
+ return addOutput(outputName, outputDescriptor, null);
+ }
+
public String getJavaOpts(){
return javaOpts;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
new file mode 100644
index 0000000..f0cee4e
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import org.apache.tez.dag.api.client.VertexStatus;
+
+import java.io.IOException;
+
+/**
+ * OutputCommitter to "finalize" the output and make it user-visible if needed.
+ * The OutputCommitter is allowed only on a terminal Output.
+ */
+public abstract class OutputCommitter {
+
+ /**
+ * Setup up the Output committer.
+ *
+ * @param context Context of the output that is being acted upon
+ * @throws java.lang.Exception
+ */
+ public abstract void initialize(OutputCommitterContext context)
+ throws Exception;
+
+ /**
+ * For the framework to setup the output during initialization. This is
+ * called from the application master process for the vertex. This will be
+ * called multiple times, once per dag attempt.
+ *
+ * @throws java.lang.Exception if setup fails
+ */
+ public abstract void setupOutput() throws Exception;
+
+ /**
+ * For committing the output after a successful vertex completion.
+ * Note that this is invoked for the outputs of vertices with a successful
+ * final state. This is called from the application master process.
+ * This is guaranteed to only be called once.
+ * If it throws an exception the entire vertex will fail.
+ *
+ * @throws java.lang.Exception
+ */
+ public abstract void commitOutput() throws Exception;
+
+ /**
+ * For aborting an unsuccessful vertex's output. Note that this is invoked for
+ * vertices with a final failed state. This is called from the application
+ * master process. This may be called multiple times.
+ *
+ * @param finalState final run-state of the vertex
+ * @throws java.lang.Exception
+ */
+ public abstract void abortOutput(VertexStatus.State finalState)
+ throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
new file mode 100644
index 0000000..2136073
--- /dev/null
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import java.util.List;
+
+/**
+ * Context through which the OutputCommitter can access all the relevant
+ * information that it needs.
+ */
+
+public interface OutputCommitterContext {
+
+ /**
+ * Get the {@link org.apache.hadoop.yarn.api.records.ApplicationId}
+ * @return the {@link org.apache.hadoop.yarn.api.records.ApplicationId}
+ */
+ public ApplicationId getApplicationId();
+
+ /**
+ * Get the current DAG Attempt Number
+ * @return DAG Attempt Number
+ */
+ public int getDAGAttemptNumber();
+
+ /**
+ * Get the name of the DAG
+ * @return the DAG name
+ */
+ public String getDAGName();
+
+ /**
+ * Get the name of the Vertex
+ * @return Vertex Name
+ */
+ public String getVertexName();
+
+ /**
+ * Get the name of the Output
+ * @return Output Name
+ */
+ public String getOutputName();
+
+ /**
+ * Get the User Payload for the Output
+ * @return User Payload
+ */
+ public byte[] getUserPayload();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
deleted file mode 100644
index f151ced..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/CustomOutputCommitter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
-import org.apache.hadoop.mapred.OutputCommitter;
-import org.apache.hadoop.mapred.TaskAttemptContext;
-
-public class CustomOutputCommitter extends OutputCommitter {
-
- public static final String JOB_SETUP_FILE_NAME = "_job_setup";
- public static final String JOB_COMMIT_FILE_NAME = "_job_commit";
- public static final String JOB_ABORT_FILE_NAME = "_job_abort";
- public static final String TASK_SETUP_FILE_NAME = "_task_setup";
- public static final String TASK_ABORT_FILE_NAME = "_task_abort";
- public static final String TASK_COMMIT_FILE_NAME = "_task_commit";
-
- @Override
- public void setupJob(JobContext jobContext) throws IOException {
- writeFile(jobContext.getJobConf(), JOB_SETUP_FILE_NAME);
- }
-
- @Override
- public void commitJob(JobContext jobContext) throws IOException {
- super.commitJob(jobContext);
- writeFile(jobContext.getJobConf(), JOB_COMMIT_FILE_NAME);
- }
-
- @Override
- public void abortJob(JobContext jobContext, int status)
- throws IOException {
- super.abortJob(jobContext, status);
- writeFile(jobContext.getJobConf(), JOB_ABORT_FILE_NAME);
- }
-
- @Override
- public void setupTask(TaskAttemptContext taskContext) throws IOException {
- writeFile(taskContext.getJobConf(), TASK_SETUP_FILE_NAME);
- }
-
- @Override
- public boolean needsTaskCommit(TaskAttemptContext taskContext)
- throws IOException {
- return true;
- }
-
- @Override
- public void commitTask(TaskAttemptContext taskContext) throws IOException {
- writeFile(taskContext.getJobConf(), TASK_COMMIT_FILE_NAME);
- }
-
- @Override
- public void abortTask(TaskAttemptContext taskContext) throws IOException {
- writeFile(taskContext.getJobConf(), TASK_ABORT_FILE_NAME);
- }
-
- private void writeFile(JobConf conf , String filename) throws IOException {
- System.out.println("writing file ----" + filename);
- Path outputPath = FileOutputFormat.getOutputPath(conf);
- FileSystem fs = outputPath.getFileSystem(conf);
- fs.create(new Path(outputPath, filename)).close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
deleted file mode 100644
index 14a0c52..0000000
--- a/tez-dag/src/main/java/org/apache/hadoop/mapred/MRVertexOutputCommitter.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.JobStatus.State;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tez.dag.api.TezUncheckedException;
-import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
-import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.dag.records.TezTaskID;
-import org.apache.tez.dag.utils.TezBuilderUtils;
-import org.apache.tez.mapreduce.hadoop.MRHelpers;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
-
-public class MRVertexOutputCommitter extends VertexOutputCommitter {
-
- private static final Log LOG = LogFactory.getLog(
- MRVertexOutputCommitter.class);
-
- private OutputCommitter committer = null;
- private JobContext jobContext = null;
- private volatile boolean initialized = false;
- private JobConf jobConf = null;
-
- public MRVertexOutputCommitter() {
- }
-
- @SuppressWarnings("rawtypes")
- private OutputCommitter getOutputCommitter(VertexContext context) {
-
- OutputCommitter committer = null;
- boolean newApiCommitter = false;
- if (jobConf.getBoolean("mapred.reducer.new-api", false)
- || jobConf.getBoolean("mapred.mapper.new-api", false)) {
- newApiCommitter = true;
- LOG.info("Using mapred newApiCommitter.");
- }
-
- LOG.info("OutputCommitter set in config for vertex: "
- + context.getVertexID() + " : "
- + jobConf.get("mapred.output.committer.class"));
-
- if (newApiCommitter) {
- TezTaskID taskId = TezBuilderUtils.newTaskId(context.getDAGID(),
- context.getVertexID().getId(), 0);
- TezTaskAttemptID attemptID =
- TezBuilderUtils.newTaskAttemptId(taskId, 0);
- TaskAttemptContext taskContext = new TaskAttemptContextImpl(jobConf,
- TezMRTypeConverter.fromTez(attemptID));
- try {
- OutputFormat outputFormat = ReflectionUtils.newInstance(taskContext
- .getOutputFormatClass(), jobConf);
- committer = outputFormat.getOutputCommitter(taskContext);
- } catch (Exception e) {
- throw new TezUncheckedException(e);
- }
- } else {
- committer = ReflectionUtils.newInstance(jobConf.getClass(
- "mapred.output.committer.class", FileOutputCommitter.class,
- org.apache.hadoop.mapred.OutputCommitter.class), jobConf);
- }
- LOG.info("OutputCommitter is " + committer.getClass().getName());
- return committer;
- }
-
- // FIXME we are using ApplicationId as DAG id
- private JobContext getJobContextFromVertexContext(VertexContext context)
- throws IOException {
- JobID jobId = TypeConverter.fromYarn(
- context.getDAGID().getApplicationId());
- return new MRJobContextImpl(jobConf, jobId);
- }
-
- private State getJobStateFromVertexStatusState(VertexStatus.State state) {
- switch(state) {
- case INITED:
- return JobStatus.State.PREP;
- case RUNNING:
- return JobStatus.State.RUNNING;
- case SUCCEEDED:
- return JobStatus.State.SUCCEEDED;
- case KILLED:
- return JobStatus.State.KILLED;
- case FAILED:
- case ERROR:
- return JobStatus.State.FAILED;
- default:
- throw new TezUncheckedException("Unknown VertexStatus.State: " + state);
- }
- }
-
- @Override
- public void init(VertexContext context) throws IOException {
- // TODO VertexContext not the best way to get ApplicationAttemptId. No
- // alternates right now.
-
- byte[] userPayload = context.getUserPayload();
- if (userPayload == null) {
- jobConf = new JobConf();
- } else {
- jobConf = new JobConf(
- MRHelpers.createConfFromUserPayload(context.getUserPayload()));
- }
-
- jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
- context.getApplicationAttemptId().getAttemptId());
- committer = getOutputCommitter(context);
- jobContext = getJobContextFromVertexContext(context);
- initialized = true;
- }
-
- @Override
- public void setupVertex() throws IOException {
- if (!initialized) {
- throw new RuntimeException("Committer not initialized");
- }
- committer.setupJob(jobContext);
- }
-
- @Override
- public void commitVertex() throws IOException {
- if (!initialized) {
- throw new RuntimeException("Committer not initialized");
- }
- committer.commitJob(jobContext);
- }
-
- @Override
- public void abortVertex(VertexStatus.State finalState) throws IOException {
- if (!initialized) {
- throw new RuntimeException("Committer not initialized");
- }
- State jobState = getJobStateFromVertexStatusState(finalState);
- committer.abortJob(jobContext, jobState);
- }
-
- private static class MRJobContextImpl
- extends org.apache.hadoop.mapred.JobContextImpl {
-
- public MRJobContextImpl(JobConf jobConf, JobID jobId) {
- super(jobConf, jobId);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
deleted file mode 100644
index 08b2d4a..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/NullVertexOutputCommitter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-import java.io.IOException;
-
-import org.apache.tez.dag.api.client.VertexStatus;
-
-public class NullVertexOutputCommitter extends VertexOutputCommitter {
-
- @Override
- public void init(VertexContext context) throws IOException {
- // Nothing to do
- }
-
- @Override
- public void setupVertex() throws IOException {
- // Nothing to do
- }
-
- @Override
- public void commitVertex() throws IOException {
- // Nothing to do
- }
-
- @Override
- public void abortVertex(VertexStatus.State finalState) throws IOException {
- // Nothing to do
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
deleted file mode 100644
index 5c4430e..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexContext.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.tez.dag.records.TezDAGID;
-import org.apache.tez.dag.records.TezVertexID;
-
-public class VertexContext {
-
- private final TezDAGID tezDAGID;
- private final byte[] userPayload;
- private final ApplicationAttemptId applicationAttemptId;
- private final TezVertexID tezVertexID;
-
- public VertexContext(TezDAGID tezDAGID, byte[] userPayload,
- TezVertexID tezVertexID,
- ApplicationAttemptId applicationAttemptId) {
- this.tezDAGID = tezDAGID;
- this.userPayload = userPayload;
- this.tezVertexID = tezVertexID;
- this.applicationAttemptId = applicationAttemptId;
- }
-
- public TezDAGID getDAGID() {
- return tezDAGID;
- }
-
- public byte[] getUserPayload() {
- return userPayload;
- }
-
- // TODO get rid of this as part of VertexContext cleanup
- public ApplicationAttemptId getApplicationAttemptId() {
- return applicationAttemptId;
- }
-
- public TezVertexID getVertexID() {
- return tezVertexID;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java b/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
deleted file mode 100644
index 34c6e89..0000000
--- a/tez-dag/src/main/java/org/apache/tez/dag/api/committer/VertexOutputCommitter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tez.dag.api.committer;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.tez.dag.api.client.VertexStatus;
-
-@Public
-@Unstable
-public abstract class VertexOutputCommitter {
-
- /**
- * Setup up the vertex output committer.
- *
- * @param context Context of the vertex whose output is being written.
- * @throws IOException
- */
- public abstract void init(VertexContext context) throws IOException;
-
- /**
- * For the framework to setup the vertex output during initialization. This is
- * called from the application master process for the vertex. This will be
- * called multiple times, once per dag attempt for each vertex.
- *
- * @throws IOException if setup fails
- */
- public abstract void setupVertex() throws IOException;
-
- /**
- * For committing vertex's output after successful vertex completion.
- * Note that this is invoked for vertices with a successful final state.
- * This is called from the application master process for the entire vertex.
- * This is guaranteed to only be called once.
- * If it throws an exception the entire vertex will fail.
- *
- * @throws IOException
- */
- public abstract void commitVertex() throws IOException;
-
- /**
- * For aborting an unsuccessful vertex's output. Note that this is invoked for
- * vertices with a final failed state. This is called from the application
- * master process for the entire vertex. This may be called multiple times.
- *
- * @param state final runstate of the vertex
- * @throws IOException
- */
- public abstract void abortVertex(VertexStatus.State finalState)
- throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
new file mode 100644
index 0000000..68c8835
--- /dev/null
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.app.dag.impl;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+
+public class OutputCommitterContextImpl implements OutputCommitterContext {
+
+ private final ApplicationId applicationId;
+ private final int dagAttemptNumber;
+ private final String dagName;
+ private final String vertexName;
+ private final String outputName;
+ private final byte[] userPayload;
+
+ public OutputCommitterContextImpl(ApplicationId applicationId,
+ int dagAttemptNumber,
+ String dagName,
+ String vertexName,
+ String outputName,
+ byte[] userPayload) {
+ this.applicationId = applicationId;
+ this.dagAttemptNumber = dagAttemptNumber;
+ this.dagName = dagName;
+ this.vertexName = vertexName;
+ this.outputName = outputName;
+ this.userPayload = userPayload;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ @Override
+ public int getDAGAttemptNumber() {
+ return dagAttemptNumber;
+ }
+
+ @Override
+ public String getDAGName() {
+ return dagName;
+ }
+
+ @Override
+ public String getVertexName() {
+ return vertexName;
+ }
+
+ @Override
+ public String getOutputName() {
+ return outputName;
+ }
+
+ @Override
+ public byte[] getUserPayload() {
+ return userPayload;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 2672609..f77521f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -37,11 +37,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.MRVertexOutputCommitter;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -65,9 +65,6 @@ import org.apache.tez.dag.api.client.ProgressBuilder;
import org.apache.tez.dag.api.client.StatusGetOpts;
import org.apache.tez.dag.api.client.VertexStatus;
import org.apache.tez.dag.api.client.VertexStatusBuilder;
-import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto;
import org.apache.tez.dag.api.records.DAGProtos.VertexPlan;
@@ -116,6 +113,9 @@ import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.apache.tez.runtime.RuntimeUtils;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.InputFailedEvent;
import org.apache.tez.runtime.api.events.TaskAttemptFailedEvent;
@@ -437,15 +437,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private final String vertexName;
private final ProcessorDescriptor processorDescriptor;
- // For committer
- private final VertexContext vertexContext;
-
@VisibleForTesting
Map<Vertex, Edge> sourceVertices;
private Map<Vertex, Edge> targetVertices;
private Map<String, RootInputLeafOutputDescriptor<InputDescriptor>> additionalInputs;
private Map<String, RootInputLeafOutputDescriptor<OutputDescriptor>> additionalOutputs;
+ private Map<String, OutputCommitter> outputCommitters;
private final List<InputSpec> additionalInputSpecs = new ArrayList<InputSpec>();
private final List<OutputSpec> additionalOutputSpecs = new ArrayList<OutputSpec>();
private Set<String> inputsWithInitializers;
@@ -461,7 +459,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
private boolean parallelismSet = false;
private TezVertexID originalOneToOneSplitSource = null;
- private VertexOutputCommitter committer;
private AtomicBoolean committed = new AtomicBoolean(false);
private VertexLocationHint vertexLocationHint;
private Map<String, LocalResource> localResources;
@@ -500,7 +497,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.writeLock = readWriteLock.writeLock();
this.credentials = credentials;
- this.committer = new NullVertexOutputCommitter();
this.vertexLocationHint = vertexLocationHint;
if (LOG.isDebugEnabled()) {
logLocationHints(this.vertexLocationHint);
@@ -521,10 +517,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
.getTaskConfig().getJavaOpts() : null;
- this.vertexContext = new VertexContext(getDAGId(),
- this.processorDescriptor.getUserPayload(), this.vertexId,
- getApplicationAttemptId());
-
this.containerContext = new ContainerContext(this.localResources,
this.credentials, this.environment, this.javaOpts, this);
@@ -1053,7 +1045,14 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// commit only once
LOG.info("Invoking committer commit for vertex, vertexId="
+ vertex.logIdentifier);
- vertex.committer.commitVertex();
+ if (vertex.outputCommitters != null) {
+ for (Entry<String, OutputCommitter> entry :
+ vertex.outputCommitters.entrySet()) {
+ LOG.info("Invoking committer commit for output=" + entry.getKey()
+ + ", vertexId=" + vertex.logIdentifier);
+ entry.getValue().commitOutput();
+ }
+ }
}
} catch (Exception e) {
LOG.error("Failed to do commit on vertex, vertexId="
@@ -1167,27 +1166,51 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// TODO TEZ-41 make commmitter type configurable per vertex
if (!this.additionalOutputSpecs.isEmpty()) {
- committer = new MRVertexOutputCommitter();
- }
- try {
- LOG.info("Invoking committer init for vertex, vertexId=" + logIdentifier);
- committer.init(vertexContext);
- LOG.info("Invoking committer setup for vertex, vertexId="
- + logIdentifier);
- committer.setupVertex();
- } catch (Exception e) {
- LOG.warn("Vertex init failed, vertexId=" + logIdentifier, e);
- addDiagnostic("Vertex init failed : "
- + StringUtils.stringifyException(e));
- trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
- abortVertex(VertexStatus.State.FAILED);
- // TODO: Metrics
- //job.metrics.endPreparingJob(vertex);
- return finished(VertexState.FAILED);
+ try {
+ LOG.info("Invoking committer inits for vertex, vertexId=" + logIdentifier);
+ for (Entry<String, RootInputLeafOutputDescriptor<OutputDescriptor>> entry:
+ additionalOutputs.entrySet()) {
+ final String outputName = entry.getKey();
+ final RootInputLeafOutputDescriptor<OutputDescriptor> od = entry.getValue();
+ if (od.getInitializerClassName() == null
+ || od.getInitializerClassName().isEmpty()) {
+ LOG.info("Ignoring committer as none specified for output="
+ + outputName
+ + ", vertexId=" + logIdentifier);
+ continue;
+ }
+ LOG.info("Instantiating committer for output=" + outputName
+ + ", vertexId=" + logIdentifier
+ + ", committerClass=" + od.getInitializerClassName());
+
+ OutputCommitter outputCommitter = RuntimeUtils.createClazzInstance(
+ od.getInitializerClassName());
+ OutputCommitterContext outputCommitterContext =
+ new OutputCommitterContextImpl(appContext.getApplicationID(),
+ appContext.getApplicationAttemptId().getAttemptId(),
+ appContext.getCurrentDAG().getName(),
+ vertexName,
+ outputName,
+ od.getDescriptor().getUserPayload());
+
+ LOG.info("Invoking committer init for output=" + outputName
+ + ", vertexId=" + logIdentifier);
+ outputCommitter.initialize(outputCommitterContext);
+ outputCommitters.put(outputName, outputCommitter);
+ LOG.info("Invoking committer setup for output=" + outputName
+ + ", vertexId=" + logIdentifier);
+ outputCommitter.setupOutput();
+ }
+ } catch (Exception e) {
+ LOG.warn("Vertex Committer init failed, vertexId=" + logIdentifier, e);
+ addDiagnostic("Vertex init failed : "
+ + StringUtils.stringifyException(e));
+ trySetTerminationCause(VertexTerminationCause.INIT_FAILURE);
+ abortVertex(VertexStatus.State.FAILED);
+ return finished(VertexState.FAILED);
+ }
}
-
// TODO: Metrics
- //vertex.metrics.endPreparingJob(job);
initedTime = clock.getTime();
return VertexState.INITED;
@@ -1588,14 +1611,20 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
private void abortVertex(VertexStatus.State finalState) {
- try {
- LOG.info("Invoking committer abort for vertex, vertexId="
- + logIdentifier);
- committer.abortVertex(finalState);
- } catch (Exception e) {
- LOG.warn("Could not abort vertex, vertexId=" + logIdentifier, e);
+ LOG.info("Invoking committer abort for vertex, vertexId="
+ + logIdentifier);
+ if (outputCommitters != null) {
+ for (Entry<String, OutputCommitter> entry : outputCommitters.entrySet()) {
+ try {
+ LOG.info("Invoking committer abort for output=" + entry.getKey()
+ + ", vertexId=" + logIdentifier);
+ entry.getValue().abortOutput(finalState);
+ } catch (Exception e) {
+ LOG.warn("Could not abort committer for output=" + entry.getKey()
+ + ", vertexId=" + logIdentifier, e);
+ }
+ }
}
-
if (finishTime == 0) {
setFinishTime();
}
@@ -1826,7 +1855,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
@Override
public VertexState transition(VertexImpl vertex, VertexEvent event) {
- if (vertex.committer instanceof NullVertexOutputCommitter) {
+ if (vertex.outputCommitters == null
+ || vertex.outputCommitters.isEmpty()) {
LOG.info(vertex.getVertexId() + " back to running due to rescheduling "
+ ((VertexEventTaskReschedule)event).getTaskID());
(new TaskRescheduledTransition()).transition(vertex, event);
@@ -2034,10 +2064,20 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
+ @Private
+ @VisibleForTesting
+ public OutputCommitter getOutputCommitter(String outputName) {
+ if (this.outputCommitters != null) {
+ return outputCommitters.get(outputName);
+ }
+ return null;
+ }
+
@Override
public void setAdditionalOutputs(List<RootInputLeafOutputProto> outputs) {
LOG.info("setting additional outputs for vertex " + this.vertexName);
this.additionalOutputs = Maps.newHashMapWithExpectedSize(outputs.size());
+ this.outputCommitters = Maps.newHashMapWithExpectedSize(outputs.size());
for (RootInputLeafOutputProto output : outputs) {
OutputDescriptor od = DagTypeConverters
@@ -2185,17 +2225,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
}
@VisibleForTesting
- VertexOutputCommitter getVertexOutputCommitter() {
- return this.committer;
- }
-
- @VisibleForTesting
- // Only to be used for testing
- void setVertexOutputCommitter(VertexOutputCommitter committer) {
- this.committer = committer;
- }
-
- @VisibleForTesting
VertexManager getVertexManager() {
return this.vertexManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
index 4e3f204..4d65c23 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerSignatureMatcher.java
@@ -1,3 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.tez.dag.app.rm.container;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index d0c26d4..40de46f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -23,16 +23,24 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.MRVertexOutputCommitter;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -49,9 +57,6 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.dag.api.client.VertexStatus;
-import org.apache.tez.dag.api.committer.NullVertexOutputCommitter;
-import org.apache.tez.dag.api.committer.VertexContext;
-import org.apache.tez.dag.api.committer.VertexOutputCommitter;
import org.apache.tez.dag.api.oldrecords.TaskState;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
@@ -98,6 +103,8 @@ import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
import org.apache.tez.runtime.api.events.RootInputConfigureVertexTasksEvent;
import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
import org.junit.After;
@@ -138,8 +145,8 @@ public class TestVertexImpl {
private VertexEventDispatcher vertexEventDispatcher;
private DagEventDispatcher dagEventDispatcher;
- private class CountingVertexOutputCommitter extends
- VertexOutputCommitter {
+ public static class CountingOutputCommitter extends
+ OutputCommitter {
public int initCounter = 0;
public int setupCounter = 0;
@@ -149,7 +156,7 @@ public class TestVertexImpl {
private boolean throwErrorOnAbort;
private boolean throwRuntimeException;
- public CountingVertexOutputCommitter(boolean throwError,
+ public CountingOutputCommitter(boolean throwError,
boolean throwOnAbort,
boolean throwRuntimeException) {
this.throwError = throwError;
@@ -157,22 +164,29 @@ public class TestVertexImpl {
this.throwRuntimeException = throwRuntimeException;
}
- public CountingVertexOutputCommitter() {
+ public CountingOutputCommitter() {
this(false, false, false);
}
@Override
- public void init(VertexContext context) throws IOException {
+ public void initialize(OutputCommitterContext context) throws IOException {
+ if (context.getUserPayload() != null) {
+ CountingOutputCommitterConfig conf =
+ new CountingOutputCommitterConfig(context.getUserPayload());
+ this.throwError = conf.throwError;
+ this.throwErrorOnAbort = conf.throwErrorOnAbort;
+ this.throwRuntimeException = conf.throwRuntimeException;
+ }
++initCounter;
}
@Override
- public void setupVertex() {
+ public void setupOutput() throws IOException {
++setupCounter;
}
@Override
- public void commitVertex() throws IOException {
+ public void commitOutput() throws IOException {
++commitCounter;
if (throwError) {
if (!throwRuntimeException) {
@@ -184,7 +198,7 @@ public class TestVertexImpl {
}
@Override
- public void abortVertex(VertexStatus.State finalState) throws IOException {
+ public void abortOutput(VertexStatus.State finalState) throws IOException {
++abortCounter;
if (throwErrorOnAbort) {
if (!throwRuntimeException) {
@@ -194,6 +208,51 @@ public class TestVertexImpl {
}
}
}
+
+ public static class CountingOutputCommitterConfig implements Writable {
+
+ boolean throwError = false;
+ boolean throwErrorOnAbort = false;
+ boolean throwRuntimeException = false;
+
+ public CountingOutputCommitterConfig() {
+ }
+
+ public CountingOutputCommitterConfig(boolean throwError,
+ boolean throwErrorOnAbort, boolean throwRuntimeException) {
+ this.throwError = throwError;
+ this.throwErrorOnAbort = throwErrorOnAbort;
+ this.throwRuntimeException = throwRuntimeException;
+ }
+
+ public CountingOutputCommitterConfig(byte[] payload) throws IOException {
+ DataInput in = new DataInputStream(new ByteArrayInputStream(payload));
+ this.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(throwError);
+ out.writeBoolean(throwErrorOnAbort);
+ out.writeBoolean(throwRuntimeException);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ throwError = in.readBoolean();
+ throwErrorOnAbort = in.readBoolean();
+ throwRuntimeException = in.readBoolean();
+ }
+
+ public byte[] toUserPayload() throws IOException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(bos);
+ write(out);
+ return bos.toByteArray();
+ }
+
+ }
+
}
private class TaskAttemptEventDispatcher implements EventHandler<TaskAttemptEvent> {
@@ -605,106 +664,107 @@ public class TestVertexImpl {
.build()
)
.addVertex(
- VertexPlan.newBuilder()
- .setName("vertex6")
- .setType(PlanVertexType.NORMAL)
- .addTaskLocationHint(
- PlanTaskLocationHint.newBuilder()
- .addHost("host6")
- .addRack("rack6")
- .build()
- )
- .addOutputs(
- DAGProtos.RootInputLeafOutputProto.newBuilder()
- .setEntityDescriptor(
- TezEntityDescriptorProto.newBuilder().setClassName("output").build()
- )
- .setName("outputx")
- )
- .setTaskConfig(
- PlanTaskConfiguration.newBuilder()
- .setNumTasks(2)
- .setVirtualCores(4)
- .setMemoryMb(1024)
- .setJavaOpts("")
- .setTaskModule("x6.y6")
- .build()
+ VertexPlan.newBuilder()
+ .setName("vertex6")
+ .setType(PlanVertexType.NORMAL)
+ .addTaskLocationHint(
+ PlanTaskLocationHint.newBuilder()
+ .addHost("host6")
+ .addRack("rack6")
+ .build()
+ )
+ .addOutputs(
+ DAGProtos.RootInputLeafOutputProto.newBuilder()
+ .setEntityDescriptor(
+ TezEntityDescriptorProto.newBuilder().setClassName("output").build()
+ )
+ .setName("outputx")
+ .setInitializerClassName(CountingOutputCommitter.class.getName())
+ )
+ .setTaskConfig(
+ PlanTaskConfiguration.newBuilder()
+ .setNumTasks(2)
+ .setVirtualCores(4)
+ .setMemoryMb(1024)
+ .setJavaOpts("")
+ .setTaskModule("x6.y6")
+ .build()
)
.addInEdgeId("e5")
.addInEdgeId("e6")
.build()
- )
+ )
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1"))
- .setInputVertexName("vertex1")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
- .setOutputVertexName("vertex3")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e1")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1"))
+ .setInputVertexName("vertex1")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1"))
+ .setOutputVertexName("vertex3")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e1")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2"))
- .setInputVertexName("vertex2")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2"))
- .setOutputVertexName("vertex3")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e2")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2"))
+ .setInputVertexName("vertex2")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2"))
+ .setOutputVertexName("vertex3")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e2")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3"))
- .setInputVertexName("vertex3")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4"))
- .setOutputVertexName("vertex4")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e3")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3"))
+ .setInputVertexName("vertex3")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4"))
+ .setOutputVertexName("vertex4")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e3")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3"))
- .setInputVertexName("vertex3")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5"))
- .setOutputVertexName("vertex5")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e4")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3"))
+ .setInputVertexName("vertex3")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5"))
+ .setOutputVertexName("vertex5")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e4")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4"))
- .setInputVertexName("vertex4")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4"))
- .setOutputVertexName("vertex6")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e5")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4"))
+ .setInputVertexName("vertex4")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4"))
+ .setOutputVertexName("vertex6")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e5")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.addEdge(
- EdgePlan.newBuilder()
- .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5"))
- .setInputVertexName("vertex5")
- .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5"))
- .setOutputVertexName("vertex6")
- .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
- .setId("e6")
- .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
- .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
- .build()
+ EdgePlan.newBuilder()
+ .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5"))
+ .setInputVertexName("vertex5")
+ .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5"))
+ .setOutputVertexName("vertex6")
+ .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER)
+ .setId("e6")
+ .setDataSourceType(PlanEdgeDataSourceType.PERSISTED)
+ .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL)
+ .build()
)
.build();
@@ -1382,12 +1442,11 @@ public class TestVertexImpl {
public void testVertexCommitterInit() {
initAllVertices(VertexState.INITED);
VertexImpl v2 = vertices.get("vertex2");
- Assert.assertTrue(v2.getVertexOutputCommitter()
- instanceof NullVertexOutputCommitter);
+ Assert.assertNull(v2.getOutputCommitter("output"));
VertexImpl v6 = vertices.get("vertex6");
- Assert.assertTrue(v6.getVertexOutputCommitter()
- instanceof MRVertexOutputCommitter);
+ Assert.assertTrue(v6.getOutputCommitter("outputx")
+ instanceof CountingOutputCommitter);
}
@Test(timeout = 5000)
@@ -1407,12 +1466,11 @@ public class TestVertexImpl {
public void testVertexTaskFailure() {
initAllVertices(VertexState.INITED);
- VertexImpl v = vertices.get("vertex2");
+ VertexImpl v = vertices.get("vertex6");
- CountingVertexOutputCommitter committer =
- new CountingVertexOutputCommitter();
- v.setVertexOutputCommitter(committer);
startVertex(v);
+ CountingOutputCommitter committer =
+ (CountingOutputCommitter) v.getOutputCommitter("outputx");
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1543,10 +1601,6 @@ public class TestVertexImpl {
VertexImpl v = vertices.get("vertex2");
- CountingVertexOutputCommitter committer =
- new CountingVertexOutputCommitter();
- v.setVertexOutputCommitter(committer);
-
startVertex(v);
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1562,13 +1616,11 @@ public class TestVertexImpl {
new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.RUNNING, v.getState());
- Assert.assertEquals(0, committer.commitCounter);
dispatcher.getEventHandler().handle(
new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED));
dispatcher.await();
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
- Assert.assertEquals(1, committer.commitCounter);
}
@@ -1580,8 +1632,6 @@ public class TestVertexImpl {
VertexImpl v = vertices.get("vertex2");
- v.setVertexOutputCommitter(new NullVertexOutputCommitter());
-
startVertex(v);
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1617,14 +1667,24 @@ public class TestVertexImpl {
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
- public void testVertexSuccessToFailedAfterTaskScheduler() {
+ public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception {
// For downstream failures
- initAllVertices(VertexState.INITED);
-
VertexImpl v = vertices.get("vertex2");
- v.setVertexOutputCommitter(new CountingVertexOutputCommitter());
+ List<RootInputLeafOutputProto> outputs =
+ new ArrayList<RootInputLeafOutputProto>();
+ outputs.add(RootInputLeafOutputProto.newBuilder()
+ .setInitializerClassName(CountingOutputCommitter.class.getName())
+ .setName("output_v2")
+ .setEntityDescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
+ new CountingOutputCommitter.CountingOutputCommitterConfig()
+ .toUserPayload())).build())
+ .build());
+ v.setAdditionalOutputs(outputs);
+ initAllVertices(VertexState.INITED);
startVertex(v);
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
@@ -1655,13 +1715,11 @@ public class TestVertexImpl {
public void testVertexCommit() {
initAllVertices(VertexState.INITED);
- VertexImpl v = vertices.get("vertex2");
-
- CountingVertexOutputCommitter committer =
- new CountingVertexOutputCommitter();
- v.setVertexOutputCommitter(committer);
+ VertexImpl v = vertices.get("vertex6");
startVertex(v);
+ CountingOutputCommitter committer =
+ (CountingOutputCommitter) v.getOutputCommitter("outputx");
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1682,8 +1740,8 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.SUCCEEDED, v.getState());
Assert.assertEquals(1, committer.commitCounter);
Assert.assertEquals(0, committer.abortCounter);
- Assert.assertEquals(0, committer.initCounter); // already done in init
- Assert.assertEquals(0, committer.setupCounter); // already done in init
+ Assert.assertEquals(1, committer.initCounter);
+ Assert.assertEquals(1, committer.setupCounter);
}
@Test(timeout = 5000)
@@ -1693,17 +1751,28 @@ public class TestVertexImpl {
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
- public void testBadCommitter() {
- initAllVertices(VertexState.INITED);
-
+ public void testBadCommitter() throws Exception {
VertexImpl v = vertices.get("vertex2");
- CountingVertexOutputCommitter committer =
- new CountingVertexOutputCommitter(true, true, false);
- v.setVertexOutputCommitter(committer);
+ List<RootInputLeafOutputProto> outputs =
+ new ArrayList<RootInputLeafOutputProto>();
+ outputs.add(RootInputLeafOutputProto.newBuilder()
+ .setInitializerClassName(CountingOutputCommitter.class.getName())
+ .setName("output_v2")
+ .setEntityDescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
+ new CountingOutputCommitter.CountingOutputCommitterConfig(
+ true, true, false).toUserPayload())).build())
+ .build());
+ v.setAdditionalOutputs(outputs);
+ initAllVertices(VertexState.INITED);
startVertex(v);
+ CountingOutputCommitter committer =
+ (CountingOutputCommitter) v.getOutputCommitter("output_v2");
+
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1714,27 +1783,39 @@ public class TestVertexImpl {
dispatcher.await();
Assert.assertEquals(VertexState.FAILED, v.getState());
Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause());
+
Assert.assertEquals(1, committer.commitCounter);
// FIXME need to verify whether abort needs to be called if commit fails
Assert.assertEquals(0, committer.abortCounter);
- Assert.assertEquals(0, committer.initCounter); // already done in init
- Assert.assertEquals(0, committer.setupCounter); // already done in init
+ Assert.assertEquals(1, committer.initCounter);
+ Assert.assertEquals(1, committer.setupCounter);
}
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
- public void testBadCommitter2() {
- initAllVertices(VertexState.INITED);
-
+ public void testBadCommitter2() throws Exception {
VertexImpl v = vertices.get("vertex2");
- CountingVertexOutputCommitter committer =
- new CountingVertexOutputCommitter(true, true, true);
- v.setVertexOutputCommitter(committer);
+ List<RootInputLeafOutputProto> outputs =
+ new ArrayList<RootInputLeafOutputProto>();
+ outputs.add(RootInputLeafOutputProto.newBuilder()
+ .setInitializerClassName(CountingOutputCommitter.class.getName())
+ .setName("output_v2")
+ .setEntityDescriptor(
+ TezEntityDescriptorProto.newBuilder()
+ .setUserPayload(ByteString.copyFrom(
+ new CountingOutputCommitter.CountingOutputCommitterConfig(
+ true, true, true).toUserPayload())).build())
+ .build());
+ v.setAdditionalOutputs(outputs);
+ initAllVertices(VertexState.INITED);
startVertex(v);
+ CountingOutputCommitter committer =
+ (CountingOutputCommitter) v.getOutputCommitter("output_v2");
+
TezTaskID t1 = TezTaskID.getInstance(v.getVertexId(), 0);
TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1);
@@ -1749,8 +1830,8 @@ public class TestVertexImpl {
// FIXME need to verify whether abort needs to be called if commit fails
Assert.assertEquals(0, committer.abortCounter);
- Assert.assertEquals(0, committer.initCounter); // already done in init
- Assert.assertEquals(0, committer.setupCounter); // already done in init
+ Assert.assertEquals(1, committer.initCounter);
+ Assert.assertEquals(1, committer.setupCounter);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/pom.xml b/tez-mapreduce-examples/pom.xml
index aa920d7..372cec8 100644
--- a/tez-mapreduce-examples/pom.xml
+++ b/tez-mapreduce-examples/pom.xml
@@ -63,7 +63,12 @@
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
- <artifactId>tez-dag</artifactId>
+ <artifactId>tez-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 8c37683..f90cc58 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -67,6 +67,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
import org.apache.tez.mapreduce.hadoop.InputSplitInfo;
@@ -223,9 +224,9 @@ public class FilterLinesByWord {
stage2Vertex.setTaskEnvironment(stage2Env);
// Configure the Output for stage2
- stage2Vertex.addOutput("MROutput",
- new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
- .createUserPayloadFromConf(stage2Conf)));
+ OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
+ .setUserPayload(MRHelpers.createUserPayloadFromConf(stage2Conf));
+ stage2Vertex.addOutput("MROutput", od, MROutputCommitter.class);
DAG dag = new DAG("FilterLinesByWord");
Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 9c05599..5d59d8c 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -61,6 +61,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
import org.apache.tez.mapreduce.examples.FilterLinesByWord.TextLongPair;
import org.apache.tez.mapreduce.examples.helpers.SplitsInClientOptionParser;
@@ -215,7 +216,8 @@ public class FilterLinesByWordOneToOne {
// Configure the Output for stage2
stage2Vertex.addOutput("MROutput",
new OutputDescriptor(MROutput.class.getName()).setUserPayload(MRHelpers
- .createUserPayloadFromConf(stage2Conf)));
+ .createUserPayloadFromConf(stage2Conf)),
+ MROutputCommitter.class);
DAG dag = new DAG("FilterLinesByWord");
Edge edge = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
index bb48e3f..7646c65 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/WordCount.java
@@ -64,6 +64,7 @@ import org.apache.tez.dag.api.EdgeProperty.SchedulingType;
import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
import org.apache.tez.mapreduce.hadoop.MRHelpers;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
@@ -227,7 +228,7 @@ public class WordCount {
summerVertex.setTaskEnvironment(reduceEnv);
OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
.setUserPayload(finalReducePayload);
- summerVertex.addOutput("MROutput", od);
+ summerVertex.addOutput("MROutput", od, MROutputCommitter.class);
DAG dag = new DAG("WordCount");
dag.addVertex(tokenizerVertex)
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
new file mode 100644
index 0000000..7e474c1
--- /dev/null
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.mapreduce.committer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.client.VertexStatus;
+import org.apache.tez.mapreduce.hadoop.MRConfig;
+import org.apache.tez.mapreduce.hadoop.MRHelpers;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+import org.apache.tez.runtime.api.OutputCommitter;
+import org.apache.tez.runtime.api.OutputCommitterContext;
+
+import java.io.IOException;
+
+public class MROutputCommitter extends OutputCommitter {
+
+ private static final Log LOG = LogFactory.getLog(MROutputCommitter.class);
+
+ private org.apache.hadoop.mapreduce.OutputCommitter committer = null;
+ private JobContext jobContext = null;
+ private volatile boolean initialized = false;
+ private JobConf jobConf = null;
+
+ @Override
+ public void initialize(OutputCommitterContext context) throws IOException {
+ byte[] userPayload = context.getUserPayload();
+ if (userPayload == null) {
+ jobConf = new JobConf();
+ } else {
+ jobConf = new JobConf(
+ MRHelpers.createConfFromUserPayload(context.getUserPayload()));
+ }
+
+ jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID,
+ context.getDAGAttemptNumber());
+ committer = getOutputCommitter(context);
+ jobContext = getJobContextFromVertexContext(context);
+ initialized = true;
+ }
+
+ @Override
+ public void setupOutput() throws IOException {
+ if (!initialized) {
+ throw new RuntimeException("Committer not initialized");
+ }
+ committer.setupJob(jobContext);
+ }
+
+ @Override
+ public void commitOutput() throws IOException {
+ if (!initialized) {
+ throw new RuntimeException("Committer not initialized");
+ }
+ committer.commitJob(jobContext);
+ }
+
+ @Override
+ public void abortOutput(VertexStatus.State finalState) throws IOException {
+ if (!initialized) {
+ throw new RuntimeException("Committer not initialized");
+ }
+ JobStatus.State jobState = getJobStateFromVertexStatusState(finalState);
+ committer.abortJob(jobContext, jobState);
+ }
+
+ private org.apache.hadoop.mapreduce.OutputCommitter
+ getOutputCommitter(OutputCommitterContext context) {
+
+ org.apache.hadoop.mapreduce.OutputCommitter committer = null;
+ boolean newApiCommitter = false;
+ if (jobConf.getBoolean("mapred.reducer.new-api", false)
+ || jobConf.getBoolean("mapred.mapper.new-api", false)) {
+ newApiCommitter = true;
+ LOG.info("Using mapred newApiCommitter.");
+ }
+
+ LOG.info("OutputCommitter set in config for outputName="
+ + context.getOutputName()
+ + ", vertexName=" + context.getVertexName()
+ + ", outputCommitterClass="
+ + jobConf.get("mapred.output.committer.class"));
+
+ if (newApiCommitter) {
+ TaskAttemptID taskAttemptID = new TaskAttemptID(
+ Long.toString(context.getApplicationId().getClusterTimestamp()),
+ context.getApplicationId().getId(),
+ (jobConf.getBoolean(MRConfig.IS_MAP_PROCESSOR,
+ false) ? TaskType.MAP : TaskType.REDUCE),
+ 0, context.getDAGAttemptNumber());
+
+ TaskAttemptContext taskContext = new TaskAttemptContextImpl(jobConf,
+ taskAttemptID);
+ try {
+ OutputFormat outputFormat = ReflectionUtils.newInstance(taskContext
+ .getOutputFormatClass(), jobConf);
+ committer = outputFormat.getOutputCommitter(taskContext);
+ } catch (Exception e) {
+ throw new TezUncheckedException(e);
+ }
+ } else {
+ committer = ReflectionUtils.newInstance(jobConf.getClass(
+ "mapred.output.committer.class", FileOutputCommitter.class,
+ org.apache.hadoop.mapred.OutputCommitter.class), jobConf);
+ }
+ LOG.info("OutputCommitter for outputName="
+ + context.getOutputName()
+ + ", vertexName=" + context.getVertexName()
+ + ", outputCommitterClass="
+ + committer.getClass().getName());
+ return committer;
+ }
+
+ // FIXME we are using ApplicationId as DAG id
+ private JobContext getJobContextFromVertexContext(OutputCommitterContext context)
+ throws IOException {
+ JobID jobId = TypeConverter.fromYarn(
+ context.getApplicationId());
+ return new MRJobContextImpl(jobConf, jobId);
+ }
+
+ private JobStatus.State getJobStateFromVertexStatusState(VertexStatus.State state) {
+ switch(state) {
+ case INITED:
+ return JobStatus.State.PREP;
+ case RUNNING:
+ return JobStatus.State.RUNNING;
+ case SUCCEEDED:
+ return JobStatus.State.SUCCEEDED;
+ case KILLED:
+ return JobStatus.State.KILLED;
+ case FAILED:
+ case ERROR:
+ return JobStatus.State.FAILED;
+ default:
+ throw new TezUncheckedException("Unknown VertexStatus.State: " + state);
+ }
+ }
+
+ private static class MRJobContextImpl
+ extends org.apache.hadoop.mapred.JobContextImpl {
+
+ public MRJobContextImpl(JobConf jobConf, JobID jobId) {
+ super(jobConf, jobId);
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/bd23c9d4/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index af1f9e6..41cd7c9 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -71,6 +71,7 @@ import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
import org.apache.tez.mapreduce.combine.MRCombiner;
+import org.apache.tez.mapreduce.committer.MROutputCommitter;
import org.apache.tez.mapreduce.input.MRInputLegacy;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.mapreduce.partition.MRPartitioner;
@@ -981,7 +982,7 @@ public class MRHelpers {
public static void addMROutput(Vertex vertex, byte[] userPayload) {
OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
.setUserPayload(userPayload);
- vertex.addOutput("MROutput", od);
+ vertex.addOutput("MROutput", od, MROutputCommitter.class);
}
@SuppressWarnings("unchecked")