You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by bi...@apache.org on 2013/09/26 21:09:31 UTC
git commit: TEZ-510. Test MRRJobs failing with NPE (bikas)
Updated Branches:
refs/heads/master 38350663c -> 1795ea84f
TEZ-510. Test MRRJobs failing with NPE (bikas)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/1795ea84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/1795ea84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/1795ea84
Branch: refs/heads/master
Commit: 1795ea84fc899f5e522680119b1853f559030a14
Parents: 3835066
Author: Bikas Saha <bi...@apache.org>
Authored: Thu Sep 26 12:07:40 2013 -0700
Committer: Bikas Saha <bi...@apache.org>
Committed: Thu Sep 26 12:07:40 2013 -0700
----------------------------------------------------------------------
.../org/apache/tez/dag/app/dag/impl/VertexImpl.java | 7 ++++---
.../tez/processor/FilterByWordInputProcessor.java | 13 +++++++------
.../java/org/apache/tez/mapreduce/input/MRInput.java | 2 +-
3 files changed, 12 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/1795ea84/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 6752328..597acb2 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -251,9 +251,6 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexEventType.V_TASK_COMPLETED,
new TaskCompletedTransition())
.addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
- VertexEventType.V_TASK_ATTEMPT_COMPLETED,
- TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) // TODO shouldnt be done for KILL_WAIT vertex
- .addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED,
SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
.addTransition(
@@ -263,6 +260,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.TERMINATING, VertexState.TERMINATING,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_TASK_ATTEMPT_COMPLETED,
VertexEventType.V_TASK_RESCHEDULED))
// Transitions from SUCCEEDED state
@@ -301,6 +299,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
// Ignore-able events
.addTransition(VertexState.KILLED, VertexState.KILLED,
EnumSet.of(VertexEventType.V_TERMINATE,
+ VertexEventType.V_SOURCE_VERTEX_STARTED,
VertexEventType.V_START,
VertexEventType.V_TASK_RESCHEDULED,
VertexEventType.V_TASK_ATTEMPT_COMPLETED,
@@ -311,6 +310,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
VertexState.ERROR,
VertexState.ERROR,
EnumSet.of(VertexEventType.V_INIT,
+ VertexEventType.V_SOURCE_VERTEX_STARTED,
+ VertexEventType.V_START,
VertexEventType.V_TERMINATE,
VertexEventType.V_TASK_COMPLETED,
VertexEventType.V_TASK_ATTEMPT_COMPLETED,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/1795ea84/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
index e8e315a..6c97430 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/processor/FilterByWordInputProcessor.java
@@ -98,13 +98,14 @@ public class FilterByWordInputProcessor implements LogicalIOProcessor {
OnFileUnorderedKVOutput kvOutput = (OnFileUnorderedKVOutput) lo;
Configuration updatedConf = mrInput.getConfigUpdates();
- String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
- LOG.info("Processing file: " + fileName);
Text srcFile = new Text();
- if (fileName == null) {
- srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
- } else {
- srcFile.set(fileName);
+ srcFile.set("UNKNOWN_FILENAME_IN_PROCESSOR");
+ if (updatedConf != null) {
+ String fileName = updatedConf.get(MRJobConfig.MAP_INPUT_FILE);
+ if (fileName != null) {
+ LOG.info("Processing file: " + fileName);
+ srcFile.set(fileName);
+ }
}
KVReader kvReader = mrInput.getReader();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/1795ea84/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index b9f2242..f0aea33 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -209,7 +209,7 @@ public class MRInput implements LogicalInput {
* @return the additional fields set by {@link MRInput}
*/
public Configuration getConfigUpdates() {
- return new Configuration(incrementalConf);
+ return incrementalConf;
}
public float getProgress() throws IOException, InterruptedException {