You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2014/09/17 12:27:53 UTC
git commit: TEZ-1587. Some tez-examples fail in local mode
(Contributed by Prakash Ramachandran) (cherry picked from commit
e328055e61a8dcaa1beed25e6da47f4a2279f3b6)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 10b1f2058 -> 2d04d64aa
TEZ-1587. Some tez-examples fail in local mode (Contributed by Prakash Ramachandran)
(cherry picked from commit e328055e61a8dcaa1beed25e6da47f4a2279f3b6)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2d04d64a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2d04d64a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2d04d64a
Branch: refs/heads/branch-0.5
Commit: 2d04d64aa770d7b1cc646bbe7a75c97dfe50afc3
Parents: 10b1f20
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Wed Sep 17 15:54:01 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Wed Sep 17 15:57:24 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/examples/JoinExample.java | 4 ++--
.../org/apache/tez/examples/JoinValidate.java | 2 +-
.../org/apache/tez/examples/OrderedWordCount.java | 4 ++--
.../java/org/apache/tez/examples/WordCount.java | 2 +-
.../examples/BroadcastAndOneToOneExample.java | 18 ++++++++++++++----
.../tez/mapreduce/examples/FilterLinesByWord.java | 3 ++-
.../examples/FilterLinesByWordOneToOne.java | 3 ++-
8 files changed, 25 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index ebaced8..e90634d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -23,6 +23,7 @@ ALL CHANGES
TEZ-1571. Add create method for DataSinkDescriptor.
TEZ-1585. Memory leak in tez session mode.
TEZ-1533. Request Events more often if a complete set of events is received by a task.
+ TEZ-1587. Some tez-examples fail in local mode.
Release 0.5.0: 2014-09-03
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
index 5ead275..3611fd6 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinExample.java
@@ -244,7 +244,7 @@ public class JoinExample extends Configured implements Tool {
UnorderedPartitionedKVEdgeConfig streamConf =
UnorderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).build();
+ HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
/**
* Connect the join vertex with the stream side
@@ -265,7 +265,7 @@ public class JoinExample extends Configured implements Tool {
* value is null.
*/
UnorderedKVEdgeConfig broadcastConf = UnorderedKVEdgeConfig.newBuilder(Text.class.getName(),
- NullWritable.class.getName()).build();
+ NullWritable.class.getName()).setFromConfiguration(tezConf).build();
hashSideEdgeProperty = broadcastConf.createDefaultBroadcastEdgeProperty();
} else {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
index 3630184..e94df09 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
@@ -186,7 +186,7 @@ public class JoinValidate extends Configured implements Tool {
// better mechanism to configure the IOs.
OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).build();
+ HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
Vertex lhsVertex = Vertex.create(LHS_INPUT_NAME, ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource("lhs",
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
index fd18592..48884f4 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -140,7 +140,7 @@ public class OrderedWordCount extends Configured implements Tool {
// Use Text key and IntWritable value to bring counts for each word in the same partition
OrderedPartitionedKVEdgeConfig summationEdgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).build();
+ HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
// This vertex will be reading intermediate data via an input edge and writing intermediate data
// via an output edge.
@@ -151,7 +151,7 @@ public class OrderedWordCount extends Configured implements Tool {
// partition. The data will be ordered by count and words grouped by count.
OrderedPartitionedKVEdgeConfig sorterEdgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(IntWritable.class.getName(), Text.class.getName(),
- HashPartitioner.class.getName()).build();
+ HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
// Use 1 task to bring all the data in one place for global sorted order. Essentially the number
// of partitions is 1. So the NoOpSorter can be used to produce the globally ordered output
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
index e0e94cb..89eb68c 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -173,7 +173,7 @@ public class WordCount extends Configured implements Tool {
// using this helper.
OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).build();
+ HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
// Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
// The number of tasks that do the work of this vertex depends on the number of partitions used
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index 81f9581..e8a137c 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -50,6 +50,7 @@ import org.apache.tez.runtime.api.ObjectRegistry;
import org.apache.tez.runtime.api.ProcessorContext;
import org.apache.tez.runtime.library.api.KeyValueReader;
import org.apache.tez.runtime.library.api.KeyValueWriter;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfig;
import org.apache.tez.runtime.library.output.UnorderedKVOutput;
import org.apache.tez.runtime.library.processor.SimpleProcessor;
@@ -159,7 +160,8 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
- .newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
+ .newBuilder(Text.class.getName(), IntWritable.class.getName())
+ .setFromConfiguration(tezConf).build();
DAG dag = DAG.create("BroadcastAndOneToOneExample");
dag.addVertex(inputVertex)
@@ -188,9 +190,9 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
// staging dir
FileSystem fs = FileSystem.get(tezConf);
- String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR
- + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR
- + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
+ String stagingDirStr = tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR +
+ "BroadcastAndOneToOneExample" + Path.SEPARATOR + Long.toString(System.currentTimeMillis());
Path stagingDir = new Path(stagingDirStr);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr);
stagingDir = fs.makeQualified(stagingDir);
@@ -240,6 +242,14 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
printUsage();
throw new TezException("Invalid command line");
}
+
+ if (doLocalityCheck &&
+ getConf().getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
+ TezConfiguration.TEZ_LOCAL_MODE_DEFAULT)) {
+ System.out.println("locality check is not valid in local mode. skipping");
+ doLocalityCheck = false;
+ }
+
boolean status = run(getConf(), doLocalityCheck);
return status ? 0 : 1;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 8430c68..f45b3d6 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -199,7 +199,8 @@ public class FilterLinesByWord extends Configured implements Tool {
stage2Vertex.addDataSink("MROutput", DataSinkDescriptor.create(od, ocd, null));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
- .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+ .newBuilder(Text.class.getName(), TextLongPair.class.getName())
+ .setFromConfiguration(tezConf).build();
DAG dag = DAG.create("FilterLinesByWord");
Edge edge =
http://git-wip-us.apache.org/repos/asf/tez/blob/2d04d64a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index 515cf70..49cb30b 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -190,7 +190,8 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
- .newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
+ .newBuilder(Text.class.getName(), TextLongPair.class.getName())
+ .setFromConfiguration(tezConf).build();
DAG dag = DAG.create("FilterLinesByWord");
Edge edge =