You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2014/10/10 00:57:29 UTC
[1/2] git commit: TEZ-1614. Use setFromConfiguration() in
SortMergeJoinExample to demonstrate the usage (Cherry picked from commit
efa6afd0f3e8a5c2cde8f1c415f6b3f87c194c89)
Repository: tez
Updated Branches:
refs/heads/branch-0.5 b48bc6ab9 -> eea4cea4e
TEZ-1614. Use setFromConfiguration() in SortMergeJoinExample to demonstrate the usage
(Cherry picked from commit efa6afd0f3e8a5c2cde8f1c415f6b3f87c194c89)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c05c4649
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c05c4649
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c05c4649
Branch: refs/heads/branch-0.5
Commit: c05c4649cee95288704672b1944a4b7398bdf1a8
Parents: b48bc6a
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Oct 10 04:12:13 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Fri Oct 10 04:22:48 2014 +0530
----------------------------------------------------------------------
.../main/java/org/apache/tez/examples/HashJoinExample.java | 4 ++--
.../src/main/java/org/apache/tez/examples/JoinValidate.java | 2 +-
.../main/java/org/apache/tez/examples/OrderedWordCount.java | 4 ++--
.../java/org/apache/tez/examples/SortMergeJoinExample.java | 2 ++
.../src/main/java/org/apache/tez/examples/WordCount.java | 2 +-
.../runtime/library/conf/OrderedPartitionedKVEdgeConfig.java | 7 +++++++
.../tez/runtime/library/conf/UnorderedKVEdgeConfig.java | 7 +++++++
.../library/conf/UnorderedPartitionedKVEdgeConfig.java | 7 +++++++
8 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
index 76e53e1..3f9085f 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java
@@ -266,7 +266,7 @@ public class HashJoinExample extends Configured implements Tool {
UnorderedPartitionedKVEdgeConfig streamConf =
UnorderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(tezConf)
+ HashPartitioner.class.getName())
.build();
/**
@@ -293,7 +293,7 @@ public class HashJoinExample extends Configured implements Tool {
UnorderedKVEdgeConfig broadcastConf =
UnorderedKVEdgeConfig
.newBuilder(Text.class.getName(), NullWritable.class.getName())
- .setFromConfiguration(tezConf).build();
+ .build();
hashSideEdgeProperty = broadcastConf.createDefaultBroadcastEdgeProperty();
} else {
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
index 17e3dc8..0b8671d 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java
@@ -186,7 +186,7 @@ public class JoinValidate extends Configured implements Tool {
// better mechanism to configure the IOs.
OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), NullWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+ HashPartitioner.class.getName()).build();
Vertex lhsVertex = Vertex.create(LHS_INPUT_NAME, ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource("lhs",
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
index 48884f4..fd18592 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -140,7 +140,7 @@ public class OrderedWordCount extends Configured implements Tool {
// Use Text key and IntWritable value to bring counts for each word in the same partition
OrderedPartitionedKVEdgeConfig summationEdgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+ HashPartitioner.class.getName()).build();
// This vertex will be reading intermediate data via an input edge and writing intermediate data
// via an output edge.
@@ -151,7 +151,7 @@ public class OrderedWordCount extends Configured implements Tool {
// partition. The data will be ordered by count and words grouped by count.
OrderedPartitionedKVEdgeConfig sorterEdgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(IntWritable.class.getName(), Text.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+ HashPartitioner.class.getName()).build();
// Use 1 task to bring all the data in one place for global sorted order. Essentially the number
// of partitions is 1. So the NoOpSorter can be used to produce the globally ordered output
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
index b9ba87a..25d2a24 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java
@@ -266,6 +266,8 @@ public class SortMergeJoinExample extends Configured implements Tool {
* JoinProcessor. The number of fragments is initially inferred from the
* number of tasks running in the join vertex because each task will be
* handling one fragment.
+ * Edge config options are derived from client-side tez-site.xml (recommended). Optionally
+ * invoke setFromConfiguration to override these config options via commandline arguments.
*/
OrderedPartitionedKVEdgeConfig edgeConf =
OrderedPartitionedKVEdgeConfig
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
index 89eb68c..e0e94cb 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -173,7 +173,7 @@ public class WordCount extends Configured implements Tool {
// using this helper.
OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName(),
- HashPartitioner.class.getName()).setFromConfiguration(tezConf).build();
+ HashPartitioner.class.getName()).build();
// Create a vertex that reads the tokenized data and calculates the sum using the SumProcessor.
// The number of tasks that do the work of this vertex depends on the number of partitions used
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index 76f73ac..0f10cf1 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -259,6 +259,13 @@ public class OrderedPartitionedKVEdgeConfig
}
@Override
+ /**
+ * Edge config options are derived from client-side tez-site.xml (recommended).
+ * Optionally invoke setFromConfiguration to override these config options via commandline arguments.
+ *
+ * @param conf
+ * @return this object for further chained method calls
+ */
public Builder setFromConfiguration(Configuration conf) {
outputBuilder.setFromConfiguration(conf);
inputBuilder.setFromConfiguration(conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 3864565..6eb1d6a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -193,6 +193,13 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
}
@Override
+ /**
+ * Edge config options are derived from client-side tez-site.xml (recommended).
+ * Optionally invoke setFromConfiguration to override these config options via commandline arguments.
+ *
+ * @param conf
+ * @return this object for further chained method calls
+ */
public Builder setFromConfiguration(Configuration conf) {
outputBuilder.setFromConfiguration(conf);
inputBuilder.setFromConfiguration(conf);
http://git-wip-us.apache.org/repos/asf/tez/blob/c05c4649/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 44ca74e..30585bd 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -199,6 +199,13 @@ public class UnorderedPartitionedKVEdgeConfig
}
@Override
+ /**
+ * Edge config options are derived from client-side tez-site.xml (recommended).
+ * Optionally invoke setFromConfiguration to override these config options via commandline arguments.
+ *
+ * @param conf
+ * @return this object for further chained method calls
+ */
public Builder setFromConfiguration(Configuration conf) {
outputBuilder.setFromConfiguration(conf);
inputBuilder.setFromConfiguration(conf);
[2/2] git commit: Adding TEZ-1614 to CHANGES.txt
Posted by rb...@apache.org.
Adding TEZ-1614 to CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/eea4cea4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/eea4cea4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/eea4cea4
Branch: refs/heads/branch-0.5
Commit: eea4cea4e6dac9f44df9bb63a253bc90ef41fe97
Parents: c05c464
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Fri Oct 10 04:25:57 2014 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Fri Oct 10 04:25:57 2014 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/eea4cea4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 00f17d6..f6f841e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ ALL CHANGES:
TEZ-1621. Should report error to AM before shuting down TezChild
TEZ-1634. Fix compressed IFile shuffle errors
TEZ-1648. Update website after 0.5.1
+ TEZ-1614. Use setFromConfiguration() in SortMergeJoinExample to demonstrate the usage
Release 0.5.1: 2014-10-02