You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/04 00:19:27 UTC
svn commit: r1622371 -
/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Author: gunther
Date: Wed Sep 3 22:19:26 2014
New Revision: 1622371
URL: http://svn.apache.org/r1622371
Log:
HIVE-7544: Changes related to TEZ-1288 (FastTezSerialization) (Rajesh Balamohan via Gunther Hagleitner)
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1622371&r1=1622370&r2=1622371&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Wed Sep 3 22:19:26 2014
@@ -106,6 +106,8 @@ import org.apache.tez.mapreduce.input.MR
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.mapreduce.partition.MRPartitioner;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.comparator.TezBytesComparator;
+import org.apache.tez.runtime.library.common.serializer.TezBytesWritableSerialization;
import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfig;
import org.apache.tez.runtime.library.conf.UnorderedKVEdgeConfig;
import org.apache.tez.runtime.library.conf.UnorderedPartitionedKVEdgeConfig;
@@ -317,14 +319,21 @@ public class DagUtils {
switch (edgeType) {
case BROADCAST_EDGE:
UnorderedKVEdgeConfig et1Conf = UnorderedKVEdgeConfig
- .newBuilder(keyClass, valClass).setFromConfiguration(conf).build();
+ .newBuilder(keyClass, valClass)
+ .setFromConfiguration(conf)
+ .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .build();
return et1Conf.createDefaultBroadcastEdgeProperty();
case CUSTOM_EDGE:
assert partitionerClassName != null;
partitionerConf = createPartitionerConf(partitionerClassName, conf);
UnorderedPartitionedKVEdgeConfig et2Conf = UnorderedPartitionedKVEdgeConfig
.newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf)
- .setFromConfiguration(conf).build();
+ .setFromConfiguration(conf)
+ .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .build();
EdgeManagerPluginDescriptor edgeDesc =
EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName());
CustomEdgeConfiguration edgeConf =
@@ -339,7 +348,10 @@ public class DagUtils {
partitionerConf = createPartitionerConf(partitionerClassName, conf);
UnorderedPartitionedKVEdgeConfig et3Conf = UnorderedPartitionedKVEdgeConfig
.newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf)
- .setFromConfiguration(conf).build();
+ .setFromConfiguration(conf)
+ .setKeySerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .build();
return et3Conf.createDefaultEdgeProperty();
case SIMPLE_EDGE:
default:
@@ -347,7 +359,11 @@ public class DagUtils {
partitionerConf = createPartitionerConf(partitionerClassName, conf);
OrderedPartitionedKVEdgeConfig et4Conf = OrderedPartitionedKVEdgeConfig
.newBuilder(keyClass, valClass, MRPartitioner.class.getName(), partitionerConf)
- .setFromConfiguration(conf).build();
+ .setFromConfiguration(conf)
+ .setKeySerializationClass(TezBytesWritableSerialization.class.getName(),
+ TezBytesComparator.class.getName(), null)
+ .setValueSerializationClass(TezBytesWritableSerialization.class.getName(), null)
+ .build();
return et4Conf.createDefaultEdgeProperty();
}
}