You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2015/11/17 21:19:30 UTC
[19/43] hive git commit: HIVE-12208: Vectorized JOIN NPE on
dynamically partitioned hash-join + map-join (Gunther Hagleitner,
reviewed by Matt McCline)
HIVE-12208: Vectorized JOIN NPE on dynamically partitioned hash-join + map-join (Gunther Hagleitner, reviewed by Matt McCline)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf0729c1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf0729c1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf0729c1
Branch: refs/heads/master-fixed
Commit: cf0729c13699546f38e9392c2039fa9214347690
Parents: 231247f
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Wed Nov 11 13:41:26 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Nov 17 12:18:32 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/MapJoinOperator.java | 2 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 22 +++++---------------
2 files changed, 6 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0729c1/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index 4af98e5..cab0fc8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -183,7 +183,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
@SuppressWarnings("unchecked")
@Override
- protected final void completeInitializationOp(Object[] os) throws HiveException {
+ protected void completeInitializationOp(Object[] os) throws HiveException {
if (os.length != 0) {
Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair =
(Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]>) os[0];
http://git-wip-us.apache.org/repos/asf/hive/blob/cf0729c1/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index 1667bf7..1d5a9de 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -614,46 +614,34 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
}
@Override
- protected Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> loadHashTable(
- ExecMapperContext mapContext, MapredContext mrContext) throws HiveException {
-
- Pair<MapJoinTableContainer[], MapJoinTableContainerSerDe[]> pair;
+ protected void completeInitializationOp(Object[] os) throws HiveException {
+ // setup mapJoinTables and serdes
+ super.completeInitializationOp(os);
VectorMapJoinDesc vectorDesc = conf.getVectorDesc();
HashTableImplementationType hashTableImplementationType = vectorDesc.hashTableImplementationType();
switch (vectorDesc.hashTableImplementationType()) {
case OPTIMIZED:
{
- // Using Tez's HashTableLoader, create either a MapJoinBytesTableContainer or
- // HybridHashTableContainer.
- pair = super.loadHashTable(mapContext, mrContext);
-
// Create our vector map join optimized hash table variation *above* the
// map join table container.
- MapJoinTableContainer[] mapJoinTableContainers = pair.getLeft();
vectorMapJoinHashTable = VectorMapJoinOptimizedCreateHashTable.createHashTable(conf,
- mapJoinTableContainers[posSingleVectorMapJoinSmallTable]);
+ mapJoinTables[posSingleVectorMapJoinSmallTable]);
}
break;
case FAST:
{
- // Use our VectorMapJoinFastHashTableLoader to create a VectorMapJoinTableContainer.
- pair = super.loadHashTable(mapContext, mrContext);
-
// Get our vector map join fast hash table variation from the
// vector map join table container.
- MapJoinTableContainer[] mapJoinTableContainers = pair.getLeft();
VectorMapJoinTableContainer vectorMapJoinTableContainer =
- (VectorMapJoinTableContainer) mapJoinTableContainers[posSingleVectorMapJoinSmallTable];
+ (VectorMapJoinTableContainer) mapJoinTables[posSingleVectorMapJoinSmallTable];
vectorMapJoinHashTable = vectorMapJoinTableContainer.vectorMapJoinHashTable();
}
break;
default:
throw new RuntimeException("Unknown vector map join hash table implementation type " + hashTableImplementationType.name());
}
-
- return pair;
}
/*