You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/12/16 06:24:01 UTC
svn commit: r1645844 - in /hive/branches/spark/ql/src:
java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
test/results/clientpositive/spark/bucket_map_join_spark4.q.out
Author: xuefu
Date: Tue Dec 16 05:24:00 2014
New Revision: 1645844
URL: http://svn.apache.org/r1645844
Log:
HIVE-9101: bucket_map_join_spark4.q failed due to NPE.[Spark Branch] (Jimmy via Xuefu)
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java?rev=1645844&r1=1645843&r2=1645844&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java Tue Dec 16 05:24:00 2014
@@ -97,13 +97,17 @@ public class HashTableLoader implements
}
String bigInputPath = currentInputPath;
if (currentInputPath != null && mapJoinCtx != null) {
- Set<String> aliases =
- ((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
- String alias = aliases.iterator().next();
- // Any one small table input path
- String smallInputPath =
- mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
- bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
+ if (!desc.isBucketMapJoin()) {
+ bigInputPath = null;
+ } else {
+ Set<String> aliases =
+ ((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
+ String alias = aliases.iterator().next();
+ // Any one small table input path
+ String smallInputPath =
+ mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
+ bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
+ }
}
String fileName = localWork.getBucketFileName(bigInputPath);
Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte)pos, fileName);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1645844&r1=1645843&r2=1645844&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java Tue Dec 16 05:24:00 2014
@@ -240,12 +240,14 @@ abstract public class AbstractBucketJoin
for (Map.Entry<String, Operator<? extends OperatorDesc>> topOpEntry : topOps.entrySet()) {
if (topOpEntry.getValue() == tso) {
String newAlias = topOpEntry.getKey();
- joinAliases.set(index, newAlias);
- if (baseBigAlias.equals(alias)) {
- baseBigAlias = newAlias;
+ if (!newAlias.equals(alias)) {
+ joinAliases.set(index, newAlias);
+ if (baseBigAlias.equals(alias)) {
+ baseBigAlias = newAlias;
+ }
+ aliasToNewAliasMap.put(alias, newAlias);
+ alias = newAlias;
}
- aliasToNewAliasMap.put(alias, newAlias);
- alias = newAlias;
break;
}
}
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out?rev=1645844&r1=1645843&r2=1645844&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out Tue Dec 16 05:24:00 2014
@@ -448,6 +448,64 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
POSTHOOK: Input: default@tbl3
#### A masked pattern was here ####
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+2 val_2 val_2 val_2
+4 val_4 val_4 val_4
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+8 val_8 val_8 val_8
+9 val_9 val_9 val_9
PREHOOK: query: explain extended
select a.key as key, a.value as val1, b.value as val2, c.value as val3
from tbl1 a join tbl2 b on a.key = b.key join tbl3 c on a.value = c.value