You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2014/12/16 06:23:05 UTC
svn commit: r1645843 - in /hive/branches/spark: data/conf/spark/
ql/src/java/org/apache/hadoop/hive/ql/exec/spark/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/test/results/clientpositive/spark/
Author: xuefu
Date: Tue Dec 16 05:23:05 2014
New Revision: 1645843
URL: http://svn.apache.org/r1645843
Log:
reverse r1645842: HIVE-9101: bucket_map_join_spark4.q failed due to NPE.[Spark Branch] (Jimmy via Xuefu)
Modified:
hive/branches/spark/data/conf/spark/hive-site.xml
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
Modified: hive/branches/spark/data/conf/spark/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/hive-site.xml?rev=1645843&r1=1645842&r2=1645843&view=diff
==============================================================================
--- hive/branches/spark/data/conf/spark/hive-site.xml (original)
+++ hive/branches/spark/data/conf/spark/hive-site.xml Tue Dec 16 05:23:05 2014
@@ -220,9 +220,4 @@
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
</property>
-<property>
- <name>hive.users.in.admin.role</name>
- <value>hive_admin_user</value>
-</property>
-
</configuration>
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java?rev=1645843&r1=1645842&r2=1645843&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java Tue Dec 16 05:23:05 2014
@@ -97,17 +97,13 @@ public class HashTableLoader implements
}
String bigInputPath = currentInputPath;
if (currentInputPath != null && mapJoinCtx != null) {
- if (!desc.isBucketMapJoin()) {
- bigInputPath = null;
- } else {
- Set<String> aliases =
- ((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
- String alias = aliases.iterator().next();
- // Any one small table input path
- String smallInputPath =
- mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
- bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
- }
+ Set<String> aliases =
+ ((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
+ String alias = aliases.iterator().next();
+ // Any one small table input path
+ String smallInputPath =
+ mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
+ bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
}
String fileName = localWork.getBucketFileName(bigInputPath);
Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte)pos, fileName);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1645843&r1=1645842&r2=1645843&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java Tue Dec 16 05:23:05 2014
@@ -240,14 +240,12 @@ abstract public class AbstractBucketJoin
for (Map.Entry<String, Operator<? extends OperatorDesc>> topOpEntry : topOps.entrySet()) {
if (topOpEntry.getValue() == tso) {
String newAlias = topOpEntry.getKey();
- if (!newAlias.equals(alias)) {
- joinAliases.set(index, newAlias);
- if (baseBigAlias.equals(alias)) {
- baseBigAlias = newAlias;
- }
- aliasToNewAliasMap.put(alias, newAlias);
- alias = newAlias;
+ joinAliases.set(index, newAlias);
+ if (baseBigAlias.equals(alias)) {
+ baseBigAlias = newAlias;
}
+ aliasToNewAliasMap.put(alias, newAlias);
+ alias = newAlias;
break;
}
}
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out?rev=1645843&r1=1645842&r2=1645843&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out Tue Dec 16 05:23:05 2014
@@ -448,64 +448,6 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
POSTHOOK: Input: default@tbl3
#### A masked pattern was here ####
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-0 val_0 val_0 val_0
-2 val_2 val_2 val_2
-4 val_4 val_4 val_4
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-5 val_5 val_5 val_5
-8 val_8 val_8 val_8
-9 val_9 val_9 val_9
PREHOOK: query: explain extended
select a.key as key, a.value as val1, b.value as val2, c.value as val3
from tbl1 a join tbl2 b on a.key = b.key join tbl3 c on a.value = c.value