You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2016/07/08 01:24:06 UTC
hive git commit: HIVE-13932:Hive SMB Map Join with small set of LIMIT
failed with NPE (Yongzhi Chen, reviewed by Chaoyu Tang)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 54ff9c708 -> c4a9ab85d
HIVE-13932:Hive SMB Map Join with small set of LIMIT failed with NPE (Yongzhi Chen, reviewed by Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c4a9ab85
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c4a9ab85
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c4a9ab85
Branch: refs/heads/branch-2.1
Commit: c4a9ab85da5ad0c258c4bb59bb0bf386369cf052
Parents: 54ff9c7
Author: Yongzhi Chen <yc...@apache.org>
Authored: Thu Jun 2 18:47:13 2016 -0400
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Thu Jul 7 18:15:46 2016 -0700
----------------------------------------------------------------------
data/files/smbdata.txt | 99 ++++++++++++++++++++
.../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 2 +-
ql/src/test/queries/clientpositive/smblimit.q | 12 +++
.../test/results/clientpositive/smblimit.q.out | 50 ++++++++++
4 files changed, 162 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c4a9ab85/data/files/smbdata.txt
----------------------------------------------------------------------
diff --git a/data/files/smbdata.txt b/data/files/smbdata.txt
new file mode 100644
index 0000000..4422a54
--- /dev/null
+++ b/data/files/smbdata.txt
@@ -0,0 +1,99 @@
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99
http://git-wip-us.apache.org/repos/asf/hive/blob/c4a9ab85/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
index 50d6f3b..57b6c67 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
@@ -357,7 +357,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp
joinOneGroup();
dataInCache = false;
for (byte pos = 0; pos < order.length; pos++) {
- if (this.candidateStorage[pos].hasRows()) {
+ if (this.candidateStorage[pos] != null && this.candidateStorage[pos].hasRows()) {
dataInCache = true;
break;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c4a9ab85/ql/src/test/queries/clientpositive/smblimit.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/smblimit.q b/ql/src/test/queries/clientpositive/smblimit.q
new file mode 100644
index 0000000..1db2167
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/smblimit.q
@@ -0,0 +1,12 @@
+drop table if exists hlp1;
+drop table if exists btl;
+create table hlp1(c string);
+load data local inpath '../../data/files/smbdata.txt' into table hlp1;
+create table btl(c string) clustered by (c) sorted by (c) into 5 buckets;
+insert overwrite table btl select * from hlp1;
+SET hive.auto.convert.sortmerge.join = true;
+SET hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+select 1 from btl join btl t1 on btl.c=t1.c limit 1;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/c4a9ab85/ql/src/test/results/clientpositive/smblimit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/smblimit.q.out b/ql/src/test/results/clientpositive/smblimit.q.out
new file mode 100644
index 0000000..64ca604
--- /dev/null
+++ b/ql/src/test/results/clientpositive/smblimit.q.out
@@ -0,0 +1,50 @@
+PREHOOK: query: drop table if exists hlp1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists hlp1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists btl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists btl
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table hlp1(c string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hlp1
+POSTHOOK: query: create table hlp1(c string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hlp1
+PREHOOK: query: load data local inpath '../../data/files/smbdata.txt' into table hlp1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@hlp1
+POSTHOOK: query: load data local inpath '../../data/files/smbdata.txt' into table hlp1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@hlp1
+PREHOOK: query: create table btl(c string) clustered by (c) sorted by (c) into 5 buckets
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@btl
+POSTHOOK: query: create table btl(c string) clustered by (c) sorted by (c) into 5 buckets
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@btl
+PREHOOK: query: insert overwrite table btl select * from hlp1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hlp1
+PREHOOK: Output: default@btl
+POSTHOOK: query: insert overwrite table btl select * from hlp1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hlp1
+POSTHOOK: Output: default@btl
+POSTHOOK: Lineage: btl.c SIMPLE [(hlp1)hlp1.FieldSchema(name:c, type:string, comment:null), ]
+PREHOOK: query: select 1 from btl join btl t1 on btl.c=t1.c limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@btl
+#### A masked pattern was here ####
+POSTHOOK: query: select 1 from btl join btl t1 on btl.c=t1.c limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@btl
+#### A masked pattern was here ####
+1