You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/05/25 21:11:16 UTC
svn commit: r948175 [1/2] - in /hadoop/hive/trunk: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: namit
Date: Tue May 25 19:11:16 2010
New Revision: 948175
URL: http://svn.apache.org/viewvc?rev=948175&view=rev
Log:
HIVE-1365. Bug in sort-merge join
(He Yongqiang via namit)
Added:
hadoop/hive/trunk/data/files/smb_bucket_input.txt
hadoop/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_8.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/smb_mapjoin_8.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=948175&r1=948174&r2=948175&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue May 25 19:11:16 2010
@@ -460,9 +460,12 @@ Trunk - Unreleased
(Edward Capriolo via namit)
HIVE-1366. inputFileFormat error if the merge job takes a different input
- file format than the default output file format
+ file format than the default output file format
(Namit Jain via Ning Zhang)
+ HIVE-1365. Bug in sort-merge join
+ (He Yongqiang via namit)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Added: hadoop/hive/trunk/data/files/smb_bucket_input.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/smb_bucket_input.txt?rev=948175&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/smb_bucket_input.txt (added)
+++ hadoop/hive/trunk/data/files/smb_bucket_input.txt Tue May 25 19:11:16 2010
@@ -0,0 +1,7 @@
+4val_356
+484val_169
+1000val_1000
+2000val_169
+3000val_169
+4000val_125
+5000val_125
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=948175&r1=948174&r2=948175&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Tue May 25 19:11:16 2010
@@ -230,34 +230,31 @@ public class SMBMapJoinOperator extends
allFetchOpDone = allFetchOpDone();
}
- if (allFetchOpDone
- && this.candidateStorage.get((byte) this.posBigTable).size() > 0) {
- // if all fetch operator for small tables are done and there are data left
- // in big table
+ while (!allFetchOpDone) {
+ List<Byte> ret = joinOneGroup();
+ if (ret == null || ret.size() == 0) {
+ break;
+ }
+ reportProgress();
+ allFetchOpDone = allFetchOpDone();
+ }
+
+ boolean dataInCache = true;
+ while (dataInCache) {
for (byte t : order) {
- if(this.foundNextKeyGroup.get(t) && this.nextKeyWritables.get(t) != null) {
+ if (this.foundNextKeyGroup.get(t)
+ && this.nextKeyWritables.get(t) != null) {
promoteNextGroupToCandidate(t);
}
}
joinOneGroup();
- } else {
- while (!allFetchOpDone) {
- List<Byte> ret = joinOneGroup();
- if (ret == null || ret.size() == 0) {
+ dataInCache = false;
+ for (byte r : order) {
+ if (this.candidateStorage.get(r).size() > 0) {
+ dataInCache = true;
break;
}
-
- reportProgress();
-
- allFetchOpDone = allFetchOpDone();
- }
- //one final table left
- for (byte t : order) {
- if(this.foundNextKeyGroup.get(t) && this.nextKeyWritables.get(t) != null) {
- promoteNextGroupToCandidate(t);
- }
}
- joinOneGroup();
}
}
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_8.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_8.q?rev=948175&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_8.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/smb_mapjoin_8.q Tue May 25 19:11:16 2010
@@ -0,0 +1,90 @@
+set hive.enforce.bucketing = true;
+set hive.exec.reducers.max = 1;
+
+drop table smb_bucket_input;
+create table smb_bucket_input (key int, value string) stored as textfile;
+load data local inpath '../data/files/smb_bucket_input.txt' into table smb_bucket_input;
+
+set hive.optimize.bucketmapjoin = true;
+set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.enforce.sorting = true;
+set hive.exec.reducers.max = 1;
+
+drop table smb_bucket4_1;
+CREATE TABLE smb_bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS;
+drop table smb_bucket4_2;
+CREATE TABLE smb_bucket4_2(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS;
+drop table smb_bucket4_3;
+CREATE TABLE smb_bucket4_3(key int, value string) CLUSTERED BY (key) SORTED BY (key) INTO 1 BUCKETS;
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4 or key=2000 or key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=484 or key=3000 or key=5000;
+
+select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=2000 or key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=3000 or key=5000;
+
+select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=5000;
+
+select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=1000 or key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=1000 or key=5000;
+
+select /*+mapjoin(a)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+select /*+mapjoin(b)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=1000 or key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=1000 or key=5000;
+insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=1000 or key=5000;
+
+select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
+full outer join smb_bucket4_3 c on a.key=c.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=1000 or key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=1000 or key=5000;
+insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=1000 or key=4000;
+
+select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
+full outer join smb_bucket4_3 c on a.key=c.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=4000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=5000;
+insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=4000;
+
+select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
+full outer join smb_bucket4_3 c on a.key=c.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=00000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=4000;
+insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=5000;
+
+select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
+full outer join smb_bucket4_3 c on a.key=c.key;
+
+
+insert overwrite table smb_bucket4_1 select * from smb_bucket_input where key=1000;
+insert overwrite table smb_bucket4_2 select * from smb_bucket_input where key=4000;
+insert overwrite table smb_bucket4_3 select * from smb_bucket_input where key=5000;
+
+select /*+mapjoin(b,c)*/ * from smb_bucket4_1 a full outer join smb_bucket4_2 b on a.key = b.key
+full outer join smb_bucket4_3 c on a.key=c.key;
+
+drop table smb_bucket4_1;
+drop table smb_bucket4_2;
+drop table smb_bucket4_3;
+drop table smb_bucket_input;
\ No newline at end of file