You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2016/05/04 09:04:08 UTC
[2/8] tajo git commit: TAJO-2135: Invalid join result when join key
columns contain nulls.
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testLeadWithNoArgs.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testLeadWithNoArgs.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testLeadWithNoArgs.result
index a85ead5..267b992 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testLeadWithNoArgs.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testLeadWithNoArgs.result
@@ -4,4 +4,7 @@ TRUCK,1,7706,1996-03-13,1996-02-12 00:00:00,21168.23,0.04,1
null,null,null,null,null,null,null,1
null,null,null,null,null,null,null,2
RAIL,2,6540,1993-11-09,1993-12-20 00:00:00,46796.47,0.1,3
-null,null,null,null,null,null,null,3
\ No newline at end of file
+null,null,null,null,null,null,null,3
+null,null,null,null,null,null,null,null
+null,null,null,null,null,null,null,null
+null,null,null,null,null,null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber1.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber1.result
index 5fc49ee..2c0f6e6 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber1.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber1.result
@@ -4,4 +4,7 @@ l_orderkey,row_num
1,2
2,3
3,4
-3,5
\ No newline at end of file
+3,5
+null,6
+null,7
+null,8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber2.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber2.result
index db02a76..0e9c8ea 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber2.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber2.result
@@ -4,4 +4,7 @@ l_orderkey,row_num
1,2
2,1
3,1
-3,2
\ No newline at end of file
+3,2
+null,1
+null,2
+null,3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber3.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber3.result
index 1d780ff..b272e14 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber3.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testRowNumber3.result
@@ -4,4 +4,7 @@ l_orderkey,row_num,l_discount,average
1,2,0.09,0.065
2,1,0.0,0.0
3,1,0.06,0.08
-3,2,0.1,0.08
\ No newline at end of file
+3,2,0.1,0.08
+null,1,null,null
+null,2,null,null
+null,3,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevPop1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevPop1.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevPop1.result
index a2faac0..882a566 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevPop1.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevPop1.result
@@ -4,4 +4,7 @@ linenumber_stddev_pop,suppkey_stddev_pop,extendedprice_stddev_pop,discount_stdde
0.5,197.5,12407.465,0.02500000223517418
0.0,0.0,0.0,0.0
0.5,2371.0,3630.790000000001,0.020000001415610313
-0.5,2371.0,3630.790000000001,0.020000001415610313
\ No newline at end of file
+0.5,2371.0,3630.790000000001,0.020000001415610313
+null,null,null,null
+null,null,null,null
+null,null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevSamp1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevSamp1.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevSamp1.result
index 625f91b..478e5d6 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevSamp1.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testStdDevSamp1.result
@@ -4,4 +4,7 @@ linenumber_stddev_samp,suppkey_stddev_samp,extendedprice_stddev_samp,discount_st
0.7071067811865476,279.30717856868625,17546.805277669493,0.035355342220341014
null,null,null,null
0.7071067811865476,3353.1003563866084,5134.7124601286105,0.028284273249437206
-0.7071067811865476,3353.1003563866084,5134.7124601286105,0.028284273249437206
\ No newline at end of file
+0.7071067811865476,3353.1003563866084,5134.7124601286105,0.028284273249437206
+null,null,null,null
+null,null,null,null
+null,null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow1.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow1.result
index f142256..38cdc5f 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow1.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow1.result
@@ -4,4 +4,7 @@
185.0
185.0
185.0
+185.0
+185.0
+185.0
185.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow2.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow2.result
index 136d66c..5fdf4a8 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow2.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow2.result
@@ -4,4 +4,7 @@ l_orderkey,l_quantity,?windowfunction
1,36.0,185.0
2,38.0,185.0
3,45.0,185.0
-3,49.0,185.0
\ No newline at end of file
+3,49.0,185.0
+null,null,185.0
+null,null,185.0
+null,null,185.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow3.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow3.result
index 8262c13..dbd16c0 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow3.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow3.result
@@ -4,4 +4,7 @@ l_orderkey,l_quantity,?windowfunction
1,36.0,53.0
2,38.0,38.0
3,45.0,94.0
-3,49.0,94.0
\ No newline at end of file
+3,49.0,94.0
+null,null,null
+null,null,null
+null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow4.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow4.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow4.result
index 3477c5b..3079b92 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow4.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow4.result
@@ -4,4 +4,7 @@ l_orderkey,l_discount,?windowfunction,?windowfunction_1
1,0.09,0.13,53.0
2,0.0,0.0,38.0
3,0.06,0.16,94.0
-3,0.1,0.16,94.0
\ No newline at end of file
+3,0.1,0.16,94.0
+null,null,null,null
+null,null,null,null
+null,null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow5.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow5.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow5.result
index e42b1d3..b17836e 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow5.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow5.result
@@ -4,4 +4,7 @@ l_orderkey,?windowfunction,l_discount,?windowfunction_1
1,0.13,0.09,53.0
2,0.0,0.0,38.0
3,0.16,0.06,94.0
-3,0.16,0.1,94.0
\ No newline at end of file
+3,0.16,0.1,94.0
+null,null,null,null
+null,null,null,null
+null,null,null,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow6.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow6.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow6.result
index 7cf7ae1..6b0c744 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow6.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow6.result
@@ -4,4 +4,7 @@ l_orderkey,l_discount,r1,r2
1,0.09,2,0.13
2,0.0,1,0.0
3,0.06,1,0.16
-3,0.1,2,0.16
\ No newline at end of file
+3,0.1,2,0.16
+null,null,1,null
+null,null,2,null
+null,null,3,null
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow7.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow7.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow7.result
index 0a7de1a..018e3a3 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow7.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow7.result
@@ -4,4 +4,7 @@ l_orderkey,l_quantity,r
1,36.0,1
2,38.0,1
3,45.0,1
-3,49.0,1
\ No newline at end of file
+3,49.0,1
+null,null,1
+null,null,1
+null,null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow8.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow8.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow8.result
index f371de5..71c9886 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow8.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindow8.result
@@ -4,4 +4,7 @@ l_orderkey,l_quantity,r,const_val
1,36.0,1,5
2,38.0,1,5
3,45.0,1,5
-3,49.0,1,5
\ No newline at end of file
+3,49.0,1,5
+null,null,1,5
+null,null,1,5
+null,null,1,5
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation2.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation2.result
index ba11e16..5b7cac0 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation2.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation2.result
@@ -2,4 +2,5 @@ l_orderkey,row_num
-------------------------------
1,1
2,1
-3,1
\ No newline at end of file
+3,1
+null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation3.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation3.result
index a8b99f6..6e30d22 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation3.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation3.result
@@ -1,3 +1,3 @@
cnt,row_num
-------------------------------
-5,1
\ No newline at end of file
+8,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result
index fffa2dd..533c7a7 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation4.result
@@ -1,5 +1,6 @@
l_orderkey,cnt,row_num
-------------------------------
-1,2,1
-3,2,2
-2,1,3
\ No newline at end of file
+null,3,1
+1,2,2
+3,2,3
+2,1,4
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation5.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation5.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation5.result
index 23ff045..798e862 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation5.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation5.result
@@ -2,4 +2,5 @@ l_orderkey,cnt,row_num
-------------------------------
1,2,1
2,1,1
-3,2,1
\ No newline at end of file
+3,2,1
+null,3,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result
index c2e02d5..a4621a4 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithAggregation6.result
@@ -1,5 +1,6 @@
l_orderkey,cnt,row_num
-------------------------------
-1,2,1
-2,1,3
-3,2,2
\ No newline at end of file
+1,2,2
+2,1,4
+3,2,3
+null,3,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy1.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy1.result
index a44b4e0..ff25a8f 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy1.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy1.result
@@ -4,4 +4,7 @@ l_orderkey,l_discount,r1
1,0.04,2
3,0.06,3
1,0.09,4
-3,0.1,5
\ No newline at end of file
+3,0.1,5
+null,null,6
+null,null,6
+null,null,6
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy2.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy2.result
index b87197a..15baa48 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy2.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy2.result
@@ -4,4 +4,7 @@ l_orderkey,l_partkey,r1
1,1,1
2,2,1
3,2,1
-3,3,2
\ No newline at end of file
+3,3,2
+null,null,1
+null,null,1
+null,null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy3.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy3.result
index 68d0f85..7dca033 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy3.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy3.result
@@ -4,4 +4,7 @@ l_orderkey,l_partkey,r1
1,1,1
2,2,1
3,3,1
-3,2,2
\ No newline at end of file
+3,2,2
+null,null,1
+null,null,1
+null,null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy4.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy4.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy4.result
index 7042fb5..96b2cf9 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy4.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithOrderBy4.result
@@ -1,7 +1,10 @@
l_orderkey,l_partkey,r1,r2
-------------------------------
-3,3,4,1
-2,2,3,2
-3,2,4,2
-1,1,1,4
-1,1,1,4
\ No newline at end of file
+null,null,6,1
+null,null,6,1
+null,null,6,1
+3,3,4,4
+2,2,3,5
+3,2,4,5
+1,1,1,7
+1,1,1,7
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery.result
index bb3d8e6..d1f7fa9 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery.result
@@ -4,4 +4,5 @@ AFRICA,1,1
AMERICA,1,2
ASIA,1,3
EUROPE,1,4
-MIDDLE EAST,1,5
\ No newline at end of file
+MIDDLE EAST,1,5
+null,3,6
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery3.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery3.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery3.result
index 1a91238..6068c44 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery3.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery3.result
@@ -4,4 +4,5 @@ AFRICA,0,1
AMERICA,1,1
ASIA,2,1
EUROPE,3,1
-MIDDLE EAST,4,1
\ No newline at end of file
+MIDDLE EAST,4,1
+null,null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery4.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery4.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery4.result
index 1a91238..6068c44 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery4.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery4.result
@@ -4,4 +4,5 @@ AFRICA,0,1
AMERICA,1,1
ASIA,2,1
EUROPE,3,1
-MIDDLE EAST,4,1
\ No newline at end of file
+MIDDLE EAST,4,1
+null,null,1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery5.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery5.result b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery5.result
index 7668823..def5085 100644
--- a/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery5.result
+++ b/tajo-core-tests/src/test/resources/results/TestWindowQuery/testWindowWithSubQuery5.result
@@ -2,4 +2,7 @@ r_name,ran
-------------------------------
ASIA,3
EUROPE,4
-MIDDLE EAST,5
\ No newline at end of file
+MIDDLE EAST,5
+null,6
+null,6
+null,6
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
index c635e5d..f1ec0c0 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForBaseTable.result
@@ -1,4 +1,4 @@
--
-- Name: db1.table2; Type: TABLE; Storage: TEXT
--
-CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul');
\ No newline at end of file
+CREATE TABLE db1.table2 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'text.null'='\\N', 'timezone'='Asia/Seoul');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
index 011dfc1..02d1607 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLForExternalTable.result
@@ -2,4 +2,4 @@
-- Name: db1.table1; Type: TABLE; Storage: TEXT
-- Path: /table1
--
-CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1';
\ No newline at end of file
+CREATE EXTERNAL TABLE db1.table1 (name BLOB, addr TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'text.null'='\\N', 'timezone'='Asia/Seoul') PARTITION BY COLUMN(key INT4, key2 TEXT) LOCATION '/table1';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
index a432033..c51a68a 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName1.result
@@ -2,4 +2,4 @@
-- Name: db1."TABLE2"; Type: TABLE; Storage: TEXT
-- Path: /table1
--
-CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1';
\ No newline at end of file
+CREATE EXTERNAL TABLE db1."TABLE2" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'text.null'='\\N', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4) LOCATION '/table1';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
index 2e175de..80ada63 100644
--- a/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
+++ b/tajo-core-tests/src/test/resources/results/testDDLBuilder/testBuildDDLQuotedTableName2.result
@@ -1,4 +1,4 @@
--
-- Name: db1."TABLE1"; Type: TABLE; Storage: TEXT
--
-CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4);
\ No newline at end of file
+CREATE TABLE db1."TABLE1" (name BLOB, addr TEXT, "FirstName" TEXT, "LastName" TEXT, "with" TEXT) USING TEXT WITH ('compression.codec'='org.apache.hadoop.io.compress.GzipCodec', 'text.delimiter'='|', 'text.null'='\\N', 'timezone'='Asia/Seoul') PARTITION BY COLUMN("BirthYear" INT4);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java
index 9cb95f7..eb5d9b6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java
@@ -48,7 +48,8 @@ public final class Rank extends WindowAggFunc {
public static boolean checkIfDistinctValue(RankContext context, Tuple params) {
for (int i = 0; i < context.latest.length; i++) {
- if (!context.latest[i].equalsTo(params.asDatum(i)).isTrue()) {
+ if ((context.latest[i].isNotNull() || params.asDatum(i).isNotNull())
+ && !context.latest[i].equalsTo(params.asDatum(i)).isTrue()) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
index 5ff2067..87a6e74 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java
@@ -251,7 +251,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
}
@VisibleForTesting
- public long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) throws IOException {
+ public static long estimateSizeRecursive(TaskAttemptContext ctx, String [] tableIds) {
long size = 0;
for (String tableId : tableIds) {
FragmentProto[] fragmentProtos = ctx.getTables(tableId);
@@ -484,21 +484,23 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
}
}
- private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, JoinNode plan,
- PhysicalExec leftExec, PhysicalExec rightExec) throws IOException {
- String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild());
- long rightTableVolume = estimateSizeRecursive(context, rightLineage);
- boolean hashJoin;
+ private static boolean isHashOuterJoinFeasible(TaskAttemptContext context, LogicalNode innerRelation) {
+ String [] rightLineage = PlannerUtil.getRelationLineage(innerRelation);
+ long estimatedVolume = estimateSizeRecursive(context, rightLineage);
QueryContext queryContext = context.getQueryContext();
if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) {
- hashJoin = rightTableVolume <= queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB;
+ return estimatedVolume <= queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB;
} else {
- hashJoin = rightTableVolume <= queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB;
+ return estimatedVolume <= queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB;
}
+ }
- if (hashJoin) {
+ private PhysicalExec createBestLeftOuterJoinPlan(TaskAttemptContext context, JoinNode plan,
+ PhysicalExec leftExec, PhysicalExec rightExec) throws IOException {
+
+ if (isHashOuterJoinFeasible(context, plan.getRightChild())) {
// we can implement left outer join using hash join, using the right operand as the build relation
LOG.info("Left Outer Join (" + plan.getPID() +") chooses [Hash Join].");
return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec);
@@ -514,19 +516,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
PhysicalExec leftExec, PhysicalExec rightExec) throws IOException {
//if the left operand is small enough => implement it as a left outer hash join with exchanged operators (note:
// blocking, but merge join is blocking as well)
- String [] outerLineage4 = PlannerUtil.getRelationLineage(plan.getLeftChild());
- long leftTableVolume = estimateSizeRecursive(context, outerLineage4);
- boolean hashJoin;
-
- QueryContext queryContext = context.getQueryContext();
-
- if (queryContext.containsKey(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT)) {
- hashJoin = leftTableVolume <= queryContext.getLong(SessionVars.OUTER_HASH_JOIN_SIZE_LIMIT) * StorageUnit.MB;
- } else {
- hashJoin = leftTableVolume <= queryContext.getLong(SessionVars.HASH_JOIN_SIZE_LIMIT)* StorageUnit.MB;
- }
-
- if (hashJoin){
+ if (isHashOuterJoinFeasible(context, plan.getLeftChild())){
LOG.info("Right Outer Join (" + plan.getPID() +") chooses [Hash Join].");
return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec);
} else {
@@ -649,12 +639,10 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
private PhysicalExec createBestFullOuterJoinPlan(TaskAttemptContext context, JoinNode plan,
PhysicalExec leftExec, PhysicalExec rightExec) throws IOException {
- String [] leftLineage = PlannerUtil.getRelationLineage(plan.getLeftChild());
- String [] rightLineage = PlannerUtil.getRelationLineage(plan.getRightChild());
- long outerSize2 = estimateSizeRecursive(context, leftLineage);
- long innerSize2 = estimateSizeRecursive(context, rightLineage);
- final long threshold = 1048576 * 128;
- if (outerSize2 < threshold || innerSize2 < threshold) {
+ // The inner relation is always expected to be smaller than the outer relation.
+ // (See GreedyHeuristicJoinOrderAlgorithm:::swapLeftAndRightIfNecessary().
+ // Thus, we need to evaluate only that the right table is able to be loaded or not.
+ if (isHashOuterJoinFeasible(context, plan.getRightChild())) {
return createFullOuterHashJoinPlan(context, plan, leftExec, rightExec);
} else {
return createFullOuterMergeJoinPlan(context, plan, leftExec, rightExec);
@@ -676,6 +664,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
return new HashLeftSemiJoinExec(context, plan, leftExec, rightExec);
default:
+ // TODO: implement sort-based semi join operator
LOG.error("Invalid Left Semi Join Algorithm Enforcer: " + algorithm.name());
LOG.error("Choose a fallback inner join algorithm: " + JoinAlgorithm.IN_MEMORY_HASH_JOIN.name());
return new HashLeftOuterJoinExec(context, plan, leftExec, rightExec);
@@ -701,6 +690,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
return new HashLeftSemiJoinExec(context, plan, rightExec, leftExec);
default:
+ // TODO: implement sort-based semi join operator
LOG.error("Invalid Left Semi Join Algorithm Enforcer: " + algorithm.name());
LOG.error("Choose a fallback inner join algorithm: " + JoinAlgorithm.IN_MEMORY_HASH_JOIN.name());
return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec);
@@ -726,6 +716,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
return new HashLeftAntiJoinExec(context, plan, leftExec, rightExec);
default:
+ // TODO: implement sort-based anti join operator
LOG.error("Invalid Left Semi Join Algorithm Enforcer: " + algorithm.name());
LOG.error("Choose a fallback inner join algorithm: " + JoinAlgorithm.IN_MEMORY_HASH_JOIN.name());
return new HashLeftAntiJoinExec(context, plan, leftExec, rightExec);
@@ -751,6 +742,7 @@ public class PhysicalPlannerImpl implements PhysicalPlanner {
return new HashLeftSemiJoinExec(context, plan, rightExec, leftExec);
default:
+ // TODO: implement sort-based anti join operator
LOG.error("Invalid Left Semi Join Algorithm Enforcer: " + algorithm.name());
LOG.error("Choose a fallback inner join algorithm: " + JoinAlgorithm.IN_MEMORY_HASH_JOIN.name());
return new HashLeftOuterJoinExec(context, plan, rightExec, leftExec);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
index 92a68bd..2cda46e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonHashJoinExec.java
@@ -19,21 +19,19 @@
package org.apache.tajo.engine.planner.physical;
import org.apache.tajo.SessionVars;
-import org.apache.tajo.catalog.Column;
+import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.engine.planner.KeyProjector;
+import org.apache.tajo.datum.Datum;
import org.apache.tajo.engine.utils.CacheHolder;
import org.apache.tajo.engine.utils.TableCacheKey;
-import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.plan.logical.JoinNode;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.worker.ExecutionBlockSharedResource;
import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Iterator;
-import java.util.List;
/**
* common exec for all hash join execs
@@ -45,66 +43,14 @@ public abstract class CommonHashJoinExec<T> extends CommonJoinExec {
// temporal tuples and states for nested loop join
protected boolean first = true;
protected TupleMap<T> tupleSlots;
-
protected Iterator<Tuple> iterator;
- protected final boolean isCrossJoin;
- protected final List<Column[]> joinKeyPairs;
-
- protected final int rightNumCols;
- protected final int leftNumCols;
-
- protected final Column[] leftKeyList;
- protected final Column[] rightKeyList;
-
- protected final KeyProjector leftKeyExtractor;
-
protected boolean finished;
protected TableStats tableStatsOfCachedRightChild = null;
public CommonHashJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec outer, PhysicalExec inner) {
super(context, plan, outer, inner);
-
- switch (plan.getJoinType()) {
-
- case CROSS:
- if (hasJoinQual) {
- throw new TajoInternalError("Cross join cannot evaluate join conditions.");
- } else {
- isCrossJoin = true;
- joinKeyPairs = null;
- rightNumCols = leftNumCols = -1;
- leftKeyList = rightKeyList = null;
- leftKeyExtractor = null;
- }
- break;
-
- case INNER:
- // Other join types except INNER join can have empty join condition.
- if (!hasJoinQual) {
- throw new TajoInternalError("Inner join must have any join conditions.");
- }
- default:
- isCrossJoin = false;
- // HashJoin only can manage equi join key pairs.
- this.joinKeyPairs = PlannerUtil.getJoinKeyPairs(joinQual, outer.getSchema(),
- inner.getSchema(), false);
-
- leftKeyList = new Column[joinKeyPairs.size()];
- rightKeyList = new Column[joinKeyPairs.size()];
-
- for (int i = 0; i < joinKeyPairs.size(); i++) {
- leftKeyList[i] = outer.getSchema().getColumn(joinKeyPairs.get(i)[0].getQualifiedName());
- rightKeyList[i] = inner.getSchema().getColumn(joinKeyPairs.get(i)[1].getQualifiedName());
- }
-
- leftNumCols = outer.getSchema().size();
- rightNumCols = inner.getSchema().size();
-
- leftKeyExtractor = new KeyProjector(leftSchema, leftKeyList);
- break;
- }
}
protected void loadRightToHashTable() throws IOException {
@@ -138,7 +84,7 @@ public abstract class CommonHashJoinExec<T> extends CommonJoinExec {
}
protected TupleMap<TupleList> buildRightToHashTable() throws IOException {
- if (isCrossJoin) {
+ if (plan.getJoinType().equals(JoinType.CROSS)) {
return buildRightToHashTableForCrossJoin();
} else {
return buildRightToHashTableForNonCrossJoin();
@@ -160,20 +106,39 @@ public abstract class CommonHashJoinExec<T> extends CommonJoinExec {
protected TupleMap<TupleList> buildRightToHashTableForNonCrossJoin() throws IOException {
Tuple tuple;
TupleMap<TupleList> map = new TupleMap<>(context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE));
- KeyProjector keyProjector = new KeyProjector(rightSchema, rightKeyList);
while (!context.isStopped() && (tuple = rightChild.next()) != null) {
- KeyTuple keyTuple = keyProjector.project(tuple);
- TupleList newValue = map.get(keyTuple);
- if (newValue == null) {
- map.put(keyTuple, newValue = new TupleList());
+ KeyTuple keyTuple = rightKeyExtractor.project(tuple);
+ if (isLoadable(plan, keyTuple)) { // filter out null values
+ TupleList newValue = map.get(keyTuple);
+ if (newValue == null) {
+ map.put(keyTuple, newValue = new TupleList());
+ }
+ // if source is scan or groupby, it needs not to be cloned
+ newValue.add(tuple);
}
- // if source is scan or groupby, it needs not to be cloned
- newValue.add(tuple);
}
return map;
}
+ /**
+ * Check the given tuple is able to be loaded into the hash table or not.
+ * When the plan is full outer join, every tuple including null values should be loaded
+ * because both input tables of the join are preserved-row relations as well as null-supplying relations.
+ *
+ * Otherwise, except for anti join, only the tuples not containing null values should be loaded.
+ *
+ * For the case of anti join, the right table is expected to be empty if there are any null values.
+ *
+ * @param plan
+ * @param tuple
+ * @return
+ */
+ private static boolean isLoadable(JoinNode plan, Tuple tuple) {
+ return plan.getJoinType().equals(JoinType.FULL_OUTER)
+ || Arrays.stream(tuple.getValues()).noneMatch(Datum::isNull);
+ }
+
// todo: convert loaded data to cache condition
protected abstract TupleMap<T> convert(TupleMap<TupleList> hashed, boolean fromCache)
throws IOException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java
index 6653157..96ed0a6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/CommonJoinExec.java
@@ -20,16 +20,16 @@ package org.apache.tajo.engine.planner.physical;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.SchemaUtil;
+import org.apache.tajo.engine.planner.KeyProjector;
import org.apache.tajo.engine.planner.Projector;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.expr.BinaryEval;
+import org.apache.tajo.exception.TajoInternalError;
import org.apache.tajo.plan.expr.EvalNode;
import org.apache.tajo.plan.expr.EvalTreeUtil;
import org.apache.tajo.plan.logical.JoinNode;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.storage.FrameTuple;
import org.apache.tajo.storage.NullTuple;
import org.apache.tajo.storage.Tuple;
@@ -38,7 +38,6 @@ import org.apache.tajo.worker.TaskAttemptContext;
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
-import java.util.LinkedHashSet;
import java.util.List;
@@ -58,6 +57,17 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
protected final Schema leftSchema;
protected final Schema rightSchema;
+ protected final KeyProjector leftKeyExtractor;
+ protected final KeyProjector rightKeyExtractor;
+
+ protected final List<Column[]> joinKeyPairs;
+
+ protected final int rightNumCols;
+ protected final int leftNumCols;
+
+ protected final Column[] leftKeyList;
+ protected final Column[] rightKeyList;
+
protected final FrameTuple frameTuple;
// projection
@@ -70,7 +80,7 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
this.leftSchema = outer.getSchema();
this.rightSchema = inner.getSchema();
if (plan.hasJoinQual()) {
- EvalNode[] extracted = extractJoinConditions(plan.getJoinQual(), leftSchema, rightSchema);
+ EvalNode[] extracted = EvalTreeUtil.extractJoinConditions(plan.getJoinQual(), leftSchema, rightSchema);
joinQual = extracted[0];
leftJoinFilter = extracted[1];
rightJoinFilter = extracted[2];
@@ -82,56 +92,46 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
// for join
this.frameTuple = new FrameTuple();
- }
- /**
- * It separates a singular CNF-formed join condition into a join condition, a left join filter, and
- * right join filter.
- *
- * @param joinQual the original join condition
- * @param leftSchema Left table schema
- * @param rightSchema Left table schema
- * @return Three element EvalNodes, 0 - join condition, 1 - left join filter, 2 - right join filter.
- */
- private EvalNode[] extractJoinConditions(EvalNode joinQual, Schema leftSchema, Schema rightSchema) {
- List<EvalNode> joinQuals = Lists.newArrayList();
- List<EvalNode> leftFilters = Lists.newArrayList();
- List<EvalNode> rightFilters = Lists.newArrayList();
- for (EvalNode eachQual : AlgebraicUtil.toConjunctiveNormalFormArray(joinQual)) {
- if (!(eachQual instanceof BinaryEval)) {
- continue; // todo 'between', etc.
- }
- BinaryEval binaryEval = (BinaryEval)eachQual;
- LinkedHashSet<Column> leftColumns = EvalTreeUtil.findUniqueColumns(binaryEval.getLeftExpr());
- LinkedHashSet<Column> rightColumns = EvalTreeUtil.findUniqueColumns(binaryEval.getRightExpr());
- boolean leftInLeft = leftSchema.containsAny(leftColumns);
- boolean rightInLeft = leftSchema.containsAny(rightColumns);
- boolean leftInRight = rightSchema.containsAny(leftColumns);
- boolean rightInRight = rightSchema.containsAny(rightColumns);
-
- boolean columnsFromLeft = leftInLeft || rightInLeft;
- boolean columnsFromRight = leftInRight || rightInRight;
- if (!columnsFromLeft && !columnsFromRight) {
- continue; // todo constant expression : this should be done in logical phase
- }
- if (columnsFromLeft ^ columnsFromRight) {
- if (columnsFromLeft) {
- leftFilters.add(eachQual);
+ switch (plan.getJoinType()) {
+
+ case CROSS:
+ if (hasJoinQual) {
+ throw new TajoInternalError("Cross join cannot evaluate join conditions.");
} else {
- rightFilters.add(eachQual);
+ joinKeyPairs = null;
+ rightNumCols = leftNumCols = -1;
+ leftKeyList = rightKeyList = null;
+ leftKeyExtractor = null;
+ rightKeyExtractor = null;
}
- continue;
- }
- if ((leftInLeft && rightInLeft) || (leftInRight && rightInRight)) {
- continue; // todo not allowed yet : this should be checked in logical phase
- }
- joinQuals.add(eachQual);
+ break;
+
+ case INNER:
+ // Other join types except INNER join can have empty join condition.
+ if (!hasJoinQual) {
+ throw new TajoInternalError("Inner join must have any join conditions.");
+ }
+ default:
+ // HashJoin only can manage equi join key pairs.
+ this.joinKeyPairs = PlannerUtil.getJoinKeyPairs(joinQual, outer.getSchema(),
+ inner.getSchema(), false);
+
+ leftKeyList = new Column[joinKeyPairs.size()];
+ rightKeyList = new Column[joinKeyPairs.size()];
+
+ for (int i = 0; i < joinKeyPairs.size(); i++) {
+ leftKeyList[i] = outer.getSchema().getColumn(joinKeyPairs.get(i)[0].getQualifiedName());
+ rightKeyList[i] = inner.getSchema().getColumn(joinKeyPairs.get(i)[1].getQualifiedName());
+ }
+
+ leftNumCols = outer.getSchema().size();
+ rightNumCols = inner.getSchema().size();
+
+ leftKeyExtractor = new KeyProjector(leftSchema, leftKeyList);
+ rightKeyExtractor = new KeyProjector(rightSchema, rightKeyList);
+ break;
}
- return new EvalNode[] {
- joinQuals.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(joinQuals),
- leftFilters.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(leftFilters),
- rightFilters.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(rightFilters)
- };
}
public JoinNode getPlan() {
@@ -145,7 +145,7 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
* @return True if an input tuple is matched to the left join filter
*/
protected boolean leftFiltered(Tuple left) {
- return leftJoinFilter != null && !leftJoinFilter.eval(left).asBool();
+ return leftJoinFilter != null && !leftJoinFilter.eval(left).isTrue();
}
/**
@@ -155,7 +155,7 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
* @return True if an input tuple is matched to the right join filter
*/
protected boolean rightFiltered(Tuple right) {
- return rightJoinFilter != null && !rightJoinFilter.eval(right).asBool();
+ return rightJoinFilter != null && !rightJoinFilter.eval(right).isTrue();
}
/**
@@ -175,7 +175,7 @@ public abstract class CommonJoinExec extends BinaryPhysicalExec {
return Iterators.filter(rightTuples.iterator(), new Predicate<Tuple>() {
@Override
public boolean apply(Tuple input) {
- return rightJoinFilter.eval(input).asBool();
+ return rightJoinFilter.eval(input).isTrue();
}
});
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
index f238ef0..0cfc15f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java
@@ -264,9 +264,7 @@ public class DistinctGroupbyThirdAggregationExec extends UnaryPhysicalExec {
}
if (seq == 0 && nonDistinctAggr != null) {
- if (!tuple.isBlankOrNull(nonDistinctAggr.inTupleIndex)) {
- nonDistinctAggr.merge(tuple);
- }
+ nonDistinctAggr.merge(tuple);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java
index 7020b9d..ca5bf48 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java
@@ -18,7 +18,10 @@
package org.apache.tajo.engine.planner.physical;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterators;
import org.apache.tajo.plan.logical.JoinNode;
+import org.apache.tajo.storage.FrameTuple;
import org.apache.tajo.storage.NullTuple;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.util.Pair;
@@ -93,7 +96,7 @@ public class HashFullOuterJoinExec extends CommonHashJoinExec<Pair<Boolean, Tupl
}
Tuple leftTuple = leftChild.next();
if (leftTuple == null) {
- // if no more tuples in left tuples, a join is completed.
+ // if no more tuples in left tuples, a join is completed.
// in this stage we can begin outputing tuples from the right operand (which were before in tupleSlots) null padded on the left side
frameTuple.setLeft(NullTuple.create(leftNumCols));
iterator = getUnmatchedRight();
@@ -102,17 +105,17 @@ public class HashFullOuterJoinExec extends CommonHashJoinExec<Pair<Boolean, Tupl
}
frameTuple.setLeft(leftTuple);
- if (leftFiltered(leftTuple)) {
- iterator = nullTupleList.iterator();
- continue;
- }
// getting corresponding right
Pair<Boolean, TupleList> hashed = tupleSlots.get(leftKeyExtractor.project(leftTuple));
if (hashed == null) {
- iterator = nullTupleList.iterator();
+ if (leftFiltered(leftTuple)) {
+ iterator = null;
+ } else {
+ iterator = nullTupleList.iterator();
+ }
continue;
}
- Iterator<Tuple> rightTuples = rightFiltered(hashed.getSecond());
+ Iterator<Tuple> rightTuples = joinQualFiltered(leftTuple, rightFiltered(hashed.getSecond()));
if (!rightTuples.hasNext()) {
iterator = nullTupleList.iterator();
continue;
@@ -124,6 +127,19 @@ public class HashFullOuterJoinExec extends CommonHashJoinExec<Pair<Boolean, Tupl
return null;
}
+ private Iterator<Tuple> joinQualFiltered(Tuple leftTuple, Iterator<Tuple> rightTuples) {
+ final FrameTuple frameTuple = new FrameTuple();
+ frameTuple.setLeft(leftTuple);
+
+ return Iterators.filter(rightTuples, new Predicate<Tuple>() {
+ @Override
+ public boolean apply(Tuple input) {
+ frameTuple.setRight(input);
+ return joinQual.eval(frameTuple).isTrue();
+ }
+ });
+ }
+
@Override
protected TupleMap<Pair<Boolean, TupleList>> convert(TupleMap<TupleList> hashed,
boolean fromCache) throws IOException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java
index e47e515..6a84de1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java
@@ -18,6 +18,7 @@
package org.apache.tajo.engine.planner.physical;
+import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -27,9 +28,12 @@ import java.util.Iterator;
public class HashJoinExec extends CommonHashJoinExec<TupleList> {
+ private final boolean isCrossJoin;
+
public HashJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec leftExec,
PhysicalExec rightExec) {
super(context, plan, leftExec, rightExec);
+ isCrossJoin = plan.getJoinType().equals(JoinType.CROSS);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java
index b652c3c..1dbca04 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java
@@ -18,8 +18,6 @@
package org.apache.tajo.engine.planner.physical;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.worker.TaskAttemptContext;
@@ -30,7 +28,6 @@ import java.util.List;
public class HashLeftOuterJoinExec extends HashJoinExec {
- private static final Log LOG = LogFactory.getLog(HashLeftOuterJoinExec.class);
private final List<Tuple> nullTupleList;
public HashLeftOuterJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec leftChild,
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
index 824fb0e..b1b1f28 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java
@@ -40,6 +40,8 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
private Tuple prevLeftTuple = null;
private Tuple prevRightTuple = null;
+ private Tuple preservedTuple = null;
+
private TupleList leftTupleSlots;
private TupleList rightTupleSlots;
@@ -67,6 +69,7 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
final int INITIAL_TUPLE_SLOT = context.getQueryContext().getInt(SessionVars.JOIN_HASH_TABLE_SIZE);
this.leftTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
this.rightTupleSlots = new TupleList(INITIAL_TUPLE_SLOT);
+
SortSpec[][] sortSpecs = new SortSpec[2][];
sortSpecs[0] = leftSortKey;
sortSpecs[1] = rightSortKey;
@@ -90,6 +93,12 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
Tuple outTuple;
while (!context.isStopped()) {
+ if (preservedTuple != null) {
+ outTuple = preservedTuple;
+ preservedTuple = null;
+ return outTuple;
+ }
+
boolean newRound = false;
if((posRightTupleSlots == -1) && (posLeftTupleSlots == -1)) {
newRound = true;
@@ -98,7 +107,7 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
newRound = true;
}
- if(newRound == true){
+ if(newRound){
if (end) {
@@ -113,7 +122,7 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
// right side and a right-padded tuple should be built for all remaining
// left side
- if (initRightDone == false) {
+ if (!initRightDone) {
// maybe the left operand was empty => the right one didn't have the chance to initialize
rightTuple = rightChild.next();
initRightDone = true;
@@ -173,7 +182,7 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
////////////////////////////////////////////////////////////////////////
// advance alternatively on each side until a match is found
int cmp;
- while (!end && ((cmp = joincomparator.compare(leftTuple, rightTuple)) != 0)) {
+ if (!end && ((cmp = joincomparator.compare(leftTuple, rightTuple)) != 0)) {
if (cmp > 0) {
@@ -205,9 +214,32 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
return outTuple;
- } // if (cmp < 0)
- } //while
+ }
+ }
+
+ // Check null values
+ Tuple leftKey = leftKeyExtractor.project(leftTuple);
+ boolean containNull = false;
+ for (int i = 0; i < leftKey.size(); i++) {
+ if (leftKey.isBlankOrNull(i)) {
+ containNull = true;
+ break;
+ }
+ }
+
+ if (containNull) {
+ frameTuple.set(leftTuple, rightNullTuple);
+ outTuple = projector.eval(frameTuple);
+ frameTuple.set(leftNullTuple, rightTuple);
+ preservedTuple = new VTuple(projector.eval(frameTuple));
+ leftTuple = leftChild.next();
+ rightTuple = rightChild.next();
+ if (leftTuple == null || rightTuple == null) {
+ end = true;
+ }
+ return outTuple;
+ }
////////////////////////////////////////////////////////////////////////
// SLOTS POPULATION STAGE
@@ -228,8 +260,7 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
endLeft = true;
}
-
- } while ((endLeft != true) && (tupleComparator[0].compare(prevLeftTuple, leftTuple) == 0));
+ } while ((!endLeft) && (tupleComparator[0].compare(prevLeftTuple, leftTuple) == 0));
posLeftTupleSlots = 0;
prevRightTuple.put(rightTuple.getValues());
@@ -240,10 +271,10 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
endRight = true;
}
- } while ((endRight != true) && (tupleComparator[1].compare(prevRightTuple, rightTuple) == 0) );
+ } while ((!endRight) && (tupleComparator[1].compare(prevRightTuple, rightTuple) == 0) );
posRightTupleSlots = 0;
- if ((endLeft == true) || (endRight == true)) {
+ if ((endLeft) || (endRight)) {
end = true;
endInPopulationStage = true;
}
@@ -262,12 +293,12 @@ public class MergeFullOuterJoinExec extends CommonJoinExec {
if(!end || (end && endInPopulationStage)){
if(posLeftTupleSlots == 0){
leftNext = leftTupleSlots.get(posLeftTupleSlots);
- posLeftTupleSlots = posLeftTupleSlots + 1;
+ posLeftTupleSlots++;
}
if(posRightTupleSlots <= (rightTupleSlots.size() -1)) {
Tuple aTuple = rightTupleSlots.get(posRightTupleSlots);
- posRightTupleSlots = posRightTupleSlots + 1;
+ posRightTupleSlots++;
frameTuple.set(leftNext, aTuple);
joinQual.eval(frameTuple);
return projector.eval(frameTuple);
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
index 706ec3e..40fb9f1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java
@@ -208,6 +208,28 @@ public class RightOuterMergeJoinExec extends CommonJoinExec {
// END MOVE FORWARDING STAGE
//////////////////////////////////////////////////////////////////////
+ // Check null values
+ Tuple leftKey = rightKeyExtractor.project(rightTuple);
+ boolean containNull = false;
+ for (int i = 0; i < leftKey.size(); i++) {
+ if (leftKey.isBlankOrNull(i)) {
+ containNull = true;
+ break;
+ }
+ }
+
+ if (containNull) {
+ frameTuple.set(nullPaddedTuple, rightTuple);
+ outTuple = projector.eval(frameTuple);
+ leftTuple = leftChild.next();
+ rightTuple = rightChild.next();
+
+ if (leftTuple == null || rightTuple == null) {
+ end = true;
+ }
+ return outTuple;
+ }
+
// once a match is found, retain all tuples with this key in tuple slots on each side
if(!end) {
endInPopulationStage = false;
@@ -282,7 +304,7 @@ public class RightOuterMergeJoinExec extends CommonJoinExec {
posRightTupleSlots = posRightTupleSlots + 1;
frameTuple.set(nextLeft, aTuple);
- if (joinQual.eval(frameTuple).asBool()) {
+ if (joinQual.eval(frameTuple).isTrue()) {
return projector.eval(frameTuple);
} else {
// padding null
@@ -301,7 +323,7 @@ public class RightOuterMergeJoinExec extends CommonJoinExec {
frameTuple.set(nextLeft, aTuple);
- if (joinQual.eval(frameTuple).asBool()) {
+ if (joinQual.eval(frameTuple).isTrue()) {
return projector.eval(frameTuple);
} else {
// padding null
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index c04e3b2..050029e 100644
--- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -36,7 +36,6 @@ import org.apache.tajo.storage.*;
import org.apache.tajo.tuple.memory.MemoryBlock;
import org.apache.tajo.tuple.memory.MemoryRowBlock;
import org.apache.tajo.util.CompressionUtil;
-import org.apache.tajo.util.KeyValueSet;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -206,11 +205,9 @@ public class TestResultSet {
String [] data = {
"2014-01-01|01:00:00|2014-01-01 01:00:00"
};
- KeyValueSet tableOptions = new KeyValueSet();
- tableOptions.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
res = TajoTestingCluster
- .run(table, schemas, tableOptions, new String[][]{data}, query, client);
+ .run(table, schemas, new String[][]{data}, query, client);
assertTrue(res.next());
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index faaba71..c36b133 100644
--- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -80,12 +80,17 @@ public class TestTajoJdbc extends QueryTestCaseBase {
Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
+ result.put(null, 3);
assertNotNull(res);
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
+ assertTrue(res.next());
+ assertNull(res.getString(1));
+ assertNull(res.getString(2));
+ assertTrue(result.get(null) == res.getInt(3));
assertFalse(res.next());
ResultSetMetaData rsmd = res.getMetaData();
@@ -380,12 +385,17 @@ public class TestTajoJdbc extends QueryTestCaseBase {
Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
+ result.put(null, 3);
assertNotNull(res);
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
+ assertTrue(res.next());
+ assertNull(res.getString(1));
+ assertNull(res.getString(2));
+ assertTrue(result.get(null) == res.getInt(3));
assertFalse(res.next());
ResultSetMetaData rsmd = res.getMetaData();
@@ -438,12 +448,17 @@ public class TestTajoJdbc extends QueryTestCaseBase {
Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
+ result.put(null, 3);
assertNotNull(res);
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
assertTrue(res.next());
assertTrue(result.get(res.getString(1) + res.getString(2)) == res.getInt(3));
+ assertTrue(res.next());
+ assertNull(res.getString(1));
+ assertNull(res.getString(2));
+ assertTrue(result.get(null) == res.getInt(3));
assertFalse(res.next());
ResultSetMetaData rsmd = res.getMetaData();
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 5336906..6495af3 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -49,6 +49,7 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode;
import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.Pair;
import org.apache.tajo.util.StringUtils;
@@ -2079,13 +2080,17 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
* @param exprs
* @return
*/
- private static String[] convertExprsToStrings(Expr[] exprs) {
+ private static String[] convertExprsToPartitionTableStringValues(Expr[] exprs) {
int exprCount = exprs.length;
String[] values = new String[exprCount];
for(int i = 0; i < exprCount; i++) {
- LiteralValue expr = (LiteralValue)exprs[i];
- values[i] = expr.getValue();
+ if (exprs[i].getType() == OpType.NullLiteral) {
+ values[i] = StorageConstants.DEFAULT_PARTITION_NAME;
+ } else {
+ LiteralValue expr = (LiteralValue) exprs[i];
+ values[i] = expr.getValue();
+ }
}
return values;
@@ -2160,7 +2165,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
}
if (alterTable.getValues() != null) {
- alterTableNode.setPartitionValues(convertExprsToStrings(alterTable.getValues()));
+ alterTableNode.setPartitionValues(convertExprsToPartitionTableStringValues(alterTable.getValues()));
}
if (alterTable.getLocation() != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
index bb48030..a83c5bd 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java
@@ -19,6 +19,7 @@
package org.apache.tajo.plan.expr;
import com.google.common.base.Function;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.tajo.algebra.ColumnReferenceExpr;
@@ -243,6 +244,56 @@ public class EvalTreeUtil {
}
/**
+ * It separates a singular CNF-formed join condition into a join condition, a left join filter, and
+ * right join filter.
+ *
+ * @param joinQual the original join condition
+ * @param leftSchema Left table schema
+ * @param rightSchema Left table schema
+ * @return Three element EvalNodes, 0 - join condition, 1 - left join filter, 2 - right join filter.
+ */
+ public static EvalNode[] extractJoinConditions(EvalNode joinQual, Schema leftSchema, Schema rightSchema) {
+ List<EvalNode> joinQuals = Lists.newArrayList();
+ List<EvalNode> leftFilters = Lists.newArrayList();
+ List<EvalNode> rightFilters = Lists.newArrayList();
+ for (EvalNode eachQual : AlgebraicUtil.toConjunctiveNormalFormArray(joinQual)) {
+ if (!(eachQual instanceof BinaryEval)) {
+ continue; // todo 'between', etc.
+ }
+ BinaryEval binaryEval = (BinaryEval)eachQual;
+ LinkedHashSet<Column> leftColumns = EvalTreeUtil.findUniqueColumns(binaryEval.getLeftExpr());
+ LinkedHashSet<Column> rightColumns = EvalTreeUtil.findUniqueColumns(binaryEval.getRightExpr());
+ boolean leftInLeft = leftSchema.containsAny(leftColumns);
+ boolean rightInLeft = leftSchema.containsAny(rightColumns);
+ boolean leftInRight = rightSchema.containsAny(leftColumns);
+ boolean rightInRight = rightSchema.containsAny(rightColumns);
+
+ boolean columnsFromLeft = leftInLeft || rightInLeft;
+ boolean columnsFromRight = leftInRight || rightInRight;
+ if (!columnsFromLeft && !columnsFromRight) {
+ continue; // todo constant expression : this should be done in logical phase
+ }
+ if (columnsFromLeft ^ columnsFromRight) {
+ if (columnsFromLeft) {
+ leftFilters.add(eachQual);
+ } else {
+ rightFilters.add(eachQual);
+ }
+ continue;
+ }
+ if ((leftInLeft && rightInLeft) || (leftInRight && rightInRight)) {
+ continue; // todo not allowed yet : this should be checked in logical phase
+ }
+ joinQuals.add(eachQual);
+ }
+ return new EvalNode[] {
+ joinQuals.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(joinQuals),
+ leftFilters.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(leftFilters),
+ rightFilters.isEmpty() ? null : AlgebraicUtil.createSingletonExprFromCNF(rightFilters)
+ };
+ }
+
+ /**
* If a given expression is join condition, it returns TRUE. Otherwise, it returns FALSE.
*
* If three conditions are satisfied, we can recognize the expression as a equi join condition.
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java
index 085d82c..2bf8d3e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java
@@ -51,9 +51,9 @@ public class SignedEval extends UnaryEval implements Cloneable {
@SuppressWarnings("unchecked")
public Datum eval(Tuple tuple) {
super.eval(tuple);
- NumericDatum result = child.eval(tuple);
- if (negative) {
- return result.inverseSign();
+ Datum result = child.eval(tuple);
+ if (result.isNotNull() && negative) {
+ return ((NumericDatum)result).inverseSign();
}
return result;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-plan/src/main/java/org/apache/tajo/plan/function/PythonAggFunctionInvoke.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/PythonAggFunctionInvoke.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/PythonAggFunctionInvoke.java
index 6f010ab..70d6348 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/PythonAggFunctionInvoke.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/PythonAggFunctionInvoke.java
@@ -116,7 +116,8 @@ public class PythonAggFunctionInvoke extends AggFunctionInvoke implements Clonea
public Datum getPartialResult(FunctionContext context) {
updateContextIfNecessary(context);
// partial results are stored as json strings.
- return DatumFactory.createText(scriptEngine.getPartialResult(context));
+ String result = scriptEngine.getPartialResult(context);
+ return DatumFactory.createText(result);
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index cf92ea0..32e41d3 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -22,12 +22,16 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.tajo.OverridableConf;
+import org.apache.tajo.annotation.Nullable;
import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
-import org.apache.tajo.catalog.proto.CatalogProtos.PartitionsByAlgebraProto;
import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto;
+import org.apache.tajo.catalog.proto.CatalogProtos.PartitionsByAlgebraProto;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.exception.*;
@@ -39,12 +43,15 @@ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
import org.apache.tajo.plan.util.EvalNodeToExprConverter;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
+import org.apache.tajo.storage.StorageConstants;
import org.apache.tajo.storage.Tuple;
import org.apache.tajo.storage.VTuple;
import org.apache.tajo.util.StringUtils;
import java.io.IOException;
-import java.util.*;
+import java.util.List;
+import java.util.Set;
+import java.util.Stack;
public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
private CatalogService catalog;
@@ -87,11 +94,13 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
private static class PartitionPathFilter implements PathFilter {
private Schema schema;
- private EvalNode partitionFilter;
- public PartitionPathFilter(Schema schema, EvalNode partitionFilter) {
+ private @Nullable EvalNode partitionFilter;
+ public PartitionPathFilter(Schema schema, @Nullable EvalNode partitionFilter) {
this.schema = schema;
this.partitionFilter = partitionFilter;
- partitionFilter.bind(null, schema);
+ if (this.partitionFilter != null) {
+ this.partitionFilter.bind(null, schema);
+ }
}
@Override
@@ -101,7 +110,11 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
return false;
}
- return partitionFilter.eval(tuple).asBool();
+ if (partitionFilter != null) {
+ return partitionFilter.eval(tuple).isTrue();
+ } else {
+ return true;
+ }
}
@Override
@@ -304,16 +317,9 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
* @return The array of path filter, accpeting all partition paths.
*/
public static PathFilter [] buildAllAcceptingPathFilters(Schema partitionColumns) {
- Column target;
PathFilter [] filters = new PathFilter[partitionColumns.size()];
- List<EvalNode> accumulatedFilters = Lists.newArrayList();
for (int i = 0; i < partitionColumns.size(); i++) { // loop from one to level
- target = partitionColumns.getColumn(i);
- accumulatedFilters.add(new IsNullEval(true, new FieldEval(target)));
-
- EvalNode filterPerLevel = AlgebraicUtil.createSingletonExprFromCNF(
- accumulatedFilters.toArray(new EvalNode[accumulatedFilters.size()]));
- filters[i] = new PartitionPathFilter(partitionColumns, filterPerLevel);
+ filters[i] = new PartitionPathFilter(partitionColumns, null);
}
return filters;
}
@@ -463,7 +469,12 @@ public class PartitionedTableRewriter implements LogicalPlanRewriteRule {
}
int columnId = partitionColumnSchema.getColumnIdByName(parts[0]);
Column keyColumn = partitionColumnSchema.getColumn(columnId);
- tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), StringUtils.unescapePathName(parts[1])));
+ String pathName = StringUtils.unescapePathName(parts[1]);
+ if (pathName.equals(StorageConstants.DEFAULT_PARTITION_NAME)){
+ tuple.put(columnId, DatumFactory.createNullDatum());
+ } else {
+ tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), pathName));
+ }
}
for (; i < partitionColumnSchema.size(); i++) {
tuple.put(i, NullDatum.get());
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
index d9405e2..4ce6928 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/fragment/FragmentConvertor.java
@@ -47,8 +47,7 @@ public class FragmentConvertor {
*/
private static final Class<?>[] DEFAULT_FRAGMENT_PARAMS = { ByteString.class };
- public static Class<? extends Fragment> getFragmentClass(Configuration conf, String dataFormat)
- throws IOException {
+ public static Class<? extends Fragment> getFragmentClass(Configuration conf, String dataFormat) {
Class<? extends Fragment> fragmentClass = CACHED_FRAGMENT_CLASSES.get(dataFormat.toLowerCase());
if (fragmentClass == null) {
fragmentClass = conf.getClass(
@@ -57,7 +56,7 @@ public class FragmentConvertor {
}
if (fragmentClass == null) {
- throw new IOException("No such a fragment for " + dataFormat.toLowerCase());
+ throw new TajoInternalError("No such a fragment for " + dataFormat.toLowerCase());
}
return fragmentClass;
@@ -80,11 +79,10 @@ public class FragmentConvertor {
return result;
}
- public static <T extends Fragment> T convert(Configuration conf, FragmentProto fragment)
- throws IOException {
+ public static <T extends Fragment> T convert(Configuration conf, FragmentProto fragment) {
Class<T> fragmentClass = (Class<T>) getFragmentClass(conf, fragment.getDataFormat().toLowerCase());
if (fragmentClass == null) {
- throw new IOException("No such a fragment class for " + fragment.getDataFormat());
+ throw new TajoInternalError("No such a fragment class for " + fragment.getDataFormat());
}
return convert(fragmentClass, fragment);
}
@@ -101,7 +99,7 @@ public class FragmentConvertor {
return list;
}
- public static <T extends Fragment> List<T> convert(Configuration conf, FragmentProto...fragments) throws IOException {
+ public static <T extends Fragment> List<T> convert(Configuration conf, FragmentProto...fragments) {
List<T> list = Lists.newArrayList();
if (fragments == null) {
return list;
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
index 0717fae..94a0ba0 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
@@ -21,6 +21,7 @@ package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.tajo.BuiltinStorages;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -43,7 +44,7 @@ public abstract class TextLineSerDe {
public static ByteBuf getNullChars(TableMeta meta) {
byte[] nullCharByteArray;
- if (meta.getDataFormat().equals("SEQUENCEFILE")) {
+ if (meta.getDataFormat().equals(BuiltinStorages.SEQUENCE_FILE)) {
nullCharByteArray = getNullCharsAsBytes(meta, StorageConstants.SEQUENCEFILE_NULL, "\\");
} else {
nullCharByteArray = getNullCharsAsBytes(meta);
@@ -55,6 +56,13 @@ public abstract class TextLineSerDe {
return nullChars;
}
+ /**
+ * Returns the bytes of null characters.
+ * The default value is '\\N' as in Hive.
+ *
+ * @param meta table meta
+ * @return a byte array of null characters
+ */
public static byte [] getNullCharsAsBytes(TableMeta meta) {
return getNullCharsAsBytes(meta, StorageConstants.TEXT_NULL, NullDatum.DEFAULT_TEXT);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
index 0a4f917..6de1573 100644
--- a/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
+++ b/tajo-storage/tajo-storage-pgsql/src/test/java/org/apache/tajo/storage/pgsql/PgSQLTestServer.java
@@ -136,7 +136,7 @@ public class PgSQLTestServer {
private String restoreTableContents(String tableName) throws IOException {
Path filePath = new Path(testPath, tableName + ".tbl");
- storeTableContents("tpch/" + tableName + ".tbl", filePath);
+ storeTableContents("dataset/" + tableName + ".tbl", filePath);
return filePath.toUri().getPath();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker b/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
deleted file mode 100644
index 158780d..0000000
--- a/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/.marker
+++ /dev/null
@@ -1 +0,0 @@
-// for keeping dataset directory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/c156e5c9/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/customer.tbl
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/customer.tbl b/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/customer.tbl
new file mode 100644
index 0000000..8721995
--- /dev/null
+++ b/tajo-storage/tajo-storage-pgsql/src/test/resources/dataset/customer.tbl
@@ -0,0 +1,5 @@
+1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e
+2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref
+3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov
+4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou
+5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor
\ No newline at end of file