You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/08/11 06:49:47 UTC
tajo git commit: TAJO-1732: During filter push down phase,
join conditions are not set properly.
Repository: tajo
Updated Branches:
refs/heads/master f87f66729 -> caf8cca94
TAJO-1732: During filter push down phase, join conditions are not set properly.
Closes #670
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/caf8cca9
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/caf8cca9
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/caf8cca9
Branch: refs/heads/master
Commit: caf8cca94699a687f28e7fcf2c8f6efb67773983
Parents: f87f667
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Aug 11 13:49:43 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Aug 11 13:49:43 2015 +0900
----------------------------------------------------------------------
CHANGES | 4 +-
.../org/apache/tajo/benchmark/TestTPCH.java | 7 +
.../queries/TestTPCH/testFirstJoinInQ7.sql | 13 ++
.../results/TestTPCH/testFirstJoinInQ7.plan | 156 +++++++++++++++++++
.../results/TestTPCH/testFirstJoinInQ7.result | 4 +
.../org/apache/tajo/plan/logical/JoinNode.java | 4 +
.../plan/rewrite/rules/FilterPushDownRule.java | 3 +
7 files changed, 190 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 2b873b4..18de625 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,5 @@
Tajo Change Log
-
Release 0.11.0 - unreleased
NEW FEATURES
@@ -212,6 +211,9 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1732: During filter push down phase, join conditions are not set properly.
+ (jihoon)
+
TAJO-1742: Remove hadoop dependency in DatumFactory. (jinho)
TAJO-1733: Finished query occasionally does not appear in Web-UI. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java b/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
index 9b7db2d..6390f90 100644
--- a/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
+++ b/tajo-core/src/test/java/org/apache/tajo/benchmark/TestTPCH.java
@@ -60,4 +60,11 @@ public class TestTPCH extends QueryTestCaseBase {
public void testTPCHQ5() throws Exception {
runSimpleTests();
}
+
+ @Test
+ @Option(withExplain = true, withExplainGlobal = true)
+ @SimpleTest
+ public void testFirstJoinInQ7() throws Exception {
+ runSimpleTests();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql b/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql
new file mode 100644
index 0000000..2dfb83d
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTPCH/testFirstJoinInQ7.sql
@@ -0,0 +1,13 @@
+select
+ n1.n_name as supp_nation, n2.n_name as cust_nation, n1.n_nationkey as s_nationkey, n2.n_nationkey as c_nationkey
+from
+ nation n1 join nation n2
+ on
+ n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY'
+UNION ALL
+select
+ n1.n_name as supp_nation, n2.n_name as cust_nation, n1.n_nationkey as s_nationkey, n2.n_nationkey as c_nationkey
+from
+ nation n1 join nation n2
+ on
+ n2.n_name = 'FRANCE' and n1.n_name = 'GERMANY'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan
new file mode 100644
index 0000000..d17f26c
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan
@@ -0,0 +1,156 @@
+explain
+-------------------------------
+UNION(8)
+ PROJECTION(7)
+ => Targets: supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)
+ => out schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)}
+ => in schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)}
+ JOIN(11)(CROSS)
+ => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)
+ => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(4) on default.nation as n1
+ => filter: default.n1.n_name (TEXT) = GERMANY
+ => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)}
+ SCAN(5) on default.nation as n2
+ => filter: default.n2.n_name (TEXT) = FRANCE
+ => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)}
+ PROJECTION(3)
+ => Targets: supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)
+ => out schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)}
+ => in schema: {(4) supp_nation (TEXT), cust_nation (TEXT), s_nationkey (INT4), c_nationkey (INT4)}
+ JOIN(10)(CROSS)
+ => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)
+ => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(0) on default.nation as n1
+ => filter: default.n1.n_name (TEXT) = FRANCE
+ => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)}
+ SCAN(1) on default.nation as n2
+ => filter: default.n2.n_name (TEXT) = GERMANY
+ => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)}
+explain
+-------------------------------
+-------------------------------------------------------------------------------
+Execution Block Graph (TERMINAL - eb_0000000000000_0000_000007)
+-------------------------------------------------------------------------------
+|-eb_0000000000000_0000_000007
+ |-eb_0000000000000_0000_000006
+ |-eb_0000000000000_0000_000005
+ |-eb_0000000000000_0000_000004
+ |-eb_0000000000000_0000_000003
+ |-eb_0000000000000_0000_000002
+ |-eb_0000000000000_0000_000001
+-------------------------------------------------------------------------------
+Order of Execution
+-------------------------------------------------------------------------------
+1: eb_0000000000000_0000_000001
+2: eb_0000000000000_0000_000002
+3: eb_0000000000000_0000_000004
+4: eb_0000000000000_0000_000005
+5: eb_0000000000000_0000_000003
+6: eb_0000000000000_0000_000006
+7: eb_0000000000000_0000_000007
+-------------------------------------------------------------------------------
+
+=======================================================
+Block Id: eb_0000000000000_0000_000001 [LEAF]
+=======================================================
+
+[Outgoing]
+[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE)
+
+SCAN(1) on default.nation as n2
+ => filter: default.n2.n_name (TEXT) = GERMANY
+ => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000002 [LEAF]
+=======================================================
+
+[Outgoing]
+[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE)
+
+SCAN(0) on default.nation as n1
+ => filter: default.n1.n_name (TEXT) = FRANCE
+ => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000004 [LEAF]
+=======================================================
+
+[Outgoing]
+[q_0000000000000_0000] 4 => 6 (type=HASH_SHUFFLE)
+
+SCAN(5) on default.nation as n2
+ => filter: default.n2.n_name (TEXT) = FRANCE
+ => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000005 [LEAF]
+=======================================================
+
+[Outgoing]
+[q_0000000000000_0000] 5 => 6 (type=HASH_SHUFFLE)
+
+SCAN(4) on default.nation as n1
+ => filter: default.n1.n_name (TEXT) = GERMANY
+ => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000003 [ROOT]
+=======================================================
+
+[Incoming]
+[q_0000000000000_0000] 1 => 3 (type=HASH_SHUFFLE)
+[q_0000000000000_0000] 2 => 3 (type=HASH_SHUFFLE)
+
+JOIN(10)(CROSS)
+ => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)
+ => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(14) on eb_0000000000000_0000_000002
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(13) on eb_0000000000000_0000_000001
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000006 [ROOT]
+=======================================================
+
+[Incoming]
+[q_0000000000000_0000] 4 => 6 (type=HASH_SHUFFLE)
+[q_0000000000000_0000] 5 => 6 (type=HASH_SHUFFLE)
+
+JOIN(11)(CROSS)
+ => target list: c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)
+ => out schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(16) on eb_0000000000000_0000_000005
+ => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ => in schema: {(2) s_nationkey (INT4), supp_nation (TEXT)}
+ SCAN(15) on eb_0000000000000_0000_000004
+ => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+ => in schema: {(2) c_nationkey (INT4), cust_nation (TEXT)}
+
+=======================================================
+Block Id: eb_0000000000000_0000_000007 [TERMINAL]
+=======================================================
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result
new file mode 100644
index 0000000..9f5b85a
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTPCH/testFirstJoinInQ7.result
@@ -0,0 +1,4 @@
+supp_nation,cust_nation,s_nationkey,c_nationkey
+-------------------------------
+FRANCE,GERMANY,6,7
+GERMANY,FRANCE,7,6
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java
index 4f36026..8b843a9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java
@@ -61,6 +61,10 @@ public class JoinNode extends BinaryNode implements Projectable, Cloneable {
this.joinSpec.setSingletonPredicate(joinQual);
}
+ public void clearJoinQual() {
+ this.joinSpec.setPredicates(null);
+ }
+
public boolean hasJoinQual() {
return this.joinSpec.hasPredicates();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/caf8cca9/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
index 4fb8aac..30ef7a8 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java
@@ -177,6 +177,9 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<FilterPushDownCo
if (joinNode.hasJoinQual()) {
onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual())));
}
+ // clear join qual
+ joinNode.clearJoinQual();
+
// we assume all the quals in pushingDownFilters as where predicates
Set<EvalNode> nonPushableQuals = extractNonPushableJoinQuals(plan, block, joinNode, onPredicates,
context.pushingDownFilters);