You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2016/03/22 02:34:08 UTC
tajo git commit: TAJO-2093: Partition Pruning doesn't handle Constant
folding occasionally for BETWEEN clause.
Repository: tajo
Updated Branches:
refs/heads/branch-0.11.2 239370a8b -> 3f994bf2e
TAJO-2093: Partition Pruning doesn't handle Constant folding occasionally for BETWEEN clause.
Closes #975
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3f994bf2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3f994bf2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3f994bf2
Branch: refs/heads/branch-0.11.2
Commit: 3f994bf2efa0bd9b5b2c5fddfbc218ddec25a8c2
Parents: 239370a
Author: JaeHwa Jung <bl...@apache.org>
Authored: Tue Mar 22 10:32:59 2016 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Tue Mar 22 10:32:59 2016 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../planner/TestPartitionedTableRewriter.java | 152 ++++++++++++++++++-
.../apache/tajo/plan/expr/AlgebraicUtil.java | 2 +-
.../plan/exprrewrite/rules/ConstantFolding.java | 16 ++
4 files changed, 171 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 71ba9d5..80c4a3f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -24,6 +24,9 @@ Release 0.11.2 - unreleased
BUG FIXES
+ TAJO-2093: Partition Pruning doesn't handle Constant folding occasionally
+ for BETWEEN clause. (jaehwa)
+
TAJO-2100: Add missing cancellation in defaultTaskScheduler when a worker is
no respond. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
index e8b1011..8005893 100644
--- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
+++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPartitionedTableRewriter.java
@@ -41,7 +41,9 @@ import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.KeyValueSet;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import static org.junit.Assert.*;
@@ -50,6 +52,10 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase {
final static String PARTITION_TABLE_NAME = "tb_partition";
final static String MULTIPLE_PARTITION_TABLE_NAME = "tb_multiple_partition";
+ // for getting a method name
+ @Rule
+ public TestName name = new TestName();
+
@BeforeClass
public static void setUp() throws Exception {
FileSystem fs = FileSystem.get(conf);
@@ -384,7 +390,7 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase {
@Test
public final void testPartitionPruningWitCTAS() throws Exception {
- String tableName = "testPartitionPruningUsingDirectories".toLowerCase();
+ String tableName = name.getMethodName().toLowerCase();
String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName);
executeString(
@@ -429,4 +435,148 @@ public class TestPartitionedTableRewriter extends QueryTestCaseBase {
executeString("DROP TABLE " + canonicalTableName + " PURGE").close();
}
+
+
+ @Test
+ public void testConstantFoldingWithStringFunctions() throws Exception {
+ Expr expr = sqlParser.parse("SELECT * FROM " + MULTIPLE_PARTITION_TABLE_NAME
+ + " WHERE key1 between lower('PART123') and lower('PART125') order by n_nationkey");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(2, filteredPaths.length);
+
+ assertEquals("key3=1", filteredPaths[0].getName());
+ assertEquals("key2=supp123", filteredPaths[0].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[0].getParent().getParent().getName());
+
+ assertEquals("key3=2", filteredPaths[1].getName());
+ assertEquals("key2=supp123", filteredPaths[1].getParent().getName());
+ assertEquals("key1=part123", filteredPaths[1].getParent().getParent().getName());
+ }
+
+ @Test
+ public final void testConstantFoldingWithExpression() throws Exception {
+ String tableName = name.getMethodName().toLowerCase();
+ String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName);
+
+ executeString(
+ "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) "
+ + " as select l_orderkey, l_partkey, l_quantity from default.lineitem");
+
+ TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName);
+ assertNotNull(tableDesc);
+
+ Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName
+ + " WHERE key between round(abs(35.0 + 1.0)) and round(abs(37.0 + 1.0)) ORDER BY key");
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(2, filteredPaths.length);
+ assertEquals("key=36.0", filteredPaths[0].getName());
+ assertEquals("key=38.0", filteredPaths[1].getName());
+
+ executeString("DROP TABLE " + canonicalTableName + " PURGE").close();
+ }
+
+ @Test
+ public final void testConstantFoldingWithDateFunctions() throws Exception {
+ String tableName = name.getMethodName().toLowerCase();
+ String canonicalTableName = CatalogUtil.getCanonicalTableName("\"TestPartitionedTableRewriter\"", tableName);
+
+ String[] partitionKeys = new String[] {"20160315", "20160316", "20160317"};
+
+ executeString(
+ "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(reg_date text) "
+ + " as select l_orderkey, l_partkey" +
+ ", case " +
+ " when l_orderkey = 2 then '20160315' " +
+ " when l_orderkey = 3 then '20160316' " +
+ " else '20160317' end as reg_date" +
+ " from default.lineitem");
+
+ TableDesc tableDesc = catalog.getTableDesc(getCurrentDatabase(), tableName);
+ assertNotNull(tableDesc);
+
+ Expr expr = sqlParser.parse("SELECT * FROM " + canonicalTableName
+ + " WHERE reg_date between TO_CHAR( ADD_DAYS(TO_DATE('2016-03-14','YYYY-MM-DD'), -1) , 'YYYYMMDD') " +
+ " AND TO_CHAR( ADD_DAYS(TO_DATE('2016-03-14','YYYY-MM-DD'), 1) , 'YYYYMMDD') ORDER BY reg_date");
+
+ QueryContext defaultContext = LocalTajoTestingUtility.createDummyContext(testingCluster.getConfiguration());
+ LogicalPlan newPlan = planner.createPlan(defaultContext, expr);
+ LogicalNode plan = newPlan.getRootBlock().getRoot();
+
+ assertEquals(NodeType.ROOT, plan.getType());
+ LogicalRootNode root = (LogicalRootNode) plan;
+
+ ProjectionNode projNode = root.getChild();
+
+ assertEquals(NodeType.SORT, projNode.getChild().getType());
+ SortNode sortNode = projNode.getChild();
+
+ assertEquals(NodeType.SELECTION, sortNode.getChild().getType());
+ SelectionNode selNode = sortNode.getChild();
+ assertTrue(selNode.hasQual());
+
+ assertEquals(NodeType.SCAN, selNode.getChild().getType());
+ ScanNode scanNode = selNode.getChild();
+ scanNode.setQual(selNode.getQual());
+
+ PartitionedTableRewriter rewriter = new PartitionedTableRewriter();
+ OverridableConf conf = CommonTestingUtil.getSessionVarsForTest();
+
+ Path[] filteredPaths = rewriter.findFilteredPartitionPaths(conf, scanNode);
+ assertNotNull(filteredPaths);
+
+ assertEquals(1, filteredPaths.length);
+ assertEquals("reg_date=" + partitionKeys[0], filteredPaths[0].getName());
+
+ executeString("DROP TABLE " + canonicalTableName + " PURGE").close();
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
index 19d5d16..e5f8596 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java
@@ -192,7 +192,7 @@ public class AlgebraicUtil {
}
private final static AlgebraicOptimizer algebraicOptimizer = new AlgebraicOptimizer();
-
+
/**
* Simplify the given expr. That is, all subexprs consisting of only constants
* are calculated immediately.
http://git-wip-us.apache.org/repos/asf/tajo/blob/3f994bf2/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
index 37b77fd..23ed78e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java
@@ -78,6 +78,22 @@ public class ConstantFolding extends SimpleEvalNodeVisitor<LogicalPlanner.PlanCo
return unaryEval;
}
+ @Override
+ protected EvalNode visitBetween(LogicalPlanner.PlanContext context, BetweenPredicateEval evalNode,
+ Stack<EvalNode> stack) {
+ stack.push(evalNode);
+
+ EvalNode predicand = visit(context, evalNode.getPredicand(), stack);
+ EvalNode begin = visit(context, evalNode.getBegin(), stack);
+ EvalNode end = visit(context, evalNode.getEnd(), stack);
+
+ evalNode.setPredicand(predicand);
+ evalNode.setBegin(begin);
+ evalNode.setEnd(end);
+
+ return evalNode;
+ }
+
// exceptional func names not to use constant folding
private static final Set<String> NON_CONSTANT_FUNC_NAMES = new HashSet<String>(Arrays.asList("sleep", "random"));