You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/01/28 17:09:05 UTC
git commit: TAJO-565: FilterPushDown rewrite rule does not push
filters on partitioned scans.
Updated Branches:
refs/heads/master bbf9b7bf8 -> 428accfd7
TAJO-565: FilterPushDown rewrite rule does not push filters on partitioned scans.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/428accfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/428accfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/428accfd
Branch: refs/heads/master
Commit: 428accfd776f65d77e195d2448b1c14feb245359
Parents: bbf9b7b
Author: Hyunsik Choi <hy...@apache.org>
Authored: Wed Jan 29 00:07:02 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Wed Jan 29 00:07:02 2014 +0900
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../java/org/apache/tajo/engine/planner/LogicalPlanner.java | 2 +-
.../java/org/apache/tajo/engine/planner/PlannerUtil.java | 2 +-
.../tajo/engine/planner/rewrite/FilterPushDownRule.java | 9 +++------
.../engine/planner/rewrite/PartitionedTableRewriter.java | 4 +---
5 files changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/428accfd/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9699782..da42b26 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -231,6 +231,9 @@ Release 0.8.0 - unreleased
BUG FIXES
+ TAJO-565: FilterPushDown rewrite rule does not push filters on
+ partitioned scans. (hyunsik)
+
TAJO-560: CTAS PARTITION BY with UNION can cause invalid global plan.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/428accfd/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
index a5d9787..5ee3c69 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/LogicalPlanner.java
@@ -1076,7 +1076,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
BinaryNode setOp;
if (setOperation.getType() == OpType.Union) {
- setOp = new UnionNode(plan.newPID());
+ setOp = block.getNodeFromExpr(setOperation);
} else if (setOperation.getType() == OpType.Except) {
setOp = new ExceptNode(plan.newPID(), leftChild, rightChild);
} else if (setOperation.getType() == OpType.Intersect) {
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/428accfd/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index ef4e60a..f5d82a7 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -314,7 +314,7 @@ public class PlannerUtil {
for (Column col : columnRefs) {
if (scan.getCanonicalName().equals(col.getQualifier())) {
- Column found = node.getInSchema().getColumnByName(col.getColumnName());
+ Column found = scan.getTableSchema().getColumnByName(col.getColumnName());
if (found == null) {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/428accfd/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java
index ddcdd03..68a0e30 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java
@@ -23,10 +23,7 @@ import com.google.common.collect.Sets;
import org.apache.tajo.algebra.JoinType;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.engine.eval.*;
-import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor;
-import org.apache.tajo.engine.planner.LogicalPlan;
-import org.apache.tajo.engine.planner.PlannerUtil;
-import org.apache.tajo.engine.planner.PlanningException;
+import org.apache.tajo.engine.planner.*;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.exception.InvalidQueryException;
@@ -199,7 +196,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<Set<EvalNode>, L
List<EvalNode> matched = Lists.newArrayList();
for (EvalNode eval : cnf) {
- if (PlannerUtil.canBeEvaluated(eval, joinNode)) {
+ if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, eval, joinNode, stack.peek().getType() != NodeType.JOIN)) {
matched.add(eval);
}
}
@@ -232,7 +229,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor<Set<EvalNode>, L
List<EvalNode> matched = Lists.newArrayList();
for (EvalNode eval : cnf) {
- if (PlannerUtil.canBeEvaluated(eval, scanNode)) {
+ if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, scanNode)) {
matched.add(eval);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/428accfd/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java
index a1cbe76..c731958 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java
@@ -150,6 +150,7 @@ public class PartitionedTableRewriter implements RewriteRule {
filteredPaths = toPathArray(fs.listStatus(filteredPaths, filters[i]));
}
+ LOG.info("Filtered directory or files: " + filteredPaths.length);
return filteredPaths;
}
@@ -234,9 +235,6 @@ public class PartitionedTableRewriter implements RewriteRule {
private Path [] findFilteredPartitionPaths(ScanNode scanNode) throws IOException {
TableDesc table = scanNode.getTableDesc();
- FileSystem fs = table.getPath().getFileSystem(systemConf);
- LOG.info("Partitioned Table Dir: " + table.getPath());
- LOG.info("Summary: " + fs.getContentSummary(table.getPath()).getDirectoryCount());
PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod();
Schema paritionValuesSchema = new Schema();