You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/04/13 15:42:35 UTC
carbondata git commit: [CARBONDATA-2343][DataMap]Improper filter
resolver cause more filter scan on data that could be skipped
Repository: carbondata
Updated Branches:
refs/heads/master 5c058acc7 -> 9ee74fe07
[CARBONDATA-2343][DataMap]Improper filter resolver cause more filter scan on data that could be skipped
Currently DataMapChooser will choose and combine datamap for
expressions and it will wrap the expression in a
TrueConditionalResolverImpl. However the executor TrueFilterExecutor
will always cause scanning the blocklet which could be skipped.
This closes #2168
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ee74fe0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ee74fe0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ee74fe0
Branch: refs/heads/master
Commit: 9ee74fe0708afead309540777c315e0d13abc010
Parents: 5c058ac
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Fri Apr 13 15:14:25 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Apr 13 23:42:23 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/DataMapChooser.java | 39 ++++++++++----------
1 file changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ee74fe0/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
index ac00e71..a40644a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapChooser.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.core.scan.expression.logical.AndExpression;
import org.apache.carbondata.core.scan.expression.logical.OrExpression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.TrueConditionalResolverImpl;
/**
* This chooser does 2 jobs.
@@ -75,12 +74,12 @@ public class DataMapChooser {
// First check for FG datamaps if any exist
List<TableDataMap> allDataMapFG =
DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.FG);
- ExpressionTuple tuple = selectDataMap(expression, allDataMapFG);
+ ExpressionTuple tuple = selectDataMap(expression, allDataMapFG, resolverIntf);
if (tuple.dataMapExprWrapper == null) {
// Check for CG datamap
List<TableDataMap> allDataMapCG =
DataMapStoreManager.getInstance().getAllDataMap(carbonTable, DataMapLevel.CG);
- tuple = selectDataMap(expression, allDataMapCG);
+ tuple = selectDataMap(expression, allDataMapCG, resolverIntf);
}
if (tuple.dataMapExprWrapper != null) {
return tuple.dataMapExprWrapper;
@@ -92,13 +91,16 @@ public class DataMapChooser {
resolverIntf);
}
- private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap> allDataMap) {
+ private ExpressionTuple selectDataMap(Expression expression, List<TableDataMap> allDataMap,
+ FilterResolverIntf filterResolverIntf) {
switch (expression.getFilterExpressionType()) {
case AND:
if (expression instanceof AndExpression) {
AndExpression andExpression = (AndExpression) expression;
- ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap);
- ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap);
+ ExpressionTuple left = selectDataMap(andExpression.getLeft(), allDataMap,
+ filterResolverIntf.getLeft());
+ ExpressionTuple right = selectDataMap(andExpression.getRight(), allDataMap,
+ filterResolverIntf.getRight());
Set<ExpressionType> filterExpressionTypes = new HashSet<>();
// If both left and right has datamap then we can either merge both datamaps to single
// datamap if possible. Otherwise apply AND expression.
@@ -118,16 +120,14 @@ public class DataMapChooser {
if (dataMap != null) {
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
- tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
- new TrueConditionalResolverImpl(expression, false, false));
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf);
return tuple;
} else {
// Apply AND expression.
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
- tuple.dataMapExprWrapper =
- new AndDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper,
- new TrueConditionalResolverImpl(expression, false, false));
+ tuple.dataMapExprWrapper = new AndDataMapExprWrapper(left.dataMapExprWrapper,
+ right.dataMapExprWrapper, filterResolverIntf);
return tuple;
}
} else if (left.dataMapExprWrapper != null && right.dataMapExprWrapper == null) {
@@ -142,8 +142,10 @@ public class DataMapChooser {
case OR:
if (expression instanceof OrExpression) {
OrExpression orExpression = (OrExpression) expression;
- ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap);
- ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap);
+ ExpressionTuple left = selectDataMap(orExpression.getLeft(), allDataMap,
+ filterResolverIntf.getLeft());
+ ExpressionTuple right = selectDataMap(orExpression.getRight(), allDataMap,
+ filterResolverIntf.getRight());
Set<ExpressionType> filterExpressionTypes = new HashSet<>();
// If both left and right has datamap then we can either merge both datamaps to single
// datamap if possible. Otherwise apply OR expression.
@@ -162,15 +164,13 @@ public class DataMapChooser {
if (dataMap != null) {
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
- tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
- new TrueConditionalResolverImpl(expression, false, false));
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf);
return tuple;
} else {
ExpressionTuple tuple = new ExpressionTuple();
tuple.columnExpressions = columnExpressions;
- tuple.dataMapExprWrapper =
- new OrDataMapExprWrapper(left.dataMapExprWrapper, right.dataMapExprWrapper,
- new TrueConditionalResolverImpl(expression, false, false));
+ tuple.dataMapExprWrapper = new OrDataMapExprWrapper(left.dataMapExprWrapper,
+ right.dataMapExprWrapper, filterResolverIntf);
return tuple;
}
} else {
@@ -187,8 +187,7 @@ public class DataMapChooser {
TableDataMap dataMap =
chooseDataMap(allDataMap, tuple.columnExpressions, filterExpressionTypes);
if (dataMap != null) {
- tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap,
- new TrueConditionalResolverImpl(expression, false, false));
+ tuple.dataMapExprWrapper = new DataMapExprWrapperImpl(dataMap, filterResolverIntf);
}
return tuple;
}