You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/03/23 09:30:28 UTC
[3/4] incubator-carbondata git commit: [CARBONDATA-792] Range Filter
Optimization
[CARBONDATA-792] Range Filter Optimization
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c2e4eb24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c2e4eb24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c2e4eb24
Branch: refs/heads/master
Commit: c2e4eb24ddcb94fcb5d1bdb6e09110ae89accb8e
Parents: d8d4c98
Author: sounakr <so...@gmail.com>
Authored: Wed Mar 15 21:36:16 2017 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Mar 23 14:53:31 2017 +0530
----------------------------------------------------------------------
.../core/scan/expression/BinaryExpression.java | 13 +
.../core/scan/expression/ColumnExpression.java | 3 +
.../core/scan/expression/Expression.java | 7 +
.../scan/expression/FilterModificationNode.java | 78 +++
.../core/scan/expression/LiteralExpression.java | 8 +
.../expression/RangeExpressionEvaluator.java | 480 +++++++++++++++
.../expression/conditional/ListExpression.java | 2 +
.../expression/logical/RangeExpression.java | 84 +++
.../scan/expression/logical/TrueExpression.java | 66 +++
.../scan/filter/FilterExpressionProcessor.java | 36 +-
.../carbondata/core/scan/filter/FilterUtil.java | 13 +
.../executer/RangeValueFilterExecuterImpl.java | 588 +++++++++++++++++++
.../filter/executer/TrueFilterExecutor.java | 69 +++
.../executer/ValueBasedFilterExecuterImpl.java | 42 ++
.../core/scan/filter/intf/ExpressionType.java | 3 +-
.../scan/filter/intf/FilterExecuterType.java | 2 +-
.../core/scan/filter/intf/FilterOptimizer.java | 24 +
.../scan/filter/intf/FilterOptimizerBasic.java | 26 +
.../scan/filter/intf/FilterOptimizerImpl.java | 35 ++
.../filter/optimizer/RangeFilterOptmizer.java | 41 ++
.../resolver/ConditionalFilterResolverImpl.java | 67 ++-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 2 +-
.../TrueConditionalResolverImpl.java | 61 ++
.../visitor/CustomTypeDictionaryVisitor.java | 2 +-
.../visitor/FilterInfoTypeVisitorFactory.java | 28 +-
.../visitor/RangeDictionaryColumnVisitor.java | 61 ++
.../visitor/RangeDirectDictionaryVisitor.java | 80 +++
.../visitor/RangeNoDictionaryTypeVisitor.java | 78 +++
.../apache/carbondata/core/util/CarbonUtil.java | 10 +-
.../core/util/RangeFilterProcessorTest.java | 433 ++++++++++++++
.../carbondata/hadoop/CarbonInputFormat.java | 2 +-
.../hadoop/util/CarbonInputFormatUtil.java | 10 +
.../src/test/resources/rangedata.csv | 11 +
.../src/test/resources/rangedatasample.csv | 12 +
.../HighCardinalityDataTypesTestCase.scala | 4 +
.../RangeFilterAllDataTypesTestCases.scala | 463 +++++++++++++++
.../detailquery/RangeFilterTestCase.scala | 453 ++++++++++++++
...TypeDirectDictionaryWithNoDictTestCase.scala | 5 +
...estampDataTypeDirectDictionaryTestCase.scala | 1 +
.../spark/sql/SparkUnknownExpression.scala | 4 +-
.../spark/sql/SparkUnknownExpression.scala | 6 +-
42 files changed, 3377 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
index 70260ac..0e73410 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
@@ -38,4 +38,17 @@ public abstract class BinaryExpression extends Expression {
return right;
}
+ @Override public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+ for (int i = 0; i < children.size(); i++) {
+ if (oldExpr.equals(children.get(i))) {
+ if (this.left.equals(children.get(i))) {
+ this.left = newExpr;
+ } else if (this.right.equals(children.get(i))) {
+ this.right = newExpr;
+ }
+ children.remove(i);
+ children.add(newExpr);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
index 1010093..69fa9d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -109,4 +109,7 @@ public class ColumnExpression extends LeafExpression {
this.carbonColumn = carbonColumn;
}
+ @Override public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
index f0580e7..da0df05 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
@@ -42,6 +42,13 @@ public abstract class Expression implements Serializable {
return children;
}
+ /**
+ * This method finds the Expression pointed by oldExpr and replace it with newExpr.
+ * @param oldExpr
+ * @param newExpr
+ */
+ public abstract void findAndSetChild(Expression oldExpr, Expression newExpr);
+
public abstract String getString();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
new file mode 100644
index 0000000..dad3975
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/FilterModificationNode.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+
+public class FilterModificationNode {
+ private Expression currentExp;
+ private Expression parentExp;
+ private ExpressionType expType;
+ private ExpressionResult literalValue;
+ private String columnName;
+
+ public FilterModificationNode(Expression currentNode, Expression parentNode,
+ ExpressionType expType, DataType dataType, Object literalVal, String colName) {
+ this.currentExp = currentNode;
+ this.parentExp = parentNode;
+ this.expType = expType;
+ this.columnName = colName;
+ this.literalValue = new ExpressionResult(dataType, literalVal);
+ }
+
+ public Expression getCurrentExp() {
+ return currentExp;
+ }
+
+ public void setCurrentExp(Expression currentExp) {
+ this.currentExp = currentExp;
+ }
+
+ public Expression getParentExp() {
+ return parentExp;
+ }
+
+ public void setParentExp(Expression parentExp) {
+ this.parentExp = parentExp;
+ }
+
+ public ExpressionType getExpType() {
+ return expType;
+ }
+
+ public void setExpType(ExpressionType expType) {
+ this.expType = expType;
+ }
+
+ public ExpressionResult getLiteralValue() {
+ return literalValue;
+ }
+
+ public void setLiteralValue(ExpressionResult literalValue) {
+ this.literalValue = literalValue;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public void setColumnName(String columnName) {
+ this.columnName = columnName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
index 0d12802..112bf03 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.scan.expression;
+
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
@@ -64,4 +65,11 @@ public class LiteralExpression extends LeafExpression {
return dataType;
}
+ public Object getLiteralExpValue() {
+ return value;
+ }
+
+ @Override public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
new file mode 100644
index 0000000..7cabf10
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+import org.apache.carbondata.core.scan.expression.logical.OrExpression;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
+import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+
+import static org.apache.carbondata.core.scan.filter.intf.ExpressionType.FALSE;
+import static org.apache.carbondata.core.scan.filter.intf.ExpressionType.GREATERTHAN;
+import static org.apache.carbondata.core.scan.filter.intf.ExpressionType.GREATERTHAN_EQUALTO;
+import static org.apache.carbondata.core.scan.filter.intf.ExpressionType.LESSTHAN;
+import static org.apache.carbondata.core.scan.filter.intf.ExpressionType.LESSTHAN_EQUALTO;
+
+public class RangeExpressionEvaluator {
+ private Expression expr;
+ private Expression srcNode;
+ private Expression srcParentNode;
+ private Expression tarNode;
+ private Expression tarParentNode;
+
+ public RangeExpressionEvaluator(Expression expr) {
+ this.expr = expr;
+ }
+
+ public Expression getExpr() {
+ return expr;
+ }
+
+ public void setExpr(Expression expr) {
+ this.expr = expr;
+ }
+
+ public Expression getSrcNode() {
+ return srcNode;
+ }
+
+ public void setTarNode(Expression expr) {
+ this.tarNode = expr;
+ }
+
+ public void setTarParentNode(Expression expr) {
+ this.tarParentNode = expr;
+ }
+
+ /**
+ * This method evaluates is any greaterthan or less than expression can be transformed
+ * into a single RANGE filter.
+ */
+ public void rangeExpressionEvaluatorMapBased() {
+ // The algorithm :
+ // Get all the nodes of the Expression Tree and fill it into a MAP.
+ // The Map structure will be currentNode, ColumnName, LessThanOrgreaterThan, Value, ParentNode
+ // Group the rows in MAP according to the columns and then evaluate if it can be transformed
+ // into a RANGE or not.
+ //
+ // AND AND
+ // | |
+ // / \ / \
+ // / \ / \
+ // Less Greater => TRUE Range
+ // / \ / \ / \
+ // / \ / \ / \
+ // a 10 a 5 Less greater
+ // /\ /\
+ // / \ / \
+ // a 10 a 5
+ //
+
+ Map<String, List<FilterModificationNode>> filterExpressionMap;
+ filterExpressionMap = convertFilterTreeToMap();
+ replaceWithRangeExpression(filterExpressionMap);
+ filterExpressionMap.clear();
+ }
+
+ public void replaceWithRangeExpression(
+ Map<String, List<FilterModificationNode>> filterExpressionMap) {
+
+ List<FilterModificationNode> deleteExp = new ArrayList<>();
+ for (String colName : filterExpressionMap.keySet()) {
+ // Check is there are multiple list for this Column.
+ List<FilterModificationNode> filterExp = filterExpressionMap.get(colName);
+ if (filterExp.size() > 1) {
+ // There are multiple Expression for the same column traverse and check if they can
+ // form a range.
+ FilterModificationNode startMin = null;
+ FilterModificationNode endMax = null;
+
+ for (FilterModificationNode exp : filterExp) {
+ if ((exp.getExpType() == GREATERTHAN) || (exp.getExpType() == GREATERTHAN_EQUALTO)) {
+ if ((null == endMax) || ((null != endMax) && (checkLiteralValue(exp.getCurrentExp(),
+ endMax.getCurrentExp())))) {
+ if (null == startMin) {
+ startMin = exp;
+ } else {
+ // There is already some value in startMin so check which one is greater.
+ LiteralExpression srcLiteral = getChildLiteralExpression(startMin.getCurrentExp());
+ LiteralExpression tarLiteral = getChildLiteralExpression(exp.getCurrentExp());
+
+ ExpressionResult srcExpResult = srcLiteral.evaluate(null);
+ ExpressionResult tarExpResult = tarLiteral.evaluate(null);
+ if (srcExpResult.compareTo(tarExpResult) < 0) {
+ // Before replacing the startMin add the current StartMin into deleteExp List
+ // as they will be replaced with TRUE expression after RANGE formation.
+ deleteExp.add(startMin);
+ startMin = exp;
+ }
+ }
+ }
+ }
+ if ((exp.getExpType() == LESSTHAN) || (exp.getExpType() == LESSTHAN_EQUALTO)) {
+ if ((null == startMin) || ((null != startMin) && (checkLiteralValue(exp.getCurrentExp(),
+ startMin.getCurrentExp())))) {
+ if (null == endMax) {
+ endMax = exp;
+ } else {
+ // There is already some value in endMax so check which one is less.
+ LiteralExpression srcLiteral = getChildLiteralExpression(endMax.getCurrentExp());
+ LiteralExpression tarLiteral = getChildLiteralExpression(exp.getCurrentExp());
+
+ ExpressionResult srcExpResult = srcLiteral.evaluate(null);
+ ExpressionResult tarExpResult = tarLiteral.evaluate(null);
+ if (srcExpResult.compareTo(tarExpResult) > 0) {
+ // Before replacing the endMax add the current endMax into deleteExp List
+ // as they will be replaced with TRUE expression after RANGE formation.
+ deleteExp.add(endMax);
+ endMax = exp;
+ }
+ }
+ }
+ }
+ }
+
+ if ((null != startMin) && (null != endMax)) {
+ // the node can be converted to RANGE.
+ Expression n1 = startMin.getCurrentExp();
+ Expression n2 = endMax.getCurrentExp();
+
+ RangeExpression rangeTree = new RangeExpression(n1, n2);
+ Expression srcParentNode = startMin.getParentExp();
+ Expression tarParentNode = endMax.getParentExp();
+
+ srcParentNode.findAndSetChild(startMin.getCurrentExp(), rangeTree);
+ tarParentNode.findAndSetChild(endMax.getCurrentExp(), new TrueExpression(null));
+
+ if (deleteExp.size() > 0) {
+ // There are some expression to Delete as they are Redundant after Range Formation.
+ for (FilterModificationNode trueExp : deleteExp) {
+ trueExp.getParentExp()
+ .findAndSetChild(trueExp.getCurrentExp(), new TrueExpression(null));
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ private Map<String, List<FilterModificationNode>> convertFilterTreeToMap() {
+ // Traverse the Filter Tree and add the nodes in filterExpressionMap.
+ // Only those nodes will be added which has got LessThan, LessThanEqualTo
+ // GreaterThan, GreaterThanEqualTo Expression Only.
+ Map<String, List<FilterModificationNode>> filterExpressionMap =
+ new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+ // Traverse the Tree.
+ fillExpressionMap(filterExpressionMap, null, null);
+
+ return filterExpressionMap;
+ }
+
+
+ private void fillExpressionMap(Map<String, List<FilterModificationNode>> filterExpressionMap,
+ Expression currentNode, Expression parentNode) {
+ if (null == currentNode) {
+ currentNode = this.getExpr();
+ parentNode = currentNode;
+ }
+
+ // if the parentNode is a ANDExpression and the current node is LessThan, GreaterThan
+ // then add the node into filterExpressionMap.
+ if ((parentNode instanceof AndExpression) && (isLessThanGreaterThanExp(currentNode)
+ && eligibleForRangeExpConv(currentNode))) {
+ addFilterExpressionMap(filterExpressionMap, currentNode, parentNode);
+ }
+
+ for (Expression exp : currentNode.getChildren()) {
+ if (null != exp) {
+ fillExpressionMap(filterExpressionMap, exp, currentNode);
+ if (exp instanceof OrExpression) {
+ replaceWithRangeExpression(filterExpressionMap);
+ filterExpressionMap.clear();
+ }
+ }
+ }
+ }
+
+ private void addFilterExpressionMap(Map<String, List<FilterModificationNode>> filterExpressionMap,
+ Expression currentNode, Expression parentNode) {
+ String colName = getColumnName(currentNode);
+ DataType dataType = getLiteralDataType(currentNode);
+ Object literalVal = getLiteralValue(currentNode);
+ ExpressionType expType = getExpressionType(currentNode);
+
+ FilterModificationNode filterExpression =
+ new FilterModificationNode(currentNode, parentNode, expType, dataType, literalVal, colName);
+
+ if (null == filterExpressionMap.get(colName)) {
+ filterExpressionMap.put(colName, new ArrayList<FilterModificationNode>());
+ }
+ filterExpressionMap.get(colName).add(filterExpression);
+ }
+
+ /**
+ * This method checks if the Expression is among LessThan, LessThanEqualTo,
+ * GreaterThan or GreaterThanEqualTo
+ *
+ * @param expr
+ * @return
+ */
+ private boolean isLessThanGreaterThanExp(Expression expr) {
+ if ((expr instanceof LessThanEqualToExpression) || (expr instanceof LessThanExpression)
+ || (expr instanceof GreaterThanEqualToExpression)
+ || (expr instanceof GreaterThanExpression)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * This method verifies if the Expression is qualified for Range Expression conversion.
+ *
+ * @param expChild
+ * @return
+ */
+ private boolean eligibleForRangeExpConv(Expression expChild) {
+ for (Expression exp : expChild.getChildren()) {
+ if (exp instanceof ColumnExpression) {
+ if (((ColumnExpression) exp).isDimension() == false) {
+ return false;
+ }
+ if ((((ColumnExpression) exp).getDimension().getDataType() == DataType.ARRAY) || (
+ ((ColumnExpression) exp).getDimension().getDataType() == DataType.STRUCT)) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This method returns the Column name from the ColumnExpression ExpressionType.
+ *
+ * @param andNode
+ * @return
+ */
+ private String getColumnName(Expression andNode) {
+ // returns the Column Name from Column Expression.
+ for (Expression exp : andNode.getChildren()) {
+ if (exp instanceof ColumnExpression) {
+ return ((ColumnExpression) exp).getColumnName();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method returns the Value from the Literal ExpressionType.
+ *
+ * @param exp
+ * @return
+ */
+ private Object getLiteralValue(Expression exp) {
+ for (Expression expr : exp.getChildren()) {
+ if (expr instanceof LiteralExpression) {
+ return (((LiteralExpression) expr).getLiteralExpValue());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method returns the DataType of the Literal Expression.
+ *
+ * @param exp
+ * @return
+ */
+ private DataType getLiteralDataType(Expression exp) {
+ for (Expression expr : exp.getChildren()) {
+ if (expr instanceof LiteralExpression) {
+ return (((LiteralExpression) expr).getLiteralExpDataType());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method returns the DataType of the Literal Expression.
+ *
+ * @param exp
+ * @return
+ */
+ private LiteralExpression getChildLiteralExpression(Expression exp) {
+ for (Expression expr : exp.getChildren()) {
+ if (expr instanceof LiteralExpression) {
+ return ((LiteralExpression) expr);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method returns the ExpressionType based on the Expression.
+ *
+ * @param exp
+ * @return
+ */
+ private ExpressionType getExpressionType(Expression exp) {
+ // return the expressionType. Note among the children of the
+ // andNode one should be columnExpression others should be
+ // LessThan, LessThanEqualTo, GreaterThan, GreaterThanEqualTo.
+ //
+ if (exp instanceof LessThanExpression) {
+ return LESSTHAN;
+ } else if (exp instanceof LessThanEqualToExpression) {
+ return LESSTHAN_EQUALTO;
+ } else if (exp instanceof GreaterThanExpression) {
+ return GREATERTHAN;
+ } else if (exp instanceof GreaterThanEqualToExpression) {
+ return GREATERTHAN_EQUALTO;
+ } else {
+ return FALSE;
+ }
+ }
+
+ /**
+ * This method checks if the Source Expression matches with Target Expression.
+ *
+ * @param src
+ * @param tar
+ * @return
+ */
+ private boolean matchExpType(ExpressionType src, ExpressionType tar) {
+ if ((((src == LESSTHAN) || (src == LESSTHAN_EQUALTO)) && ((tar == GREATERTHAN) || (tar
+ == GREATERTHAN_EQUALTO))) || (((src == GREATERTHAN) || (src == GREATERTHAN_EQUALTO)) && (
+ (tar == LESSTHAN) || (tar == LESSTHAN_EQUALTO)))) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * This Method Traverses the Expression Tree to find the corresponding node of the Range
+ * Expression. If one node of Range Expression is LessThan then a corresponding GreaterThan
+ * will be choosen or vice versa.
+ *
+ * @param currentNode
+ * @param parentNode
+ * @return
+ */
+ private Expression traverseTree(Expression currentNode, Expression parentNode) {
+ Expression result = null;
+
+ if (null == parentNode) {
+ currentNode = this.getExpr();
+ parentNode = currentNode;
+ }
+
+ if (!this.getSrcNode().equals(currentNode) && isLessThanGreaterThanExp(currentNode)) {
+ String srcColumnName = getColumnName(this.getSrcNode());
+ String tarColumnName = getColumnName(currentNode);
+ ExpressionType srcExpType = getExpressionType(this.getSrcNode());
+ ExpressionType tarExpType = getExpressionType(currentNode);
+
+ if ((null != srcColumnName) && (null != tarColumnName) && (srcColumnName == tarColumnName)
+ && (srcExpType != ExpressionType.FALSE) && (tarExpType != ExpressionType.FALSE) && (
+ (matchExpType(srcExpType, tarExpType)) && checkLiteralValue(this.getSrcNode(),
+ currentNode))) {
+ this.setTarNode(currentNode);
+ this.setTarParentNode(parentNode);
+ return parentNode;
+ }
+ }
+
+ for (Expression exp : currentNode.getChildren()) {
+ if (null != exp && !(exp instanceof RangeExpression)) {
+ result = traverseTree(exp, currentNode);
+ if (null != result) {
+ return result;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This method will check if the literal values attached to GreaterThan of GreaterThanEqualTo
+ * is less or equal to LessThan and LessThanEqualTo literal.
+ *
+ * @param src
+ * @param tar
+ * @return
+ */
+ private boolean checkLiteralValue(Expression src, Expression tar) {
+ ExpressionType srcExpressionType = getExpressionType(src);
+ ExpressionType tarExpressionType = getExpressionType(tar);
+ LiteralExpression srcLiteral = getChildLiteralExpression(src);
+ LiteralExpression tarLiteral = getChildLiteralExpression(tar);
+
+ ExpressionResult srcExpResult = srcLiteral.evaluate(null);
+ ExpressionResult tarExpResult = tarLiteral.evaluate(null);
+
+ switch (srcExpressionType) {
+ case LESSTHAN:
+ case LESSTHAN_EQUALTO:
+ switch (tarExpressionType) {
+ case GREATERTHAN:
+ if (srcExpResult.compareTo(tarExpResult) > 0) {
+ return true;
+ }
+ break;
+ case GREATERTHAN_EQUALTO:
+ if (srcExpResult.compareTo(tarExpResult) >= 0) {
+ return true;
+ }
+ break;
+ }
+ break;
+ case GREATERTHAN:
+ case GREATERTHAN_EQUALTO:
+ switch (tarExpressionType) {
+ case LESSTHAN:
+ if (srcExpResult.compareTo(tarExpResult) < 0) {
+ return true;
+ }
+ break;
+ case LESSTHAN_EQUALTO:
+ if (srcExpResult.compareTo(tarExpResult) <= 0) {
+ return true;
+ }
+ break;
+ }
+ break;
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
index 98258ab..32ca11c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
@@ -57,4 +57,6 @@ public class ListExpression extends Expression {
return null;
}
+ @Override public void findAndSetChild(Expression oldExpr, Expression newExpr) {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
new file mode 100644
index 0000000..bce9320
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/RangeExpression.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class RangeExpression extends BinaryConditionalExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ public RangeExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult resultLeft = left.evaluate(value);
+ ExpressionResult resultRight = right.evaluate(value);
+ switch (resultLeft.getDataType()) {
+ case BOOLEAN:
+ resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() && resultRight.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying RANGE Expression Filter");
+ }
+ return resultLeft;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.RANGE;
+ }
+
+ @Override public String getString() {
+ return null;
+ }
+
+ @Override public List<ExpressionResult> getLiterals() {
+ List<ExpressionResult> listOfExp =
+ new ArrayList<ExpressionResult>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getLiteralsResult(this, listOfExp);
+ Collections.sort(listOfExp);
+ return listOfExp;
+ }
+
+ private void getLiteralsResult(Expression expression, List<ExpressionResult> listOfExp) {
+ for (Expression child : expression.getChildren()) {
+ if (child instanceof LiteralExpression) {
+ ExpressionResult colExp = ((LiteralExpression) child).getExpressionResult();
+ listOfExp.add(colExp);
+ } else {
+ getLiteralsResult(child, listOfExp);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
new file mode 100644
index 0000000..7897f4a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/TrueExpression.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+/**
+ * This class will form an expression whose evaluation will be always true.
+ */
+public class TrueExpression extends BinaryConditionalExpression {
+
+
+ private static final long serialVersionUID = -8390184061336799370L;
+
+ public TrueExpression(Expression child1) {
+ super(child1, new LiteralExpression(null,null));
+ }
+
+ /**
+ * This method will always return false, mainly used in the filter expressions
+ * which are illogical.
+ * eg: columnName NOT IN('Java',NULL)
+ * @param value
+ * @return
+ * @throws FilterUnsupportedException
+ * @throws FilterIllegalMemberException
+ */
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return new ExpressionResult(DataType.BOOLEAN,true);
+ }
+
+ /**
+ * This method will return the expression types
+ * @return
+ */
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.TRUE;
+ }
+ @Override public String getString() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 9eb1be6..264c0ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -44,6 +44,7 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.LogicalFilterResolverImpl;
import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
import org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.TrueConditionalResolverImpl;
public class FilterExpressionProcessor implements FilterProcessor {
@@ -211,6 +212,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
currentExpression);
+ case RANGE:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.RANGE, true,
+ expressionTree, tableIdentifier, expressionTree);
case EQUALS:
case IN:
return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS,
@@ -230,6 +234,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
case FALSE:
return getFilterResolverBasedOnExpressionType(ExpressionType.FALSE, false,
expressionTree, tableIdentifier, expressionTree);
+ case TRUE:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.TRUE, false,
+ expressionTree, tableIdentifier, expressionTree);
default:
return getFilterResolverBasedOnExpressionType(ExpressionType.UNKNOWN, false, expressionTree,
tableIdentifier, expressionTree);
@@ -248,6 +255,8 @@ public class FilterExpressionProcessor implements FilterProcessor {
switch (filterExpressionType) {
case FALSE:
return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ case TRUE:
+ return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
case EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
if (currentCondExpression.isSingleDimension()
@@ -277,10 +286,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
}
break;
+ case RANGE:
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
case NOT_EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
if (currentCondExpression.isSingleDimension()
@@ -308,24 +321,27 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
}
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
}
break;
+
default:
if (expression instanceof ConditionalExpression) {
condExpression = (ConditionalExpression) expression;
if (condExpression.isSingleDimension()
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.ARRAY
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.STRUCT) {
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
condExpression = (ConditionalExpression) expression;
if (condExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return new ConditionalFilterResolverImpl(expression, true, true);
+ .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new ConditionalFilterResolverImpl(expression, true, true, tableIdentifier);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index eeb7be7..770aa7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -70,14 +70,17 @@ import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RangeValueFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RestructureExcludeFilterExecutorImpl;
import org.apache.carbondata.core.scan.filter.executer.RestructureIncludeFilterExecutorImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
+import org.apache.carbondata.core.scan.filter.executer.TrueFilterExecutor;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -136,6 +139,16 @@ public final class FilterUtil {
return RowLevelRangeTypeExecuterFacory
.getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree,
segmentProperties);
+ case RANGE:
+ return new RangeValueFilterExecuterImpl(
+ ((ConditionalFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColResolvedFilterInfo(),
+ null, filterExpressionResolverTree.getFilterExpression(),
+ ((ConditionalFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ ((ConditionalFilterResolverImpl) filterExpressionResolverTree)
+ .getFilterRangeValues(segmentProperties), segmentProperties);
+ case TRUE:
+ return new TrueFilterExecutor();
case ROWLEVEL:
default:
return new RowLevelFilterExecuterImpl(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
new file mode 100644
index 0000000..9dc6a99
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * This Class get a Min And Max value of a RANGE expression. Finds out if scan is required
+ * for this Range. Also search the data block and set the required bitsets which falls within
+ * the Range of the RANGE Expression.
+ */
+public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
+
+ private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ private MeasureColumnResolvedFilterInfo msrColEvalutorInfo;
+ private AbsoluteTableIdentifier tableIdentifier;
+ private Expression exp;
+ private byte[][] filterRangesValues;
+ private SegmentProperties segmentProperties;
+ private boolean isDefaultValuePresentInFilter;
+ /**
+ * it has index at which given dimension is stored in file
+ */
+ protected int dimensionBlocksIndex;
+
+ /**
+ * flag to check whether the filter dimension is present in current block list of dimensions.
+ * Applicable for restructure scenarios
+ */
+ protected boolean isDimensionPresentInCurrentBlock;
+ boolean lessThanExp;
+ boolean lessThanEqualExp;
+ boolean greaterThanExp;
+ boolean greaterThanEqualExp;
+ boolean startBlockMinIsDefaultStart;
+ boolean endBlockMaxisDefaultEnd;
+ boolean isRangeFullyCoverBlock;
+
+ public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
+ MeasureColumnResolvedFilterInfo msrColEvaluatorInfo, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
+ SegmentProperties segmentProperties) {
+
+ this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+ this.msrColEvalutorInfo = msrColEvaluatorInfo;
+ this.exp = exp;
+ this.segmentProperties = segmentProperties;
+ this.tableIdentifier = tableIdentifier;
+ this.filterRangesValues = filterRangeValues;
+ this.lessThanExp = isLessThan();
+ this.lessThanEqualExp = isLessThanEqualTo();
+ this.greaterThanExp = isGreaterThan();
+ this.greaterThanEqualExp = isGreaterThanEqualTo();
+ startBlockMinIsDefaultStart = false;
+ endBlockMaxisDefaultEnd = false;
+ isRangeFullyCoverBlock = false;
+ initDimensionBlockIndexes();
+ ifDefaultValueMatchesFilter();
+
+ }
+
+ /**
+ * This method will initialize the dimension info for the current block to be
+ * used for filtering the data
+ */
+ private void initDimensionBlockIndexes() {
+ // find the dimension in the current block dimensions list
+ CarbonDimension dimensionFromCurrentBlock =
+ segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension());
+ if (null != dimensionFromCurrentBlock) {
+ dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+ this.dimensionBlocksIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimensionFromCurrentBlock.getOrdinal());
+ isDimensionPresentInCurrentBlock = true;
+ }
+ }
+
+ /**
+ * This method will check whether default value is present in the given filter values
+ */
+ private void ifDefaultValueMatchesFilter() {
+ isDefaultValuePresentInFilter = false;
+ if (this.isDimensionPresentInCurrentBlock) {
+ CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension();
+ byte[] defaultValue = dimension.getDefaultValue();
+ if (null != defaultValue) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterRangesValues[0]);
+ int minCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangesValues[1], defaultValue);
+
+ if (((greaterThanExp && maxCompare > 0) || (greaterThanEqualExp && maxCompare >= 0))
+ && ((lessThanExp && minCompare > 0) || (lessThanEqualExp && minCompare >= 0))) {
+ isDefaultValuePresentInFilter = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * Method to apply the filter.
+ * @param blockChunkHolder
+ * @return
+ * @throws FilterUnsupportedException
+ * @throws IOException
+ */
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ return applyNoAndDirectFilter(blockChunkHolder);
+ }
+
+ /**
+ * Method to find presence of LessThan Expression.
+ * @return
+ */
+ private boolean isLessThan() {
+ for (Expression result : this.exp.getChildren()) {
+ if (result instanceof LessThanExpression) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Method to find presence of LessThanEqualTo Expression.
+ * @return
+ */
+ private boolean isLessThanEqualTo() {
+ for (Expression result : this.exp.getChildren()) {
+ if (result instanceof LessThanEqualToExpression) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Method to find presence of GreaterThan Expression.
+ * @return
+ */
+ private boolean isGreaterThan() {
+ for (Expression result : this.exp.getChildren()) {
+ if (result instanceof GreaterThanExpression) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Method to find presence of GreaterThanEqual Expression.
+ * @return
+ */
+ private boolean isGreaterThanEqualTo() {
+ for (Expression result : this.exp.getChildren()) {
+ if (result instanceof GreaterThanEqualToExpression) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+
+ /**
+ * Method to identify if scanning of Data Block required or not by comparing the Block Min and Max
+ * values and comparing them with filter min and max value.
+ * @param blockMinValue
+ * @param blockMaxValue
+ * @param filterValues
+ * @return
+ */
+ public boolean isScanRequired(byte[] blockMinValue, byte[] blockMaxValue, byte[][] filterValues) {
+ boolean isScanRequired = true;
+ isRangeFullyCoverBlock = false;
+ startBlockMinIsDefaultStart = false;
+ endBlockMaxisDefaultEnd = false;
+
+ /*
+ For Undertsanding the below logic kept the value evaluation code intact.
+ int filterMinlessThanBlockMin =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]);
+ int filterMaxLessThanBlockMin =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]);
+
+ int filterMinGreaterThanBlockMax =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue);
+ int filterMaxGreaterThanBlockMax =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue);
+ */
+
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ // There are 4 cases which has been explicitly handled.
+ // case A: Outside Boundary > No Scan required
+ // Block Min <----------> Block Max
+ // Filter Min <----------> Filter Max
+ // case B: Filter Overlaps Upper Part Block then Block Start Scan Becomes StartIndex of Block.
+ // Block Min <-----------------> Block Max
+ // Filter Min <-------------------------> Filter Min
+ // Case C: Filter Overlaps Lower Part of Block then Block End Scan becomes Number Of Rows - 1
+ // Block Min <-------------------------> Block Max
+ // Filter Min <--------------------------> Filter Max
+ // Case D: Filter Values Completely overlaps Block Min and Max then all bits are set.
+ // Block Min <-----------------------> Block Max
+ // Filter Min <-----------------------------------------------> Filter Max
+
+ if (isDimensionPresentInCurrentBlock == true) {
+ if (((lessThanExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || (
+ (lessThanEqualExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || (
+ (greaterThanExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || (
+ (greaterThanEqualExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) {
+ // completely out of block boundary
+ isScanRequired = false;
+ } else {
+ if (((greaterThanExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || (
+ (greaterThanEqualExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) {
+ startBlockMinIsDefaultStart = true;
+ }
+
+ if (((lessThanExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || (
+ (lessThanEqualExp == true) && (
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) {
+ endBlockMaxisDefaultEnd = true;
+ }
+
+ if (startBlockMinIsDefaultStart == true && endBlockMaxisDefaultEnd == true) {
+ isRangeFullyCoverBlock = true;
+ }
+ }
+ return isScanRequired;
+ } else {
+ return isDefaultValuePresentInFilter;
+ }
+ }
+
+ /**
+ * Method checks is the scan lies within the range values or not.
+ * @param blockMaxValue
+ * @param blockMinValue
+ * @return
+ */
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = this.filterRangesValues;
+ int columnIndex = this.dimColEvaluatorInfo.getColumnIndex();
+ boolean isScanRequired =
+ isScanRequired(blockMinValue[columnIndex], blockMaxValue[columnIndex], filterValues);
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+ /**
+ * Method to apply the Range Filter.
+ * @param blockChunkHolder
+ * @return
+ * @throws FilterUnsupportedException
+ * @throws IOException
+ */
+ public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+
+ // In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be
+ // false, in that scenario the default values of the column should be shown.
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock) {
+ int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ return FilterUtil
+ .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ numberOfRows, true);
+ }
+
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i],
+ this.filterRangesValues)) {
+ if (isRangeFullyCoverBlock == true) {
+ // Set all the bits in this case as filter Min Max values cover the whole block.
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ }
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all members
+ * will be considered for applying range filters. this method will be called if the
+ * column is not supported by default so column index mapping will be present for
+ * accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int startIndex = 0;
+ int startMin = 0;
+ int endMax = 0;
+ byte[][] filterValues = this.filterRangesValues;
+
+ // For Range expression we expect two values. The First is the Min Value and Second is the
+ // Max value.
+ // Get the Min Value
+ if (startBlockMinIsDefaultStart == false) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[0], greaterThanExp);
+ if (greaterThanExp == true && start >= 0) {
+ start = CarbonUtil
+ .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ numerOfRows);
+ }
+
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if ((greaterThanExp == true) && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
+ .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) {
+ start = start + 1;
+ } else if ((greaterThanEqualExp == true) && (ByteUtil.compare(filterValues[0],
+ dimensionColumnDataChunk
+ .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) >= 0) {
+ start = start + 1;
+ }
+ }
+
+ startMin = start;
+ } else {
+ startMin = startIndex;
+ }
+
+ // Get the Max value
+ if (endBlockMaxisDefaultEnd == false) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[1], lessThanEqualExp);
+ if (lessThanExp == true && start >= 0) {
+ start =
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ }
+
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if ((lessThanExp == true) && (ByteUtil.compare(filterValues[1],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ < 0)) {
+ start = start - 1;
+ } else if ((lessThanEqualExp == true) && (ByteUtil.compare(filterValues[1],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ <= 0)) {
+ start = start - 1;
+ }
+ }
+ endMax = start;
+ } else {
+ endMax = numerOfRows - 1;
+ }
+
+ for (int j = startMin; j <= endMax; j++) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ }
+
+ // Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
+ // matching row.
+ if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ }
+ return bitSet;
+ }
+
+ private void updateForNoDictionaryColumn(int start, int end, DimensionColumnDataChunk dataChunk,
+ BitSet bitset) {
+ for (int j = start; j <= end; j++) {
+ if (dataChunk.compareTo(j, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) == 0) {
+ bitset.flip(j);
+ }
+ }
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all
+ * members will be considered for applying range filters. this method will
+ * be called if the column is sorted default so column index
+ * mapping will be present for accesaing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ // if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ int start = 0;
+ int startMin = 0;
+ int endMax = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangesValues;
+ // For Range expression we expect two values. The First is the Min Value and Second is the
+ // Max value.
+ if (startBlockMinIsDefaultStart == false) {
+
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk,
+ startIndex, numerOfRows - 1, filterValues[0], greaterThanExp);
+
+ if (greaterThanExp == true && start >= 0) {
+ start = CarbonUtil
+ .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ numerOfRows);
+ }
+
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if ((greaterThanExp == true)
+ && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ > 0) {
+ start = start + 1;
+ } else if ((greaterThanEqualExp == true)
+ && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ >= 0) {
+ start = start + 1;
+ }
+ }
+ startMin = start;
+ } else {
+ startMin = startIndex;
+ }
+
+ if (endBlockMaxisDefaultEnd == false) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[1], lessThanEqualExp);
+
+ if (lessThanExp == true && start >= 0) {
+ start =
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ }
+
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if ((lessThanExp == true) && (
+ ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start)) < 0)) {
+ start = start - 1;
+ } else if ((lessThanEqualExp == true) && (
+ ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start)) <= 0)) {
+ start = start - 1;
+ }
+ }
+ endMax = start;
+ } else {
+ endMax = numerOfRows - 1;
+ }
+
+ for (int j = startMin; j <= endMax; j++) {
+ bitSet.set(j);
+ }
+
+ // Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
+ // matching row.
+ if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * Method to read the blocks.
+ * @param blockChunkHolder
+ * @throws IOException
+ */
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ if (isDimensionPresentInCurrentBlock == true) {
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
new file mode 100644
index 0000000..ac8c423
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class TrueFilterExecutor implements FilterExecuter {
+
+ /**
+ * API will apply filter based on resolver instance
+ *
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
+ BitSetGroup group = new BitSetGroup(numberOfPages);
+ for (int i = 0; i < numberOfPages; i++) {
+ BitSet set = new BitSet();
+ set.flip(0, blockChunkHolder.getDataBlock().nodeSize());
+ group.setBitSet(set, i);
+ }
+ return group;
+ }
+
+ /**
+ * API will verify whether the block can be shortlisted based on block
+ * max and min key.
+ *
+ * @param blockMaxValue, maximum value of the
+ * @param blockMinValue
+ * @return BitSet
+ */
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.flip(0, 1);
+ return bitSet;
+ }
+
+ /**
+ * It just reads necessary block for filter executor, it does not uncompress the data.
+ *
+ * @param blockChunkHolder
+ */
+ public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ // do nothing
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
new file mode 100644
index 0000000..003d15e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class ValueBasedFilterExecuterImpl implements FilterExecuter {
+
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+
+ BitSetGroup bitSetGroup = new BitSetGroup(0);
+ return bitSetGroup;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ return bitSet;
+ }
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
index 67a05e6..a3f9199 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
@@ -38,6 +38,7 @@ public enum ExpressionType {
UNKNOWN,
LITERAL,
RANGE,
- FALSE
+ FALSE,
+ TRUE
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
index a894ab2..42defc6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
@@ -21,6 +21,6 @@ import java.io.Serializable;
public enum FilterExecuterType implements Serializable {
INCLUDE, EXCLUDE, OR, AND, RESTRUCTURE, ROWLEVEL, RANGE, ROWLEVEL_GREATERTHAN,
- ROWLEVEL_GREATERTHAN_EQUALTO, ROWLEVEL_LESSTHAN_EQUALTO, ROWLEVEL_LESSTHAN
+ ROWLEVEL_GREATERTHAN_EQUALTO, ROWLEVEL_LESSTHAN_EQUALTO, ROWLEVEL_LESSTHAN, TRUE
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizer.java
new file mode 100644
index 0000000..9fa3d69
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizer.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+
+public interface FilterOptimizer {
+ Expression optimizeFilter();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
new file mode 100644
index 0000000..b3ed34e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerBasic.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+
+public class FilterOptimizerBasic implements FilterOptimizer {
+ @Override public Expression optimizeFilter() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
new file mode 100644
index 0000000..145fc09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+
+public class FilterOptimizerImpl implements FilterOptimizer {
+
+ protected FilterOptimizer filterOptimizer;
+ protected Expression filterExpression;
+
+ public FilterOptimizerImpl(FilterOptimizer filterOptimizer, Expression filterExpression) {
+ this.filterOptimizer = filterOptimizer;
+ this.filterExpression = filterExpression;
+ }
+
+ @Override public Expression optimizeFilter() {
+ return this.optimizeFilter();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
new file mode 100644
index 0000000..6c61b62
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/optimizer/RangeFilterOptmizer.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.optimizer;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.RangeExpressionEvaluator;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerImpl;
+
+public class RangeFilterOptmizer extends FilterOptimizerImpl {
+
+ RangeExpressionEvaluator rangeExpEvaluator;
+
+ public RangeFilterOptmizer(FilterOptimizer filterOptimizer, Expression filterExpression) {
+ super(filterOptimizer, filterExpression);
+ this.rangeExpEvaluator = new RangeExpressionEvaluator(filterExpression);
+
+ }
+
+ @Override public Expression optimizeFilter() {
+ // Check if Range Filter can be applied.
+ // rangeExpEvaluator.rangeExpressionEvaluatorGraphBased(null, null);
+ rangeExpEvaluator.rangeExpressionEvaluatorMapBased();
+ return this.rangeExpEvaluator.getExpr();
+ }
+}