You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2017/03/23 09:30:27 UTC
[2/4] incubator-carbondata git commit: [CARBONDATA-792] Range Filter
Optimization
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 69560c3..609a130 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -30,6 +30,7 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
@@ -43,15 +44,19 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
protected boolean isExpressionResolve;
protected boolean isIncludeFilter;
private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
+ private AbsoluteTableIdentifier tableIdentifier;
public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter) {
+ boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
this.exp = exp;
this.isExpressionResolve = isExpressionResolve;
this.isIncludeFilter = isIncludeFilter;
+ this.tableIdentifier = tableIdentifier;
this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
+
}
+
/**
* This API will resolve the filter expression and generates the
* dictionaries for executing/evaluating the filter expressions in the
@@ -92,7 +97,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
//the visitable instance as per its buisness logic which is different for all the
// visitors.
dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
metadata);
}
} else if (rightExp instanceof ColumnExpression) {
@@ -115,7 +120,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
} else {
dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
metadata);
}
@@ -130,10 +135,12 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
metadata.setColumnExpression(columnList.get(0));
metadata.setExpression(exp);
metadata.setIncludeFilter(isIncludeFilter);
- if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)
- || columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) || columnList.get(0)
+ .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
+ || (exp instanceof RangeExpression)) {
dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0)), metadata);
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0), exp),
+ metadata);
} else if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(
columnList.get(0).getDimension().getDataType()
@@ -182,6 +189,10 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
return dimColResolvedFilterInfo;
}
+ public AbsoluteTableIdentifier getTableIdentifier() {
+ return tableIdentifier;
+ }
+
/**
* method will calculates the start key based on the filter surrogates
*/
@@ -215,13 +226,55 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
case NOT_EQUALS:
case NOT_IN:
return FilterExecuterType.EXCLUDE;
-
+ case RANGE:
+ if (isColDictionary()) {
+ return FilterExecuterType.INCLUDE;
+ } else {
+ return FilterExecuterType.RANGE;
+ }
default:
return FilterExecuterType.INCLUDE;
}
}
+ private boolean isColDictionary() {
+ RangeExpression condExp = (RangeExpression) exp;
+ List<ColumnExpression> columnList = condExp.getColumnList();
+ if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ if (columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return false;
+ } else {
+ return true;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * This method will return the filter values which is present in the range level
+ * conditional expressions.
+ *
+ * @return
+ */
+ public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
+
+ if (null != dimColResolvedFilterInfo.getFilterValues() && !dimColResolvedFilterInfo
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ List<byte[]> noDictFilterValuesList =
+ dimColResolvedFilterInfo.getFilterValues().getNoDictionaryFilterValuesList();
+ return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
+ } else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo
+ .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(),
+ this.dimColResolvedFilterInfo.getDimension(), segmentProperties);
+ }
+ return null;
+
+ }
+
+
@Override public Expression getFilterExpression() {
// TODO Auto-generated method stub
return exp;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index acb3f8b..919faaf 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -39,7 +39,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter);
+ super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2aa4543..2a1af65 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -56,7 +56,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter);
+ super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
new file mode 100644
index 0000000..6ccae90
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/TrueConditionalResolverImpl.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo;
+
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl;
+
+/* The expression with If TRUE will be resolved setting all bits to TRUE. */
+
+public class TrueConditionalResolverImpl extends ConditionalFilterResolverImpl {
+
+ public TrueConditionalResolverImpl(Expression exp, boolean isExpressionResolve,
+ boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+
+ super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier);
+ }
+
+ @Override
+ public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
+ }
+
+ /**
+ * This method will provide the executer type to the callee inorder to identify
+ * the executer type for the filter resolution, Row level filter executer is a
+ * special executer since it get all the rows of the specified filter dimension
+ * and will be send to the spark for processing
+ */
+ @Override public FilterExecuterType getFilterExecuterType() {
+ return FilterExecuterType.TRUE;
+ }
+
+ /**
+ * Method will the read filter expression corresponding to the resolver.
+ * This method is required in row level executer inorder to evaluate the filter
+ * expression against spark, as mentioned above row level is a special type
+ * filter resolver.
+ *
+ * @return Expression
+ */
+ public Expression getFilterExpresion() {
+ return exp;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 257cca0..e8fb4c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -68,7 +68,7 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
visitableObj.setFilterValues(resolvedFilterObject);
}
- private DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
+ protected DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
ColumnExpression columnExpression, List<String> evaluateResultListFinal,
boolean isIncludeFilter, DataType dataType) {
List<Integer> surrogates = new ArrayList<Integer>(20);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
index 5fd978d..5d8cb8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
@@ -18,6 +18,8 @@ package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
public class FilterInfoTypeVisitorFactory {
@@ -29,15 +31,25 @@ public class FilterInfoTypeVisitorFactory {
* @return
*/
public static ResolvedFilterInfoVisitorIntf getResolvedFilterInfoVisitor(
- ColumnExpression columnExpression) {
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return new CustomTypeDictionaryVisitor();
- } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return new NoDictionaryTypeVisitor();
- } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return new DictionaryColumnVisitor();
+ ColumnExpression columnExpression, Expression exp) {
+ if (exp instanceof RangeExpression) {
+ if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new RangeDirectDictionaryVisitor();
+ } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new RangeNoDictionaryTypeVisitor();
+ } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new RangeDictionaryColumnVisitor();
+ }
+ }
+ else {
+ if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new CustomTypeDictionaryVisitor();
+ } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new NoDictionaryTypeVisitor();
+ } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new DictionaryColumnVisitor();
+ }
}
-
return null;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
new file mode 100644
index 0000000..a35f6ff
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
+ implements ResolvedFilterInfoVisitorIntf {
+ /**
+ * This Visitor method is used to populate the visitableObj with direct dictionary filter details
+ * where the filters values will be resolve using dictionary cache.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ * @throws IOException
+ * @throws FilterUnsupportedException
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ resolvedFilterObject = FilterUtil
+ .getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
+ metadata.getColumnExpression(), metadata.isIncludeFilter());
+
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
new file mode 100644
index 0000000..263fa9c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class RangeDirectDictionaryVisitor extends CustomTypeDictionaryVisitor
+ implements ResolvedFilterInfoVisitorIntf {
+
+ /**
+ * This Visitor method is used to populate the visitableObj with direct dictionary filter details
+ * where the filters values will be resolve using dictionary cache.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ * @throws IOException
+ * @throws FilterUnsupportedException
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>();
+ try {
+ listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
+
+ for (ExpressionResult result : listOfExpressionResults) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
+ }
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+
+ resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDimension().getDataType());
+
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
new file mode 100644
index 0000000..8d0a8b4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
+ implements ResolvedFilterInfoVisitorIntf {
+ /**
+ * Visitor Method will update the filter related details in visitableObj, For no dictionary
+ * type columns the filter members will resolved directly, no need to look up in dictionary
+ * since it will not be part of dictionary, directly the actual data can be converted as
+ * byte[] and can be set. this type of encoding is effective when the particular column
+ * is having very high cardinality.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>();
+ try {
+ // Add The Range Filter Values.
+ if (metadata.getExpression() instanceof RangeExpression) {
+ listOfExpressionResults = ((RangeExpression) metadata.getExpression()).getLiterals();
+ }
+
+ for (ExpressionResult result : listOfExpressionResults) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
+ }
+ // evaluateResultListFinal.add(metadata.getExpression().evaluate().getListAsString());
+ if (!metadata.isIncludeFilter() && !evaluateResultListFinal
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+
+ }
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index c7a6f2d..6d510f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -45,8 +45,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
@@ -390,7 +390,7 @@ public final class CarbonUtil {
}
}
- public static int getFirstIndexUsingBinarySearch(FixedLengthDimensionDataChunk dimColumnDataChunk,
+ public static int getFirstIndexUsingBinarySearch(DimensionColumnDataChunk dimColumnDataChunk,
int low, int high, byte[] compareValue, boolean matchUpLimit) {
int cmpResult = 0;
while (high >= low) {
@@ -429,7 +429,7 @@ public final class CarbonUtil {
* @return the compareValue's range index in the dimColumnDataChunk
*/
public static int[] getRangeIndexUsingBinarySearch(
- FixedLengthDimensionDataChunk dimColumnDataChunk, int low, int high, byte[] compareValue) {
+ DimensionColumnDataChunk dimColumnDataChunk, int low, int high, byte[] compareValue) {
int[] rangeIndex = new int[2];
int cmpResult = 0;
@@ -523,7 +523,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextLesserValueToTarget(int currentIndex,
- FixedLengthDimensionDataChunk dimColumnDataChunk, byte[] compareValue) {
+ DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue) {
while (currentIndex - 1 >= 0
&& dimColumnDataChunk.compareTo(currentIndex - 1, compareValue) >= 0) {
--currentIndex;
@@ -543,7 +543,7 @@ public final class CarbonUtil {
* @return index value
*/
public static int nextGreaterValueToTarget(int currentIndex,
- FixedLengthDimensionDataChunk dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
+ DimensionColumnDataChunk dimColumnDataChunk, byte[] compareValue, int numerOfRows) {
while (currentIndex + 1 < numerOfRows
&& dimColumnDataChunk.compareTo(currentIndex + 1, compareValue) <= 0) {
++currentIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
new file mode 100644
index 0000000..2d8927b
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
+import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.NotEqualsExpression;
+import org.apache.carbondata.core.scan.expression.logical.AndExpression;
+import org.apache.carbondata.core.scan.expression.logical.OrExpression;
+import org.apache.carbondata.core.scan.expression.logical.RangeExpression;
+import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
+import org.apache.carbondata.core.scan.filter.executer.RangeValueFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
+import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
+
+import mockit.Deencapsulation;
+import mockit.MockUp;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/* Test Cases for Range Filter */
+public class RangeFilterProcessorTest {
+ @BeforeClass public static void setUp() throws Exception {
+ }
+
+ public boolean checkBothTrees(Expression a, Expression b) {
+
+ if (null == a && null == b) {
+ return true;
+ }
+
+ if (null == a) return false;
+ if (null == b) return false;
+
+ if ((a != null) && (b != null)) {
+ return ((a.getClass() == b.getClass()) && (
+ (a.getChildren().size() == 0 && b.getChildren().size() == 0) || (
+ a.getChildren().size() == 2 && b.getChildren().size() == 2 && checkBothTrees(
+ a.getChildren().get(0), b.getChildren().get(0)) && checkBothTrees(
+ a.getChildren().get(1), b.getChildren().get(1)))));
+ }
+ return false;
+ }
+
+ @Test public void createFilterTree() {
+
+ Expression inputFilter;
+ boolean result = false;
+ ColumnExpression cola = new ColumnExpression("a", DataType.STRING);
+ cola.setDimension(true);
+
+ ColumnSchema empColumnSchema = new ColumnSchema();
+ empColumnSchema.setColumnName("empNameCol");
+ empColumnSchema.setColumnUniqueId("empNameCol");
+ empColumnSchema.setDimensionColumn(true);
+ empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
+ empColumnSchema.setDataType(DataType.STRING);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ cola.setDimension(empDimension);
+
+ Expression greaterThan =
+ new GreaterThanEqualToExpression(cola, new LiteralExpression("11", DataType.STRING));
+
+ ColumnExpression colb = new ColumnExpression("a", DataType.STRING);
+ colb.setDimension(true);
+ colb.setDimension(empDimension);
+ Expression lessThan =
+ new LessThanEqualToExpression(colb, new LiteralExpression("20", DataType.STRING));
+ Expression andExp = new AndExpression(greaterThan, lessThan);
+ inputFilter = andExp;
+
+ Expression output = new AndExpression(new RangeExpression(
+ new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
+ new LiteralExpression("11", DataType.STRING)),
+ new LessThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
+ new LiteralExpression("20", DataType.STRING))), new TrueExpression(null));
+ FilterOptimizer rangeFilterOptimizer =
+ new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ rangeFilterOptimizer.optimizeFilter();
+ result = checkBothTrees(inputFilter, output);
+ Assert.assertTrue(result);
+
+ }
+
+ @Test public void createFilterTree_noChange() {
+
+ // Build 2nd Tree no change a < 5 and a > 20
+
+ Expression inputFilter;
+ boolean result = false;
+ ColumnExpression cola = new ColumnExpression("a", DataType.STRING);
+ cola.setDimension(true);
+
+ ColumnSchema empColumnSchema = new ColumnSchema();
+ empColumnSchema.setColumnName("empNameCol");
+ empColumnSchema.setColumnUniqueId("empNameCol");
+ empColumnSchema.setDimensionColumn(true);
+ empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
+ empColumnSchema.setDataType(DataType.STRING);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+ cola.setDimension(empDimension);
+
+ Expression greaterThan =
+ new GreaterThanEqualToExpression(cola, new LiteralExpression("20", DataType.STRING));
+
+ ColumnExpression colb = new ColumnExpression("a", DataType.STRING);
+ colb.setDimension(true);
+ colb.setDimension(empDimension);
+ Expression lessThan =
+ new LessThanEqualToExpression(colb, new LiteralExpression("05", DataType.STRING));
+ Expression andExp = new AndExpression(greaterThan, lessThan);
+ inputFilter = andExp;
+
+ Expression output = new AndExpression(
+ new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
+ new LiteralExpression("20", DataType.STRING)),
+ new LessThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
+ new LiteralExpression("05", DataType.STRING)));
+ FilterOptimizer rangeFilterOptimizer =
+ new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ rangeFilterOptimizer.optimizeFilter();
+ result = checkBothTrees(inputFilter, output);
+ // no change
+ Assert.assertTrue(result);
+ }
+
+ @Test public void createFilterTree_flavor1() {
+
+ // Build 3rd BTree a >= '11' and a > '12' and a <= '20' and a <= '15'
+
+ Expression inputFilter;
+ boolean result = false;
+
+ ColumnSchema empColumnSchema = new ColumnSchema();
+ empColumnSchema.setColumnName("a");
+ empColumnSchema.setColumnUniqueId("a");
+ empColumnSchema.setDimensionColumn(true);
+ empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
+ empColumnSchema.setDataType(DataType.STRING);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+
+ ColumnExpression cola1 = new ColumnExpression("a", DataType.STRING);
+ cola1.setDimension(true);
+ cola1.setDimension(empDimension);
+
+ ColumnExpression cola2 = new ColumnExpression("a", DataType.STRING);
+ cola2.setDimension(true);
+ cola2.setDimension(empDimension);
+
+ ColumnExpression cola3 = new ColumnExpression("a", DataType.STRING);
+ cola3.setDimension(true);
+ cola3.setDimension(empDimension);
+
+ ColumnExpression cola4 = new ColumnExpression("a", DataType.STRING);
+ cola4.setDimension(true);
+ cola4.setDimension(empDimension);
+
+ Expression lessThan1 =
+ new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
+ Expression lessThan2 =
+ new LessThanEqualToExpression(cola2, new LiteralExpression("20", DataType.STRING));
+ Expression greaterThan1 =
+ new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
+ Expression greaterThan2 =
+ new GreaterThanEqualToExpression(cola4, new LiteralExpression("11", DataType.STRING));
+
+ Expression And1 = new AndExpression(new NotEqualsExpression(null, null), greaterThan2);
+ Expression And2 = new AndExpression(And1, greaterThan1);
+ Expression And3 = new AndExpression(And2, lessThan2);
+ Expression And4 = new AndExpression(And3, lessThan1);
+ inputFilter = And4;
+
+ // Build The output
+
+ ColumnExpression colb1 = new ColumnExpression("a", DataType.STRING);
+ cola1.setDimension(true);
+ cola1.setDimension(empDimension);
+
+ ColumnExpression colb2 = new ColumnExpression("a", DataType.STRING);
+ cola2.setDimension(true);
+ cola2.setDimension(empDimension);
+
+ Expression greaterThanb1 =
+ new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
+
+ Expression lessThanb1 =
+ new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
+
+ Expression Andb1 =
+ new AndExpression(new NotEqualsExpression(null, null), new TrueExpression(null));
+
+ Expression Andb2 = new AndExpression(Andb1, new RangeExpression(greaterThanb1, lessThanb1));
+ Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
+ Expression Andb4 = new AndExpression(Andb3, new TrueExpression(null));
+
+ Expression output = Andb4;
+
+ FilterOptimizer rangeFilterOptimizer =
+ new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ rangeFilterOptimizer.optimizeFilter();
+ result = checkBothTrees(inputFilter, output);
+ // no change
+ Assert.assertTrue(result);
+ }
+
+ @Test public void createFilterTree_flavor2() {
+
+ // Build 3rd BTree a >= '11' or a > '12' or a <= '20' or a <= '15'
+
+ Expression inputFilter;
+ boolean result = false;
+
+ ColumnSchema empColumnSchema = new ColumnSchema();
+ empColumnSchema.setColumnName("a");
+ empColumnSchema.setColumnUniqueId("a");
+ empColumnSchema.setDimensionColumn(true);
+ empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
+ empColumnSchema.setDataType(DataType.STRING);
+ CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
+
+ ColumnExpression cola1 = new ColumnExpression("a", DataType.STRING);
+ cola1.setDimension(true);
+ cola1.setDimension(empDimension);
+
+ ColumnExpression cola2 = new ColumnExpression("a", DataType.STRING);
+ cola2.setDimension(true);
+ cola2.setDimension(empDimension);
+
+ ColumnExpression cola3 = new ColumnExpression("a", DataType.STRING);
+ cola3.setDimension(true);
+ cola3.setDimension(empDimension);
+
+ ColumnExpression cola4 = new ColumnExpression("a", DataType.STRING);
+ cola4.setDimension(true);
+ cola4.setDimension(empDimension);
+
+ Expression lessThan1 =
+ new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
+ Expression lessThan2 =
+ new LessThanEqualToExpression(cola2, new LiteralExpression("20", DataType.STRING));
+ Expression greaterThan1 =
+ new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
+ Expression greaterThan2 =
+ new GreaterThanEqualToExpression(cola4, new LiteralExpression("11", DataType.STRING));
+
+ Expression Or1 = new OrExpression(new NotEqualsExpression(null, null), greaterThan2);
+ Expression Or2 = new OrExpression(Or1, greaterThan1);
+ Expression Or3 = new OrExpression(Or2, lessThan2);
+ Expression Or4 = new OrExpression(Or3, lessThan1);
+ inputFilter = Or4;
+
+ // Build The output
+
+ ColumnExpression colb1 = new ColumnExpression("a", DataType.STRING);
+ cola1.setDimension(true);
+ cola1.setDimension(empDimension);
+
+ ColumnExpression colb2 = new ColumnExpression("a", DataType.STRING);
+ cola2.setDimension(true);
+ cola2.setDimension(empDimension);
+
+ ColumnExpression colb3 = new ColumnExpression("a", DataType.STRING);
+ cola3.setDimension(true);
+ cola3.setDimension(empDimension);
+
+ ColumnExpression colb4 = new ColumnExpression("a", DataType.STRING);
+ cola4.setDimension(true);
+ cola4.setDimension(empDimension);
+
+ Expression lessThanb1 =
+ new LessThanEqualToExpression(colb1, new LiteralExpression("15", DataType.STRING));
+ Expression lessThanb2 =
+ new LessThanEqualToExpression(colb2, new LiteralExpression("20", DataType.STRING));
+ Expression greaterThanb1 =
+ new GreaterThanExpression(colb3, new LiteralExpression("12", DataType.STRING));
+ Expression greaterThanb2 =
+ new GreaterThanEqualToExpression(colb4, new LiteralExpression("11", DataType.STRING));
+
+ Expression Orb1 = new OrExpression(new NotEqualsExpression(null, null), greaterThanb2);
+ Expression Orb2 = new OrExpression(Orb1, greaterThanb1);
+ Expression Orb3 = new OrExpression(Orb2, lessThanb2);
+ Expression Orb4 = new OrExpression(Orb3, lessThanb1);
+ Expression output = Orb4;
+
+ FilterOptimizer rangeFilterOptimizer =
+ new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
+ rangeFilterOptimizer.optimizeFilter();
+ result = checkBothTrees(inputFilter, output);
+ // no change
+ Assert.assertTrue(result);
+ }
+
+ @Test public void checkIsScanRequired1() {
+ byte[] BlockMin = { 1 };
+ byte[] BlockMax = { 2 };
+ boolean result;
+
+ byte[][] filterMinMax = { { (byte) 10 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ Assert.assertFalse(result);
+ }
+
+ @Test public void checkIsScanRequired1_1() {
+ byte[] BlockMin = { 21 };
+ byte[] BlockMax = { 22 };
+ boolean result;
+
+ byte[][] filterMinMax = { { (byte) 10 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ Assert.assertFalse(result);
+ }
+
+ @Test public void checkIsScanRequired2() {
+ byte[] BlockMin = { 12 };
+ byte[] BlockMax = { 16 };
+ boolean result;
+
+ byte[][] filterMinMax = { { (byte) 10 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ Assert.assertTrue(result);
+ }
+
+ @Test public void checkIsScanRequired3() {
+ byte[] BlockMin = { 12 };
+ byte[] BlockMax = { 16 };
+ boolean result;
+ boolean rangeCovered;
+
+ byte[][] filterMinMax = { { (byte) 10 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ rangeCovered = Deencapsulation.getField(range, "isRangeFullyCoverBlock");
+ Assert.assertTrue(result);
+ Assert.assertTrue(rangeCovered);
+ }
+
+ @Test public void checkIsScanRequired4() {
+ byte[] BlockMin = { 12 };
+ byte[] BlockMax = { 30 };
+ boolean result;
+ boolean startBlockMinIsDefaultStart;
+
+ byte[][] filterMinMax = { { (byte) 10 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ startBlockMinIsDefaultStart = Deencapsulation.getField(range, "startBlockMinIsDefaultStart");
+ Assert.assertTrue(result);
+ Assert.assertTrue(startBlockMinIsDefaultStart);
+ }
+
+ @Test public void checkIsScanRequired5() {
+ byte[] BlockMin = { 10 };
+ byte[] BlockMax = { 16 };
+ boolean result;
+ boolean endBlockMaxisDefaultEnd;
+
+ byte[][] filterMinMax = { { (byte) 15 }, { (byte) 20 } };
+
+ RangeValueFilterExecuterImpl range = new MockUp<RangeValueFilterExecuterImpl>() {
+ }.getMockInstance();
+ Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
+ Deencapsulation.setField(range, "lessThanExp", true);
+ Deencapsulation.setField(range, "greaterThanExp", true);
+
+ result = range.isScanRequired(BlockMin, BlockMax, filterMinMax);
+ endBlockMaxisDefaultEnd = Deencapsulation.getField(range, "endBlockMaxisDefaultEnd");
+ Assert.assertTrue(result);
+ Assert.assertTrue(endBlockMaxisDefaultEnd);
+ }
+
+ @AfterClass public static void testcleanUp() {
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 8187089..b330f12 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -619,7 +619,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// set the filter to the query model in order to filter blocklet before scan
Expression filter = getFilterPredicates(configuration);
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
- FilterResolverIntf filterIntf = CarbonInputFormatUtil.resolveFilter(filter, identifier);
+ FilterResolverIntf filterIntf = CarbonInputFormatUtil.resolveFilter(filter, identifier);
queryModel.setFilterExpressionResolverTree(filterIntf);
// update the file level index store if there are invalid segment
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 07538e7..8270304 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -26,6 +26,9 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
+import org.apache.carbondata.core.scan.filter.intf.FilterOptimizerBasic;
+import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryDimension;
@@ -102,6 +105,13 @@ public class CarbonInputFormatUtil {
List<CarbonMeasure> measures =
carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
QueryModel.processFilterExpression(filterExpression, dimensions, measures);
+
+ if (null != filterExpression) {
+ // Optimize Filter Expression and fit RANGE filters is conditions apply.
+ FilterOptimizer rangeFilterOptimizer =
+ new RangeFilterOptmizer(new FilterOptimizerBasic(), filterExpression);
+ rangeFilterOptimizer.optimizeFilter();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/integration/spark-common-test/src/test/resources/rangedata.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/rangedata.csv b/integration/spark-common-test/src/test/resources/rangedata.csv
new file mode 100644
index 0000000..909116a
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/rangedata.csv
@@ -0,0 +1,11 @@
+empname,empno,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary
+arvind,11,SE,17-01-2007,1,developer,10,network,928478,17-02-2007,29-11-2016,96,96,5040
+krithin,12,SSE,29-05-2008,1,developer,11,protocol,928378,29-06-2008,30-12-2016,85,95,7124
+madhan,13,TPL,7/7/2009,2,tester,10,network,928478,7/8/2009,30-12-2016,88,99,9054
+anandh,14,SA,29-12-2010,3,manager,11,protocol,928278,29-01-2011,29-06-2016,77,92,11248
+ayushi,15,SSA,9/11/2011,1,developer,12,security,928375,9/12/2011,29-05-2016,99,91,13245
+pramod,16,SE,14-10-2012,1,developer,13,configManagement,928478,14-11-2012,29-12-2016,86,93,5040
+gawrav,17,PL,22-09-2013,2,tester,12,security,928778,22-10-2013,15-11-2016,78,97,9574
+sibi,18,TL,15-08-2014,2,tester,14,Learning,928176,15-09-2014,29-05-2016,84,98,7245
+shivani,19,PL,12/5/2015,1,developer,10,network,928977,12/6/2015,12/11/2016,88,91,11254
+bill,20,PM,1/12/2015,3,manager,14,Learning,928479,1/1/2016,30-11-2016,75,94,13547
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/integration/spark-common-test/src/test/resources/rangedatasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/rangedatasample.csv b/integration/spark-common-test/src/test/resources/rangedatasample.csv
new file mode 100644
index 0000000..b729470
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/rangedatasample.csv
@@ -0,0 +1,12 @@
+empno,doj,salary
+11,2016-03-14 15:00:09,5040
+12,2016-03-14 15:00:10,1040
+13,2016-03-14 15:00:11,1041
+14,2016-03-14 15:00:12,1042
+15,2016-03-14 15:00:13,1043
+16,2016-03-14 15:00:14,1044
+17,2016-03-14 15:00:15,1045
+18,2016-03-14 15:00:16,1046
+19,2016-03-14 15:00:17,1047
+20,2016-03-14 15:00:18,1048
+21,,1040
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
index 31de171..5aa30b1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
@@ -88,12 +88,15 @@ class NO_DICTIONARY_COL_TestCase extends QueryTest with BeforeAndAfterAll {
}
test("Count (*) with filter") {
+ // sql("select * from NO_DICTIONARY_CARBON_6 where empno != '16'").show()
+ sql("select * from NO_DICTIONARY_CARBON_6 where empno > '11' and empno <= '30'").show()
checkAnswer(
sql("select count(*) from NO_DICTIONARY_CARBON_6 where empno='11'"),
Seq(Row(1))
)
}
+
test("Detail Query with NO_DICTIONARY_COLUMN Compare With HIVE RESULT") {
@@ -238,6 +241,7 @@ test("filter with arithmetic expression") {
}
+
override def afterAll {
sql("drop table if exists filtertestTable")
sql("drop table if exists NO_DICTIONARY_HIVE_6")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c2e4eb24/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
new file mode 100644
index 0000000..5a72afe
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.detailquery
+
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for Range Filters.
+ */
+class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ //For the Hive table creation and data loading
+ sql("drop table if exists filtertestTable")
+ sql("drop table if exists NO_DICTIONARY_HIVE_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_2")
+ sql("drop table if exists NO_DICTIONARY_HIVE_6")
+ sql("drop table if exists dictionary_hive_6")
+ sql("drop table if exists NO_DICTIONARY_HIVE_7")
+ sql("drop table if exists NO_DICTIONARY_CARBON_6")
+ sql("drop table if exists NO_DICTIONARY_CARBON")
+ sql("drop table if exists NO_DICTIONARY_HIVE")
+ sql("drop table if exists complexcarbontable")
+
+ sql("drop table if exists DICTIONARY_CARBON_6")
+ sql("drop table if exists NO_DICTIONARY_CARBON_7")
+ sql("drop table if exists NO_DICTIONARY_CARBON_8")
+ sql("drop table if exists NO_DICTIONARY_HIVE_8")
+
+ //For Carbon cube creation.
+ sql("CREATE TABLE DICTIONARY_CARBON_6 (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE DICTIONARY_CARBON_6 " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table DICTIONARY_HIVE_6(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "DICTIONARY_HIVE_6"
+ );
+
+ sql("CREATE TABLE NO_DICTIONARY_CARBON_6 (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation','DICTIONARY_INCLUDE' = 'deptno')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_6 " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table NO_DICTIONARY_HIVE_6(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "NO_DICTIONARY_HIVE_6"
+ );
+
+ sql("CREATE TABLE NO_DICTIONARY_CARBON (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation','DICTIONARY_INCLUDE' = 'deptno')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table NO_DICTIONARY_HIVE(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "NO_DICTIONARY_HIVE"
+ );
+
+ sql("CREATE TABLE NO_DICTIONARY_CARBON_8 (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation','DICTIONARY_INCLUDE' = 'deptno')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/rangedata.csv' INTO TABLE NO_DICTIONARY_CARBON_8 " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table NO_DICTIONARY_HIVE_8(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "NO_DICTIONARY_HIVE_8"
+ );
+
+ sql("create table complexcarbontable(deviceInformationId int, channelsId string," +
+ "ROMSize string, purchasedate string, mobile struct<imei:string, imsi:string>," +
+ "MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string, " +
+ "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>," +
+ "proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId " +
+ "double,contractNumber double) " +
+ "STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId')"
+ )
+ //CarbonProperties.getInstance()
+ // .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+
+ sql(
+ s"LOAD DATA local inpath '$resourcesPath/complexdata.csv' INTO table " +
+ "complexcarbontable " +
+ "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId," +
+ "ROMSize,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber'," +
+ "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')"
+ )
+
+
+
+
+ try {
+ CarbonProperties.getInstance()
+ .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00")
+ CarbonProperties.getInstance()
+ .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY,
+ TimeStampGranularityConstants.TIME_GRAN_SEC.toString
+ )
+ CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true")
+ sql("drop table if exists directDictionaryTable")
+ sql("drop table if exists directDictionaryTable_hive")
+ sql(
+ "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, salary int) " +
+ "STORED BY 'org.apache.carbondata.format'"
+ )
+
+ sql(
+ "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp, salary int) " +
+ "row format delimited fields terminated by ','"
+ )
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
+ val csvFilePath = s"$resourcesPath/rangedatasample.csv"
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable OPTIONS" +
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive")
+
+ } catch {
+ case x: Throwable => CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ }
+ }
+
+ test("test for dictionary columns"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where workgroupcategory > 1 and workgroupcategory < 5"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where workgroupcategory > 1 and workgroupcategory < 5")
+ )
+ }
+
+ test("test for dictionary columns OR "){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where workgroupcategory > 1 or workgroupcategory < 5"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where workgroupcategory > 1 or workgroupcategory < 5")
+ )
+ }
+
+ test("test for dictionary columns OR AND"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where workgroupcategory > 1 or workgroupcategory < 5 and workgroupcategory > 3"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where workgroupcategory > 1 or workgroupcategory < 5 and workgroupcategory > 3")
+ )
+ }
+
+ test("test for dictionary columns OR AND OR"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where workgroupcategory > 1 or workgroupcategory < 5 and workgroupcategory > 3 or workgroupcategory < 10"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where workgroupcategory > 1 or workgroupcategory < 5 and workgroupcategory > 3 or workgroupcategory < 10")
+ )
+ }
+
+ test("test range filter for direct dictionary"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:17'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:18.0"))
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary and"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:16' and doj < '2016-03-14 15:00:18'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary and boundary"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:18.0' and doj < '2016-03-14 15:00:09.0'"),
+ Seq(Row(0)
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary more values after filter"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:09'"),
+ sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:09'")
+ )
+ }
+
+ test("test range filter for direct dictionary or condition"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:09' or doj > '2016-03-14 15:00:15'"),
+ sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:09' or doj > '2016-03-14 15:00:15'")
+ )
+ }
+
+ test("test range filter for direct dictionary or and condition"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:09' or doj > '2016-03-14 15:00:15' and doj < '2016-03-14 15:00:13'"),
+ sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:09' or doj > '2016-03-14 15:00:15' and doj < '2016-03-14 15:00:13'")
+ )
+ }
+
+ test("test range filter for direct dictionary with no data in csv"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > '2016-03-14 15:05:09' or doj > '2016-03-14 15:05:15' and doj < '2016-03-14 15:50:13'"),
+ sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:05:09' or doj > '2016-03-14 15:05:15' and doj < '2016-03-14 15:50:13'")
+ )
+ }
+
+ test("test range filter for measure in dictionary include"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 "),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 10 ")
+ )
+ }
+
+ test("test range filter for measure in dictionary include and condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 and deptno < 10"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 10 and deptno < 10")
+ )
+ }
+
+ test("test range filter for measure in dictionary include or condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 or deptno < 10"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 10 or deptno < 10")
+ )
+ }
+
+ test("test range filter for measure in dictionary include or and condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 or deptno < 15 and deptno >12"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 10 or deptno < 10 and deptno >12")
+ )
+ }
+
+ test("test range filter for measure in dictionary include or and condition 1"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 or deptno < 15 and deptno > 12"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 10 or deptno < 15 and deptno > 12")
+ )
+ }
+
+ test("test range filter for measure in dictionary include boundary values"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 14 or deptno < 10 and deptno > 12"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 14 or deptno < 10 and deptno > 12")
+ )
+ }
+
+ test("test range filter for measure in dictionary include same values and"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 14 and deptno < 14"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 14 and deptno < 14")
+ )
+ }
+
+ test("test range filter for measure in dictionary include same values or"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 14 or deptno < 14"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where deptno > 14 or deptno < 14")
+ )
+ }
+
+ test("test for dictionary exclude columns"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' and empno < '15'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno > '11' and empno < '15'")
+ )
+ }
+
+ test("test for dictionary exclude columns or condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' or empno > '15'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno > '11' or empno > '15'")
+ )
+ }
+
+ test("test for dictionary exclude columns or and condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' or empno > '20' and empno < '18'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno > '11' or empno > '20' and empno < '18'")
+ )
+ }
+
+ test("test for dictionary exclude columns boundary condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno < '11' or empno > '20'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno < '11' or empno > '20'")
+ )
+ }
+
+ test("test range filter for multiple columns and condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' and workgroupcategory > 2"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno > '11' and workgroupcategory > 2")
+ )
+ }
+
+ test("test range filter for multiple columns or condition"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' or workgroupcategory > 2"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_6 where empno > '11' or workgroupcategory > 2")
+ )
+ }
+
+ test("test range filter for multiplecolumns conditions"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and workgroupcategory < 3 "),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' and workgroupcategory < 3 ")
+ )
+ }
+
+ test("test range filter for more columns conditions"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and workgroupcategory < 3 and deptno > 12 and empno < '17'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' and workgroupcategory < 3 and deptno >12 and empno < '17'")
+ )
+ }
+
+ test("test range filter for multiple columns and or combination"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' or workgroupcategory < 3 and deptno > 12 and projectcode > 928478 and empno > '17' "),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' or workgroupcategory < 3 and deptno >12 and projectcode > 928478 and empno > '17'")
+ )
+ }
+
+ test("test range filter for more columns boundary conditions"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and empno < '17' and workgroupcategory < 1 and deptno > 14 "),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' and empno < '17' and workgroupcategory < 1 and deptno > 14 ")
+ )
+ }
+
+ /*
+ test("test range filter with complex datatypes 1"){
+ sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
+ checkAnswer(
+ sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
+ Seq(Row("30-12-2015"))
+ )
+ }
+ */
+
+ /*
+ test("test range filter with complex datatypes 2"){
+ sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
+ checkAnswer(
+ sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
+ Seq(Row("8ROM size",""))
+ )
+ }
+ */
+
+
+
+ override def afterAll {
+ sql("drop table if exists filtertestTable")
+ sql("drop table if exists NO_DICTIONARY_HIVE_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_2")
+ sql("drop table if exists NO_DICTIONARY_HIVE_6")
+ sql("drop table if exists dictionary_hive_6")
+ sql("drop table if exists NO_DICTIONARY_HIVE_7")
+ sql("drop table if exists NO_DICTIONARY_CARBON_6")
+ sql("drop table if exists NO_DICTIONARY_CARBON")
+ sql("drop table if exists NO_DICTIONARY_HIVE")
+ sql("drop table if exists complexcarbontable")
+
+ sql("drop table if exists DICTIONARY_CARBON_6")
+ sql("drop table if exists NO_DICTIONARY_CARBON_7")
+ sql("drop table if exists NO_DICTIONARY_CARBON_8")
+ sql("drop table if exists NO_DICTIONARY_HIVE_8")
+ //sql("drop cube NO_DICTIONARY_CARBON_1")
+ }
+}
\ No newline at end of file