You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/11/14 12:20:21 UTC
carbondata git commit: [CARBONDATA-1704] [FILTER] Filter Optimization
Repository: carbondata
Updated Branches:
refs/heads/master 796905fb1 -> d2319f067
[CARBONDATA-1704] [FILTER] Filter Optimization
If the include filter (on a dictionary column)gives more than 60% of data from a block, then Include Filter will be converted to Exclude Filter
Scenario:
column1
a
b
c
d
Query --> Select * from table_name where column1 in (a,b,c);
This will consume more than 60% of the data. In this case internally query will be optimized to exclude filter as below,
Select * from table_name where column1 not in (d);
This closes #1492
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d2319f06
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d2319f06
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d2319f06
Branch: refs/heads/master
Commit: d2319f067d436b0d8a7e099c44aab5858b859dd3
Parents: 796905f
Author: dhatchayani <dh...@huawei.com>
Authored: Mon Nov 13 14:24:55 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Nov 14 17:50:02 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 4 +
.../core/scan/filter/ColumnFilterInfo.java | 10 +
.../carbondata/core/scan/filter/FilterUtil.java | 240 ++++++++++++++-----
.../executer/DimColumnExecuterFilterInfo.java | 10 +
.../ExcludeColGroupFilterExecuterImpl.java | 2 +-
.../executer/ExcludeFilterExecuterImpl.java | 11 +-
.../executer/RestructureEvaluatorImpl.java | 7 +-
.../resolver/ConditionalFilterResolverImpl.java | 4 +-
.../RowLevelRangeFilterResolverImpl.java | 8 +-
.../visitor/CustomTypeDictionaryVisitor.java | 13 +-
.../visitor/DictionaryColumnVisitor.java | 34 +--
.../visitor/RangeDictionaryColumnVisitor.java | 11 +-
.../visitor/RangeDirectDictionaryVisitor.java | 7 +-
13 files changed, 266 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 5c04e8d..5f24381 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -23,6 +23,10 @@ import org.apache.carbondata.core.util.CarbonProperty;
public final class CarbonCommonConstants {
/**
+ * surrogate value of null
+ */
+ public static final int DICT_VALUE_NULL = 1;
+ /**
* integer size in bytes
*/
public static final int INT_SIZE_IN_BYTE = 4;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
index ce137ee..6a7fced 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
@@ -38,6 +38,16 @@ public class ColumnFilterInfo implements Serializable {
*/
private List<byte[]> noDictionaryFilterValuesList;
+ public boolean isOptimized() {
+ return isOptimized;
+ }
+
+ public void setOptimized(boolean optimized) {
+ isOptimized = optimized;
+ }
+
+ private boolean isOptimized;
+
private List<Object> measuresFilterValuesList;
public List<byte[]> getNoDictionaryFilterValuesList() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 56b27aa..f67b20c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -60,6 +60,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -102,6 +103,9 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.commons.lang.ArrayUtils;
+import org.roaringbitmap.RoaringBitmap;
+
public final class FilterUtil {
private static final LogService LOGGER =
LogServiceFactory.getLogService(FilterUtil.class.getName());
@@ -127,6 +131,14 @@ public final class FilterUtil {
if (null != filterExecuterType) {
switch (filterExecuterType) {
case INCLUDE:
+ if (null != filterExpressionResolverTree.getDimColResolvedFilterInfo()
+ && null != filterExpressionResolverTree.getDimColResolvedFilterInfo()
+ .getFilterValues() && filterExpressionResolverTree.getDimColResolvedFilterInfo()
+ .getFilterValues().isOptimized()) {
+ return getExcludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(),
+ filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties);
+ }
return getIncludeFilterExecuter(
filterExpressionResolverTree.getDimColResolvedFilterInfo(),
filterExpressionResolverTree.getMsrColResolvedFilterInfo(), segmentProperties);
@@ -507,23 +519,33 @@ public final class FilterUtil {
* @param evaluateResultList
* @param isIncludeFilter
* @return
- * @throws IOException
+ * @throws QueryExecutionException
*/
public static ColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier,
ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter,
TableProvider tableProvider)
- throws IOException {
+ throws QueryExecutionException, FilterUnsupportedException, IOException {
Dictionary forwardDictionary = null;
+ ColumnFilterInfo filterInfo = null;
+ List<Integer> surrogates =
+ new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
try {
// Reading the dictionary value from cache.
forwardDictionary =
getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension(),
tableProvider);
- return getFilterValues(columnExpression, evaluateResultList, forwardDictionary,
- isIncludeFilter);
+ sortFilterModelMembers(columnExpression, evaluateResultList);
+ getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
+ filterInfo =
+ getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ if (filterInfo.isOptimized()) {
+ return getDimColumnFilterInfoAfterApplyingCBO(columnExpression,
+ forwardDictionary, filterInfo);
+ }
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
}
+ return filterInfo;
}
/**
@@ -531,28 +553,128 @@ public final class FilterUtil {
* expression value to its respective surrogates.
*
* @param columnExpression
- * @param evaluateResultList
* @param forwardDictionary
* @param isIncludeFilter
+ * @param filterInfo
+ * @param surrogates
* @return
*/
private static ColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
- List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter) {
- sortFilterModelMembers(columnExpression, evaluateResultList);
- List<Integer> surrogates =
- new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // Reading the dictionary value from cache.
- getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
+ Dictionary forwardDictionary, boolean isIncludeFilter, ColumnFilterInfo filterInfo,
+ List<Integer> surrogates) throws QueryExecutionException {
+ // Default value has to be added
+ if (surrogates.isEmpty()) {
+ surrogates.add(0);
+ }
+ boolean isExcludeFilterNeedsToApply = false;
+ if (null == filterInfo && isIncludeFilter) {
+ isExcludeFilterNeedsToApply =
+ isExcludeFilterNeedsToApply(forwardDictionary, surrogates.size());
+ }
Collections.sort(surrogates);
ColumnFilterInfo columnFilterInfo = null;
if (surrogates.size() > 0) {
columnFilterInfo = new ColumnFilterInfo();
+ if (isExcludeFilterNeedsToApply) {
+ columnFilterInfo.setOptimized(true);
+ }
columnFilterInfo.setIncludeFilter(isIncludeFilter);
- columnFilterInfo.setFilterList(surrogates);
+ if (null != filterInfo) {
+ filterInfo.setIncludeFilter(isIncludeFilter);
+ filterInfo.setOptimized(true);
+ filterInfo.setExcludeFilterList(surrogates);
+ return filterInfo;
+ } else {
+ if (!isIncludeFilter) {
+ columnFilterInfo.setExcludeFilterList(surrogates);
+ } else {
+ columnFilterInfo.setFilterList(surrogates);
+ }
+ }
}
return columnFilterInfo;
}
+ private static boolean isExcludeFilterNeedsToApply(Dictionary forwardDictionary,
+ int size) {
+ if ((size * 100) / forwardDictionary.getDictionaryChunks().getSize() >= 60) {
+ LOGGER.info("Applying CBO to convert include filter to exclude filter.");
+ return true;
+ }
+ return false;
+ }
+
+ private static ColumnFilterInfo getDimColumnFilterInfoAfterApplyingCBO(
+ ColumnExpression columnExpression, Dictionary forwardDictionary,
+ ColumnFilterInfo filterInfo) throws FilterUnsupportedException, QueryExecutionException {
+ List<Integer> excludeMemberSurrogates =
+ prepareExcludeFilterMembers(forwardDictionary, filterInfo.getFilterList());
+ filterInfo.setExcludeFilterList(excludeMemberSurrogates);
+ return filterInfo;
+ }
+
+ private static void prepareIncludeFilterMembers(Expression expression,
+ final ColumnExpression columnExpression, boolean isIncludeFilter,
+ Dictionary forwardDictionary, List<Integer> surrogates)
+ throws FilterUnsupportedException {
+ DictionaryChunksWrapper dictionaryWrapper;
+ dictionaryWrapper = forwardDictionary.getDictionaryChunks();
+ int surrogateCount = 0;
+ while (dictionaryWrapper.hasNext()) {
+ byte[] columnVal = dictionaryWrapper.next();
+ ++surrogateCount;
+ try {
+ RowIntf row = new RowImpl();
+ String stringValue =
+ new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ stringValue = null;
+ }
+ row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt) {
+ if (rslt) {
+ if (null == stringValue) {
+ // this is for query like select name from table unknowexpr(name,1)
+ // != 'value' -> for null dictionary value
+ surrogates.add(CarbonCommonConstants.DICT_VALUE_NULL);
+ } else if (isIncludeFilter) {
+ // this is for query like select ** from * where unknwonexpr(*) == 'value'
+ surrogates.add(surrogateCount);
+ }
+ } else if (null != stringValue && !isIncludeFilter) {
+ // this is for isNot null or not in query( e.x select ** from t where name is not null
+ surrogates.add(surrogateCount);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.debug(e.getMessage());
+ }
+ }
+ }
+
+ private static List<Integer> prepareExcludeFilterMembers(
+ Dictionary forwardDictionary,List<Integer> includeSurrogates)
+ throws FilterUnsupportedException {
+ DictionaryChunksWrapper dictionaryWrapper;
+ RoaringBitmap bitMapOfSurrogates = RoaringBitmap.bitmapOf(
+ ArrayUtils.toPrimitive(includeSurrogates.toArray(new Integer[includeSurrogates.size()])));
+ dictionaryWrapper = forwardDictionary.getDictionaryChunks();
+ List<Integer> excludeFilterList = new ArrayList<Integer>(includeSurrogates.size());
+ int surrogateCount = 0;
+ while (dictionaryWrapper.hasNext()) {
+ dictionaryWrapper.next();
+ ++surrogateCount;
+ if (!bitMapOfSurrogates.contains(surrogateCount)) {
+ excludeFilterList.add(surrogateCount);
+ }
+ }
+ return excludeFilterList;
+ }
+
+
+
/**
* This API will get the Dictionary value for the respective filter member
* string.
@@ -578,45 +700,30 @@ public final class FilterUtil {
* @throws FilterUnsupportedException
* @throws IOException
*/
- public static ColumnFilterInfo getFilterListForAllValues(
- AbsoluteTableIdentifier tableIdentifier, Expression expression,
- final ColumnExpression columnExpression, boolean isIncludeFilter, TableProvider tableProvider)
- throws IOException, FilterUnsupportedException {
+ public static ColumnFilterInfo getFilterListForAllValues(AbsoluteTableIdentifier tableIdentifier,
+ Expression expression, final ColumnExpression columnExpression, boolean isIncludeFilter,
+ TableProvider tableProvider, boolean isExprEvalReqd)
+ throws FilterUnsupportedException, IOException {
Dictionary forwardDictionary = null;
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- DictionaryChunksWrapper dictionaryWrapper = null;
+ List<Integer> surrogates = new ArrayList<Integer>(20);
try {
forwardDictionary =
getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension(),
tableProvider);
- dictionaryWrapper = forwardDictionary.getDictionaryChunks();
- while (dictionaryWrapper.hasNext()) {
- byte[] columnVal = dictionaryWrapper.next();
- try {
- RowIntf row = new RowImpl();
- String stringValue =
- new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- stringValue = null;
- }
- row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
- columnExpression.getCarbonColumn().getDataType()) });
- Boolean rslt = expression.evaluate(row).getBoolean();
- if (null != rslt && rslt == isIncludeFilter) {
- if (null == stringValue) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- } else {
- evaluateResultListFinal.add(stringValue);
- }
- }
- } catch (FilterIllegalMemberException e) {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(e.getMessage());
- }
- }
+ if (isExprEvalReqd && !isIncludeFilter) {
+ surrogates.add(CarbonCommonConstants.DICT_VALUE_NULL);
+ }
+ prepareIncludeFilterMembers(expression, columnExpression, isIncludeFilter, forwardDictionary,
+ surrogates);
+ ColumnFilterInfo filterInfo =
+ getFilterValues(columnExpression, forwardDictionary, isIncludeFilter, null, surrogates);
+ if (filterInfo.isOptimized()) {
+ return getDimColumnFilterInfoAfterApplyingCBO(columnExpression, forwardDictionary,
+ filterInfo);
}
- return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
- isIncludeFilter);
+ return filterInfo;
+ } catch (QueryExecutionException e) {
+ throw new FilterUnsupportedException(e.getMessage());
} finally {
CarbonUtil.clearDictionaryCache(forwardDictionary);
}
@@ -745,7 +852,7 @@ public final class FilterUtil {
* @return
*/
public static byte[][] getKeyArray(ColumnFilterInfo columnFilterInfo,
- CarbonDimension carbonDimension, SegmentProperties segmentProperties) {
+ CarbonDimension carbonDimension, SegmentProperties segmentProperties, boolean isExclude) {
if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
return columnFilterInfo.getNoDictionaryFilterValuesList()
.toArray((new byte[columnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
@@ -759,20 +866,27 @@ public final class FilterUtil {
if (null != columnFilterInfo) {
int[] rangesForMaskedByte =
getRangesForMaskedByte(keyOrdinalOfDimensionFromCurrentBlock, blockLevelKeyGenerator);
- for (Integer surrogate : columnFilterInfo.getFilterList()) {
- try {
- if (surrogate <= dimColumnsCardinality[keyOrdinalOfDimensionFromCurrentBlock]) {
- keys[keyOrdinalOfDimensionFromCurrentBlock] = surrogate;
- filterValuesList
- .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
- } else {
- break;
+ List<Integer> listOfsurrogates = null;
+ if (!isExclude && columnFilterInfo.isIncludeFilter()) {
+ listOfsurrogates = columnFilterInfo.getFilterList();
+ } else if (isExclude || !columnFilterInfo.isIncludeFilter()) {
+ listOfsurrogates = columnFilterInfo.getExcludeFilterList();
+ }
+ if (null != listOfsurrogates) {
+ for (Integer surrogate : listOfsurrogates) {
+ try {
+ if (surrogate <= dimColumnsCardinality[keyOrdinalOfDimensionFromCurrentBlock]) {
+ keys[keyOrdinalOfDimensionFromCurrentBlock] = surrogate;
+ filterValuesList
+ .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+ } else {
+ break;
+ }
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
}
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
}
}
-
}
return filterValuesList.toArray(new byte[filterValuesList.size()][]);
@@ -1188,8 +1302,16 @@ public final class FilterUtil {
}
msrColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
} else {
- byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties);
- dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
+ if (filterValues == null) {
+ dimColumnExecuterInfo.setFilterKeys(new byte[0][]);
+ } else {
+ byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties, false);
+ if (!filterValues.isIncludeFilter() || filterValues.isOptimized()) {
+ dimColumnExecuterInfo
+ .setExcludeFilterKeys(getKeyArray(filterValues, dimension, segmentProperties, true));
+ }
+ dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
index b7202da..e907ea4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
@@ -20,6 +20,16 @@ public class DimColumnExecuterFilterInfo {
byte[][] filterKeys;
+ public byte[][] getExcludeFilterKeys() {
+ return filterKeysForExclude;
+ }
+
+ public void setExcludeFilterKeys(byte[][] filterKeysForExclude) {
+ this.filterKeysForExclude = filterKeysForExclude;
+ }
+
+ byte[][] filterKeysForExclude;
+
public void setFilterKeys(byte[][] filterKeys) {
this.filterKeys = filterKeys;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index b56c8a0..9391ebd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -61,7 +61,7 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
bitSet.flip(0, numerOfRows);
try {
KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
for (int i = 0; i < filterValues.length; i++) {
byte[] filterVal = filterValues[i];
for (int rowId = 0; rowId < numerOfRows; rowId++) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index df94d46..f680579 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -286,7 +286,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether applying filtered based on previous bitset will be optimal
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
- dimColumnExecuterInfo.getFilterKeys().length)) {
+ dimColumnExecuterInfo.getExcludeFilterKeys().length)) {
return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
numberOfRows);
} else {
@@ -313,8 +313,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
- BitSet bitSet = new BitSet(numberOfRows);
- byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
+ BitSet bitSet = new BitSet();
+ bitSet.or(prvPageBitSet);
+ byte[][] filterKeys = dimColumnExecuterInfo.getExcludeFilterKeys();
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
@@ -354,7 +355,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
int startIndex = 0;
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
for (int i = 0; i < filterValues.length; i++) {
if (startIndex >= numerOfRows) {
break;
@@ -376,7 +377,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
// binary search can only be applied if column is sorted
if (isNaturalSorted) {
int startIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
index cf3c11e..f3c066a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
@@ -76,7 +76,12 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
if (null != defaultValue) {
defaultSurrogateValueToCompare++;
}
- List<Integer> filterList = filterValues.getFilterList();
+ List<Integer> filterList = null;
+ if (filterValues.isIncludeFilter() && !filterValues.isOptimized()) {
+ filterList = filterValues.getFilterList();
+ } else {
+ filterList = filterValues.getExcludeFilterList();
+ }
for (Integer filterValue : filterList) {
if (defaultSurrogateValueToCompare == filterValue) {
isDefaultValuePresentInFilterValues = true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index d966802..bb66d18 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -171,7 +171,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
! columnList.get(0).getDimension().getDataType().isComplexType())) {
dimColResolvedFilterInfo.setFilterValues(FilterUtil
.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
- isIncludeFilter, tableProvider));
+ isIncludeFilter, tableProvider, isExpressionResolve));
dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension());
@@ -319,7 +319,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
} else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo
.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(),
- this.dimColResolvedFilterInfo.getDimension(), segmentProperties);
+ this.dimColResolvedFilterInfo.getDimension(), segmentProperties, false);
}
return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 425c725..0e7666c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -90,7 +90,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
.getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension());
if (null != dimensionFromCurrentBlock) {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
- dimensionFromCurrentBlock, segmentProperties);
+ dimensionFromCurrentBlock, segmentProperties, false);
}
}
return null;
@@ -243,7 +243,11 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- filterInfo.setFilterList(getDirectSurrogateValues(columnExpression));
+ if (!isIncludeFilter) {
+ filterInfo.setExcludeFilterList(getDirectSurrogateValues(columnExpression));
+ } else {
+ filterInfo.setFilterList(getDirectSurrogateValues(columnExpression));
+ }
} else {
filterInfo.setFilterListForNoDictionaryCols(getNoDictionaryRangeValues());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 496cf6a..882623d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -62,12 +62,13 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
evaluateResultListFinal, metadata.isIncludeFilter(),
metadata.getColumnExpression().getDimension().getDataType());
if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
- .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ .getExcludeFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
+ resolvedFilterObject.getExcludeFilterList()
.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ Collections.sort(resolvedFilterObject.getExcludeFilterList());
}
resolveDimension.setFilterValues(resolvedFilterObject);
}
@@ -88,7 +89,11 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
if (surrogates.size() > 0) {
columnFilterInfo = new ColumnFilterInfo();
columnFilterInfo.setIncludeFilter(isIncludeFilter);
- columnFilterInfo.setFilterList(surrogates);
+ if (!isIncludeFilter) {
+ columnFilterInfo.setExcludeFilterList(surrogates);
+ } else {
+ columnFilterInfo.setFilterList(surrogates);
+ }
}
return columnFilterInfo;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 47f1c8f..ee205ee 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.ColumnFilterInfo;
@@ -53,21 +54,26 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
} catch (FilterIllegalMemberException e) {
throw new FilterUnsupportedException(e);
}
- resolvedFilterObject = FilterUtil
- .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- // first check of surrogate key for null value is already added then
- // no need to add again otherwise result will be wrong in case of exclude filter
- // this is because two times it will flip the same bit
- if (!resolvedFilterObject.getFilterList()
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ try {
+
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter(), metadata.getTableProvider());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ // first check of surrogate key for null value is already added then
+ // no need to add again otherwise result will be wrong in case of exclude filter
+ // this is because two times it will flip the same bit
+ if (!resolvedFilterObject.getExcludeFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ resolvedFilterObject.getExcludeFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ }
+ Collections.sort(resolvedFilterObject.getExcludeFilterList());
}
- Collections.sort(resolvedFilterObject.getFilterList());
+ } catch (QueryExecutionException e) {
+ throw new FilterUnsupportedException(e);
}
resolveDimension.setFilterValues(resolvedFilterObject);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
index 40a5430..c90816a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDictionaryColumnVisitor.java
@@ -51,14 +51,17 @@ public class RangeDictionaryColumnVisitor extends DictionaryColumnVisitor
resolvedFilterObject = FilterUtil
.getFilterListForAllValues(metadata.getTableIdentifier(), metadata.getExpression(),
metadata.getColumnExpression(), metadata.isIncludeFilter(),
- metadata.getTableProvider());
+ metadata.getTableProvider(), true);
if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ if (!resolvedFilterObject.getExcludeFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ resolvedFilterObject.getExcludeFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
}
resolveDimension.setFilterValues(resolvedFilterObject);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d2319f06/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
index 918fd9e..f7a9f59 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeDirectDictionaryVisitor.java
@@ -71,12 +71,13 @@ public class RangeDirectDictionaryVisitor extends CustomTypeDictionaryVisitor
metadata.getColumnExpression().getDimension().getDataType());
if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
- .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ .getExcludeFilterList()
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
// Adding default surrogate key of null member inorder to not display the same while
// displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
+ resolvedFilterObject.getExcludeFilterList()
.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
+ Collections.sort(resolvedFilterObject.getExcludeFilterList());
}
resolveDimension.setFilterValues(resolvedFilterObject);
}