You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/10/30 09:21:53 UTC
[03/35] carbondata git commit: [CARBONDATA-1538] Added code to
support BITSET PIPE LINE in filter
[CARBONDATA-1538] Added code to support BITSET PIPE LINE in filter
This closes #1403
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75e0bd41
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75e0bd41
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75e0bd41
Branch: refs/heads/streaming_ingest
Commit: 75e0bd4199e4c8123bd4dde6dbae208ab5f8311b
Parents: ac6c1d2
Author: kumarvishal <ku...@gmail.com>
Authored: Sun Oct 8 16:48:59 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Oct 12 13:34:21 2017 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 9 ++
.../chunk/DimensionColumnDataChunk.java | 7 +
.../chunk/impl/AbstractDimensionDataChunk.java | 7 +
.../impl/ColumnGroupDimensionDataChunk.java | 8 +
.../chunk/store/ColumnPageWrapper.java | 4 +
.../chunk/store/DimensionDataChunkStore.java | 7 +
.../SafeAbsractDimensionDataChunkStore.java | 10 ++
.../UnsafeAbstractDimensionDataChunkStore.java | 12 ++
.../filter/executer/AndFilterExecuterImpl.java | 8 +-
.../executer/ExcludeFilterExecuterImpl.java | 152 +++++++++++++++++--
.../scan/filter/executer/FilterExecuter.java | 2 +-
.../IncludeColGroupFilterExecuterImpl.java | 4 +-
.../executer/IncludeFilterExecuterImpl.java | 151 ++++++++++++++++--
.../filter/executer/OrFilterExecuterImpl.java | 9 +-
.../executer/RangeValueFilterExecuterImpl.java | 2 +-
.../RestructureExcludeFilterExecutorImpl.java | 4 +-
.../RestructureIncludeFilterExecutorImpl.java | 4 +-
.../executer/RowLevelFilterExecuterImpl.java | 50 ++++--
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 3 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 3 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 3 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 3 +-
.../filter/executer/TrueFilterExecutor.java | 2 +-
.../executer/ValueBasedFilterExecuterImpl.java | 3 +-
.../core/scan/processor/BlocksChunkHolder.java | 11 ++
.../core/scan/scanner/impl/FilterScanner.java | 11 +-
.../apache/carbondata/core/util/CarbonUtil.java | 46 ++++++
27 files changed, 482 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c1a3ec7..51b92cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1367,6 +1367,15 @@ public final class CarbonCommonConstants {
public static final String CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT = "true";
+ /**
+ * this will be used to pass bitset value in filter to another filter for
+ * faster execution of filter query
+ */
+ @CarbonProperty
+ public static final String BITSET_PIPE_LINE = "carbon.use.bitset.pipe.line";
+
+ public static final String BITSET_PIPE_LINE_DEFAULT = "true";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
index db54885..3791314 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
@@ -78,6 +78,13 @@ public interface DimensionColumnDataChunk {
int getInvertedIndex(int index);
/**
+ *
+ * @param invertedIndex
+ * @return index reverse index
+ */
+ int getInvertedReverseIndex(int invertedIndex);
+
+ /**
* @return whether column is dictionary column or not
*/
boolean isNoDicitionaryColumn();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
index ac2b2cb..eac062f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
@@ -54,6 +54,13 @@ public abstract class AbstractDimensionDataChunk implements DimensionColumnDataC
}
/**
+ * @param invertedIndex
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int invertedIndex) {
+ return dataChunkStore.getInvertedReverseIndex(invertedIndex);
+ }
+ /**
* @return length of each column
*/
@Override public int getColumnValueSize() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
index 8234f67..b76ae53 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -103,6 +103,14 @@ public class ColumnGroupDimensionDataChunk extends AbstractDimensionDataChunk {
}
/**
+ * @param invertedIndex
+ * @return inverted index reverse
+ */
+ @Override public int getInvertedReverseIndex(int invertedIndex) {
+ throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+ }
+
+ /**
* @return whether columns where explictly sorted or not
*/
@Override public boolean isExplicitSorted() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index 21e130b..fbdb499 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -66,6 +66,10 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
throw new UnsupportedOperationException("internal error");
}
+ @Override public int getInvertedReverseIndex(int invertedIndex) {
+ throw new UnsupportedOperationException("internal error");
+ }
+
@Override
public boolean isNoDicitionaryColumn() {
return true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
index 5301945..5072c75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
@@ -68,6 +68,13 @@ public interface DimensionDataChunkStore {
int getInvertedIndex(int rowId);
/**
+ * Below method will be used to get the reverse Inverted Index
+ * @param invertedIndex
+ * @return reverse Inverted Index
+ */
+ int getInvertedReverseIndex(int invertedIndex);
+
+ /**
* Below method will be used to get the surrogate key of the
* based on the row id passed
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
index cb4c1a7..f7189e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
@@ -86,6 +86,16 @@ public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDat
}
/**
+ * Below method will be used to get the inverted index reverse
+ *
+ * @param rowId row id
+ * @return inverted index based on row id passed
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ return invertedIndexReverse[rowId];
+ }
+
+ /**
* Below method will be used to get the surrogate key of the based on the row
* id passed
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 22c2e16..940ca1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -142,6 +142,18 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
}
/**
+ * Below method will be used to get the reverse inverted index
+ *
+ * @param rowId row id
+ * @return inverted index based on row id passed
+ */
+ @Override public int getInvertedReverseIndex(int rowId) {
+ return CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+
+ /**
* Below method will be used to get the surrogate key of the based on the row
* id passed
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index 971f9b4..f79e788 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -33,17 +33,19 @@ public class AndFilterExecuterImpl implements FilterExecuter {
this.rightExecuter = rightExecuter;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
if (leftFilters.isEmpty()) {
return leftFilters;
}
- BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
if (rightFilter.isEmpty()) {
return rightFilter;
}
leftFilters.and(rightFilter);
+ blockChunkHolder.setBitSetGroup(leftFilters);
return leftFilters;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 4471ced..841ecdc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -74,7 +74,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+ throws IOException {
if (isDimensionPresentInCurrentBlock) {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
@@ -89,7 +91,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
- dimensionRawColumnChunk.getRowCount()[i]);
+ dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -109,8 +112,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < ColumnPages.length; i++) {
BitSet bitSet =
- getFilteredIndexes(ColumnPages[i], measureRawColumnChunk.getRowCount()[i],
- msrType);
+ getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i, msrType);;
bitSetGroup.setBitSet(bitSet, i);
}
return bitSetGroup;
@@ -167,14 +171,144 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
- int numerOfRows) {
- if (dimColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimColumnDataChunk, numerOfRows);
+ /**
+ * Below method will be used to apply filter on measure column
+ * @param measureColumnPage
+ * @param numberOfRows
+ * @param useBitsetPipeLine
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param msrDataType
+ * @return filtered indexes bitset
+ */
+ private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
+ boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
+ // check whether previous indexes can be optimal to apply filter on measure column
+ if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+ msrColumnExecutorInfo.getFilterKeys().length)) {
+ return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,
+ numberOfRows, msrDataType);
+ } else {
+ return getFilteredIndexes(measureColumnPage, numberOfRows, msrDataType);
+ }
+ }
+ /**
+ * Below method will be used to apply filter on measure column based on previous filtered indexes
+ * @param measureColumnPage
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param numberOfRows
+ * @param msrDataType
+ * @return filtred indexes bitset
+ */
+ private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
+ BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
+ BitSet bitSet = new BitSet(numberOfRows);
+ bitSet.flip(0, numberOfRows);
+ Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
+ BitSet nullBitSet = measureColumnPage.getNullBits();
+ BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i] == null) {
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.flip(j);
+ }
+ continue;
+ }
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ if (!nullBitSet.get(index)) {
+ // Check if filterValue[i] matches with measure Values.
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnPage, index,
+ msrDataType, msrColumnEvaluatorInfo.getMeasure());
+
+ if (comparator.compare(msrValue, filterValues[i]) == 0) {
+ // This is a match.
+ bitSet.flip(index);
+ }
+ }
+ }
+ }
+ return bitSet;
+ }
+
+ /**
+ * Below method will be used to apply filter on dimension column
+ * @param dimensionColumnDataChunk
+ * @param numberOfRows
+ * @param useBitsetPipeLine
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @return filtered indexes bitset
+ */
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
+ // check whether applying filtered based on previous bitset will be optimal
+ if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+ dimColumnExecuterInfo.getFilterKeys().length)) {
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+ numberOfRows);
+ } else {
+ return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ }
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numberOfRows) {
+ if (dimensionColumnDataChunk.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
}
- return setFilterdIndexToBitSet(dimColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
}
+ /**
+ * Below method will be used to apply filter based on previous filtered bitset
+ * @param dimensionColumnDataChunk
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param numberOfRows
+ * @return filtered indexes bitset
+ */
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+ BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+ BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ BitSet bitSet = new BitSet(numberOfRows);
+ byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
+ int compareResult = 0;
+ // if dimension data was natural sorted then get the index from previous bitset
+ // and use the same in next column data, otherwise use the inverted index reverse
+ if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ compareResult = CarbonUtil
+ .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ if (compareResult != 0) {
+ bitSet.set(index);
+ } else {
+ if (bitSet.get(index)) {
+ bitSet.flip(index);
+ }
+ }
+ }
+ } else {
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ compareResult = CarbonUtil
+ .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
+ dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ if (compareResult != 0) {
+ bitSet.set(index);
+ } else {
+ if (bitSet.get(index)) {
+ bitSet.flip(index);
+ }
+ }
+ }
+ }
+ return bitSet;
+ }
private BitSet setFilterdIndexToBitSetWithColumnIndex(
DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 7182dd5..93640fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -31,7 +31,7 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder)
+ BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException;
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 45831e3..1cbc0bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -84,7 +84,9 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
return bitSet;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+ throws IOException {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 2195c0a..7d6fd4f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -81,7 +81,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+ throws IOException {
if (isDimensionPresentInCurrentBlock == true) {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
@@ -97,12 +99,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
- dimensionRawColumnChunk.getRowCount()[i]);
+ dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
- dimensionRawColumnChunk.getRowCount()[i]);
+ dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
}
@@ -124,14 +128,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasures(measureRawColumnChunk.convertToColumnPage(i),
- measureRawColumnChunk.getRowCount()[i], msrType);
+ getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasures(measureRawColumnChunk.convertToColumnPage(i),
- measureRawColumnChunk.getRowCount()[i], msrType);
+ getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
}
@@ -189,14 +195,137 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
+ /**
+ * Below method will be used to apply filter on measure column
+ * @param measureColumnPage
+ * @param numberOfRows
+ * @param useBitsetPipeLine
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param msrDataType
+ * @return filtered indexes bitset
+ */
+ private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
+ boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
+ // check whether previous indexes can be optimal to apply filter on measure column
+ if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+ msrColumnExecutorInfo.getFilterKeys().length)) {
+ return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,
+ numberOfRows, msrDataType);
+ } else {
+ return getFilteredIndexesForMeasures(measureColumnPage, numberOfRows, msrDataType);
+ }
+ }
+
+ /**
+ * Below method will be used to apply filter on measure column based on previous filtered indexes
+ * @param measureColumnPage
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param numberOfRows
+ * @param msrDataType
+ * @return filtred indexes bitset
+ */
+ private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
+ BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
+ BitSet bitSet = new BitSet(numberOfRows);
+ Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
+ BitSet nullBitSet = measureColumnPage.getNullBits();
+ BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i] == null) {
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.set(j);
+ }
+ continue;
+ }
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ if (!nullBitSet.get(index)) {
+ // Check if filterValue[i] matches with measure Values.
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnPage, index,
+ msrDataType, msrColumnEvaluatorInfo.getMeasure());
+
+ if (comparator.compare(msrValue, filterValues[i]) == 0) {
+ // This is a match.
+ bitSet.set(index);
+ }
+ }
+ }
+ }
+ return bitSet;
+ }
+
+ /**
+ * Below method will be used to apply filter on dimension column
+ * @param dimensionColumnDataChunk
+ * @param numberOfRows
+ * @param useBitsetPipeLine
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @return filtered indexes bitset
+ */
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
+ // check whether previous indexes can be optimal to apply filter on dimension column
+ if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+ dimColumnExecuterInfo.getFilterKeys().length)) {
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+ numberOfRows);
+ } else {
+ return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ }
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numberOfRows) {
if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
}
+ /**
+ * Below method will be used to apply filter on dimension
+ * column based on previous filtered indexes
+ * @param dimensionColumnDataChunk
+ * @param prvBitSetGroup
+ * @param pageNumber
+ * @param numberOfRows
+ * @return filtered bitset
+ */
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+ BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+ BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+ BitSet bitSet = new BitSet(numberOfRows);
+ byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
+ int compareResult = 0;
+ // if dimension data was natural sorted then get the index from previous bitset
+ // and use the same in next column data, otherwise use the inverted index reverse
+ if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ compareResult = CarbonUtil
+ .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ if (compareResult == 0) {
+ bitSet.set(index);
+ }
+ }
+ } else {
+ for (int index = prvPageBitSet.nextSetBit(0);
+ index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+ compareResult = CarbonUtil
+ .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
+ dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ if (compareResult == 0) {
+ bitSet.set(index);
+ }
+ }
+ }
+ return bitSet;
+ }
private BitSet setFilterdIndexToBitSetWithColumnIndex(
DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 119bda7..7eed8ee 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -33,12 +33,13 @@ public class OrFilterExecuterImpl implements FilterExecuter {
this.rightExecuter = rightExecuter;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
- BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false);
+ BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false);
leftFilters.or(rightFilters);
-
+ blockChunkHolder.setBitSetGroup(leftFilters);
return leftFilters;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index c7a0ae7..acce53e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -140,7 +140,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
return applyNoAndDirectFilter(blockChunkHolder);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 1509949..2c0d39f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -45,7 +45,9 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
}
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+ throws IOException {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 0e9a5f0..5ec6971 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -44,7 +44,9 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
}
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+ throws IOException {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8f3eb93..08ce565 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -186,7 +186,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
readBlocks(blockChunkHolder);
// CHECKSTYLE:ON
@@ -222,20 +223,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
for (int i = 0; i < pageNumbers; i++) {
BitSet set = new BitSet(numberOfRows[i]);
RowIntf row = new RowImpl();
- for (int index = 0; index < numberOfRows[i]; index++) {
- createRow(blockChunkHolder, row ,i, index);
- Boolean rslt = false;
- try {
- rslt = exp.evaluate(row).getBoolean();
- }
- // Any invalid member while evaluation shall be ignored, system will log the
- // error only once since all rows the evaluation happens so inorder to avoid
- // too much log inforation only once the log will be printed.
- catch (FilterIllegalMemberException e) {
- FilterUtil.logError(e, false);
+ BitSet prvBitset = null;
+ // if bitset pipe line is enabled then use rowid from previous bitset
+ // otherwise use older flow
+ if (!useBitsetPipeLine || null == blockChunkHolder.getBitSetGroup() || null == bitSetGroup
+ .getBitSet(i) || blockChunkHolder.getBitSetGroup().getBitSet(i).isEmpty()) {
+ for (int index = 0; index < numberOfRows[i]; index++) {
+ createRow(blockChunkHolder, row, i, index);
+ Boolean rslt = false;
+ try {
+ rslt = exp.evaluate(row).getBoolean();
+ }
+ // Any invalid member while evaluation shall be ignored, system will log the
+ // error only once since all rows the evaluation happens so inorder to avoid
+ // too much log inforation only once the log will be printed.
+ catch (FilterIllegalMemberException e) {
+ FilterUtil.logError(e, false);
+ }
+ if (null != rslt && rslt) {
+ set.set(index);
+ }
}
- if (null != rslt && rslt) {
- set.set(index);
+ } else {
+ prvBitset = blockChunkHolder.getBitSetGroup().getBitSet(i);
+ for (int index = prvBitset.nextSetBit(0);
+ index >= 0; index = prvBitset.nextSetBit(index + 1)) {
+ createRow(blockChunkHolder, row, i, index);
+ Boolean rslt = false;
+ try {
+ rslt = exp.evaluate(row).getBoolean();
+ } catch (FilterIllegalMemberException e) {
+ FilterUtil.logError(e, false);
+ }
+ if (null != rslt && rslt) {
+ set.set(index);
+ }
}
}
bitSetGroup.setBitSet(set, i);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index d34ab2c..306f3fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -166,7 +166,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return false;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 3f8eddd..db55e42 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -164,7 +164,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
return false;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index f8886f9..67fe848 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -165,7 +165,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return false;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 580f963..8db37c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -167,7 +167,8 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
return false;
}
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
index ac8c423..91cebc5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -32,7 +32,7 @@ public class TrueFilterExecutor implements FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
BitSetGroup group = new BitSetGroup(numberOfPages);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 3e33d04..6dc1375 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -25,7 +25,8 @@ import org.apache.carbondata.core.util.BitSetGroup;
public class ValueBasedFilterExecuterImpl implements FilterExecuter {
- @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override
+ public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
throws FilterUnsupportedException, IOException {
return new BitSetGroup(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 5227115..60090d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -20,6 +20,7 @@ import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
/**
* Block chunk holder which will hold the dimension and
@@ -47,6 +48,8 @@ public class BlocksChunkHolder {
*/
private DataRefNode dataBlock;
+ private BitSetGroup bitSetGroup;
+
public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
@@ -127,4 +130,12 @@ public class BlocksChunkHolder {
this.dimensionRawDataChunk[i] = null;
}
}
+
+ public BitSetGroup getBitSetGroup() {
+ return bitSetGroup;
+ }
+
+ public void setBitSetGroup(BitSetGroup bitSetGroup) {
+ this.bitSetGroup = bitSetGroup;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 919b837..abfc5f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -64,6 +64,8 @@ public class FilterScanner extends AbstractBlockletScanner {
private QueryStatisticsModel queryStatisticsModel;
+ private boolean useBitSetPipeLine;
+
public FilterScanner(BlockExecutionInfo blockExecutionInfo,
QueryStatisticsModel queryStatisticsModel) {
super(blockExecutionInfo);
@@ -77,6 +79,13 @@ public class FilterScanner extends AbstractBlockletScanner {
// get the filter tree
this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
this.queryStatisticsModel = queryStatisticsModel;
+
+ String useBitSetPipeLine = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
+ CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
+ if (null != useBitSetPipeLine) {
+ this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
+ }
}
/**
@@ -145,7 +154,7 @@ public class FilterScanner extends AbstractBlockletScanner {
totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
totalBlockletStatistic.getCount() + 1);
// apply filter on actual data
- BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder);
+ BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
// if indexes is empty then return with empty result
if (bitSetGroup.isEmpty()) {
CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d0171b9..b5b3e63 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1994,5 +1994,51 @@ public final class CarbonUtil {
}
}
+ /**
+ * Below method will be used to check whether bitset applied on previous filter
+ * can be used to apply on next column filter
+ * @param usePrvBitSetGroup
+ * @param prvBitsetGroup
+ * @param pageNumber
+ * @param numberOfFilterValues
+ * @return
+ */
+ public static boolean usePreviousFilterBitsetGroup(boolean usePrvBitSetGroup,
+ BitSetGroup prvBitsetGroup, int pageNumber, int numberOfFilterValues) {
+ if (!usePrvBitSetGroup || null == prvBitsetGroup || null == prvBitsetGroup.getBitSet(pageNumber)
+ || prvBitsetGroup.getBitSet(pageNumber).isEmpty()) {
+ return false;
+ }
+ int numberOfRowSelected = prvBitsetGroup.getBitSet(pageNumber).cardinality();
+ return numberOfFilterValues > numberOfRowSelected;
+ }
+
+ /**
+ * Below method will be used to check filter value is present in the data chunk or not
+ * @param filterValues
+ * @param dimensionColumnDataChunk
+ * @param low
+ * @param high
+ * @param chunkRowIndex
+ * @return
+ */
+ public static int isFilterPresent(byte[][] filterValues,
+ DimensionColumnDataChunk dimensionColumnDataChunk, int low, int high, int chunkRowIndex) {
+ int compareResult = 0;
+ int mid = 0;
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ compareResult = dimensionColumnDataChunk.compareTo(chunkRowIndex, filterValues[mid]);
+ if (compareResult < 0) {
+ high = mid - 1;
+ } else if (compareResult > 0) {
+ low = mid + 1;
+ } else {
+ return compareResult;
+ }
+ }
+ return -1;
+ }
+
}