You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/10/26 10:56:35 UTC
carbondata git commit: [CARBONDATA-3013] Added support for pruning
pages for vector direct fill.
Repository: carbondata
Updated Branches:
refs/heads/master 3d3b6ff16 -> e6d15da74
[CARBONDATA-3013] Added support for pruning pages for vector direct fill.
First, apply page level pruning using the min/max of each page and get the valid pages of blocklet.
Decompress only valid pages and fill the vector directly as mentioned in full scan query scenario.
For this purpose to prune pages first before decompressing the data, added new method inside a class FilterExecuter.
BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks) throws FilterUnsupportedException, IOException;
The above method reads the necessary column chunk metadata and prunes the pages as per the min/max meta.
Based on the pruned pages BlockletScannedResult decompresses and fills the column page data to vector as described in full scan in above mentioned PR .
This closes #2820
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e6d15da7
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e6d15da7
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e6d15da7
Branch: refs/heads/master
Commit: e6d15da74a3a4d9c0af9d6886b811ac5bb2d89e9
Parents: 3d3b6ff
Author: ravipesala <ra...@gmail.com>
Authored: Tue Oct 16 14:53:14 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Fri Oct 26 16:24:38 2018 +0530
----------------------------------------------------------------------
.../filter/executer/AndFilterExecuterImpl.java | 15 ++
.../executer/ExcludeFilterExecuterImpl.java | 10 ++
.../filter/executer/FalseFilterExecutor.java | 8 +
.../scan/filter/executer/FilterExecuter.java | 6 +
.../ImplicitIncludeFilterExecutorImpl.java | 9 +
.../executer/IncludeFilterExecuterImpl.java | 87 ++++++++--
.../filter/executer/OrFilterExecuterImpl.java | 9 +
.../executer/RangeValueFilterExecuterImpl.java | 38 +++++
.../executer/RestructureEvaluatorImpl.java | 10 ++
.../executer/RowLevelFilterExecuterImpl.java | 10 ++
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 85 ++++++++--
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 88 ++++++++--
...velRangeLessThanEqualFilterExecuterImpl.java | 87 ++++++++--
...RowLevelRangeLessThanFilterExecuterImpl.java | 86 ++++++++--
.../filter/executer/TrueFilterExecutor.java | 9 +
.../scanner/impl/BlockletFilterScanner.java | 166 ++++++++++++++++++-
16 files changed, 656 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index d743151..f0feb0e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -50,6 +50,21 @@ public class AndFilterExecuterImpl implements FilterExecuter, ImplicitColumnFilt
return leftFilters;
}
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ BitSet leftFilters = leftExecuter.prunePages(rawBlockletColumnChunks);
+ if (leftFilters.isEmpty()) {
+ return leftFilters;
+ }
+ BitSet rightFilter = rightExecuter.prunePages(rawBlockletColumnChunks);
+ if (rightFilter.isEmpty()) {
+ return rightFilter;
+ }
+ leftFilters.and(rightFilter);
+ return leftFilters;
+ }
+
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return leftExecuter.applyFilter(value, dimOrdinalMax) &&
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 15a43c5..fc9fbae 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -144,6 +145,15 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
index c2fa1ef..4066af8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FalseFilterExecutor.java
@@ -45,6 +45,14 @@ public class FalseFilterExecutor implements FilterExecuter {
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawChunks)
+ throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawChunks.getDataBlock().numberOfPages();
+ BitSet set = new BitSet(numberOfPages);
+ return set;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index edf88b1..88b9d6c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -36,6 +36,12 @@ public interface FilterExecuter {
boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException;
/**
+ * Prune pages as per the filter
+ */
+ BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException;
+
+ /**
* apply range filter on a row
* @return true: if the value satisfy the filter; or else false.
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index bb30d71..fd93e59 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -54,6 +54,15 @@ public class ImplicitIncludeFilterExecutorImpl
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 8070c2f..4668242 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -109,20 +110,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
boolean isDecoded = false;
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
- boolean scanRequired;
- // for no dictionary measure column comparison can be done
- // on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
- && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
- dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys(),
- dimColumnEvaluatorInfo.getDimension().getDataType());
- } else {
- scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
- dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys(),
- dimensionRawColumnChunk.getMinMaxFlagArray()[i]);
- }
- if (scanRequired) {
+ if (isScanRequired(dimensionRawColumnChunk, i)) {
DimensionColumnPage dimensionColumnPage = dimensionRawColumnChunk.decodeColumnPage(i);
if (!isDecoded) {
filterValues = FilterUtil
@@ -179,6 +167,77 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return null;
}
+ private boolean isScanRequired(DimensionRawColumnChunk dimensionRawColumnChunk, int columnIndex) {
+ boolean scanRequired;
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
+ && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex],
+ dimensionRawColumnChunk.getMinValues()[columnIndex],
+ dimColumnExecuterInfo.getFilterKeys(),
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else {
+ scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[columnIndex],
+ dimensionRawColumnChunk.getMinValues()[columnIndex],
+ dimColumnExecuterInfo.getFilterKeys(),
+ dimensionRawColumnChunk.getMinMaxFlagArray()[columnIndex]);
+ }
+ return scanRequired;
+ }
+
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
+ .get(dimColumnEvaluatorInfo.getColumnIndex());
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk dimensionRawColumnChunk =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ filterValues = dimColumnExecuterInfo.getFilterKeys();
+ BitSet bitSet = new BitSet(dimensionRawColumnChunk.getPagesCount());
+ for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
+ if (dimensionRawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(dimensionRawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else if (isMeasurePresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColumnEvaluatorInfo.getColumnIndex());
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ MeasureRawColumnChunk measureRawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(measureRawColumnChunk.getPagesCount());
+ for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) {
+ if (measureRawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(measureRawColumnChunk.getMaxValues()[i],
+ measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
+ msrColumnEvaluatorInfo.getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+ return null;
+ }
+
@Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 4e7dec2..951d965 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -45,6 +45,15 @@ public class OrFilterExecuterImpl implements FilterExecuter {
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ BitSet leftFilters = leftExecuter.prunePages(rawBlockletColumnChunks);
+ BitSet rightFilters = rightExecuter.prunePages(rawBlockletColumnChunks);
+ leftFilters.or(rightFilters);
+ return leftFilters;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return leftExecuter.applyFilter(value, dimOrdinalMax) ||
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 886a13b..970756a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -146,6 +146,44 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter {
return applyNoAndDirectFilter(rawBlockletColumnChunks, useBitsetPipeLine);
}
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ // In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be
+ // false, in that scenario the default values of the column should be shown.
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock) {
+ int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet();
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex] = blockChunkHolder.getDataBlock()
+ .readDimensionChunk(blockChunkHolder.getFileReader(), chunkIndex);
+ }
+
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i],
+ this.filterRangesValues, rawColumnChunk.getMinMaxFlagArray()[i])) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+
+ }
+
/**
* apply range filter on a row
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
index c403846..a25394f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
@@ -17,7 +17,9 @@
package org.apache.carbondata.core.scan.filter.executer;
+import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.BitSet;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -28,10 +30,12 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.ColumnFilterInfo;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;
@@ -104,6 +108,12 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
return isDefaultValuePresentInFilterValues;
}
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ return new BitSet();
+ }
+
/**
* This method will check whether a default value for the non-existing column is present
* in the filter values list
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 28c3f87..7ca2579 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -290,6 +290,16 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ readColumnChunks(rawBlockletColumnChunks);
+ int pages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet();
+ bitSet.set(0, pages);
+ return bitSet;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index c6835f8..06dc3c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -148,6 +148,61 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColEvalutorInfoList.get(0).getColumnIndex());
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+ return null;
+ }
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, boolean isMinMaxSet) {
if (!isMinMaxSet) {
@@ -236,19 +291,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
- boolean scanRequired;
- DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
- // for no dictionary measure column comparison can be done
- // on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
- scanRequired =
- isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
- } else {
- scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i],
- this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]);
- }
- if (scanRequired) {
+ if (isScanRequired(rawColumnChunk, i)) {
int compare = ByteUtil.UnsafeComparer.INSTANCE
.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
if (compare < 0) {
@@ -333,6 +376,22 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return null;
}
+ private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i],
+ this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ return scanRequired;
+ }
+
@Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index afb646a..e4c507d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -234,19 +234,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
- boolean scanRequired;
- DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
- // for no dictionary measure column comparison can be done
- // on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
- scanRequired =
- isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
- } else {
- scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i]);
- }
- if (scanRequired) {
+ if (isScanRequired(rawColumnChunk, i)) {
int compare = ByteUtil.UnsafeComparer.INSTANCE
.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
if (compare <= 0) {
@@ -331,6 +319,80 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
}
+ private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ return scanRequired;
+ }
+
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColEvalutorInfoList.get(0).getColumnIndex());
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+
+ }
+
@Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 647425d..ac9661e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -235,19 +235,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
- boolean scanRequired;
- DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
- // for no dictionary measure column comparison can be done
- // on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
- scanRequired =
- isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
- } else {
- scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i]);
- }
- if (scanRequired) {
+ if (isScanRequired(rawColumnChunk, i)) {
BitSet bitSet;
DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i);
if (null != rawColumnChunk.getLocalDictionary()) {
@@ -314,6 +302,79 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return null;
}
+ private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ return scanRequired;
+ }
+
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColEvalutorInfoList.get(0).getColumnIndex());
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+ return null;
+ }
+
@Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
index 4ef9999..644cf02 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -233,19 +233,7 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
- boolean scanRequired;
- DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
- // for no dictionary measure column comparison can be done
- // on the original data as like measure column
- if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
- scanRequired =
- isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
- } else {
- scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i]);
- }
- if (scanRequired) {
+ if (isScanRequired(rawColumnChunk, i)) {
BitSet bitSet;
DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i);
if (null != rawColumnChunk.getLocalDictionary()) {
@@ -311,6 +299,78 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
}
}
+ private boolean isScanRequired(DimensionRawColumnChunk rawColumnChunk, int i) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ return scanRequired;
+ }
+
+ @Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ // select all rows if dimension does not exists in the current block
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet bitSet = new BitSet(numberOfPages);
+ bitSet.set(0, numberOfPages);
+ return bitSet;
+ }
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int chunkIndex =
+ segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk, i)) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ } else {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
+ .get(msrColEvalutorInfoList.get(0).getColumnIndex());
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ BitSet bitSet = new BitSet(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.msrFilterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ bitSet.set(i);
+ }
+ } else {
+ bitSet.set(i);
+ }
+ }
+ return bitSet;
+ }
+ }
+
@Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
index 08831a4..3fd3716 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -46,6 +46,15 @@ public class TrueFilterExecutor implements FilterExecuter {
}
@Override
+ public BitSet prunePages(RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ int numberOfPages = rawBlockletColumnChunks.getDataBlock().numberOfPages();
+ BitSet set = new BitSet(numberOfPages);
+ set.set(0, numberOfPages);
+ return set;
+ }
+
+ @Override
public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
return true;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e6d15da7/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
index 57849e2..fc32862 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/BlockletFilterScanner.java
@@ -98,7 +98,11 @@ public class BlockletFilterScanner extends BlockletFullScanner {
@Override
public BlockletScannedResult scanBlocklet(RawBlockletColumnChunks rawBlockletColumnChunks)
throws IOException, FilterUnsupportedException {
- return executeFilter(rawBlockletColumnChunks);
+ if (blockExecutionInfo.isDirectVectorFill()) {
+ return executeFilterForPages(rawBlockletColumnChunks);
+ } else {
+ return executeFilter(rawBlockletColumnChunks);
+ }
}
@Override
@@ -316,4 +320,164 @@ public class BlockletFilterScanner extends BlockletFullScanner {
readTime.getCount() + dimensionReadTime);
return scannedResult;
}
+
+ /**
+ * This method will process the data in below order
+ * 1. first apply min max on the filter tree and check whether any of the filter
+ * is fall on the range of min max, if not then return empty result
+ * 2. If filter falls on min max range then apply filter on actual
+ * data and get the pruned pages.
+ * 3. if pruned pages are not empty then read only those blocks(measure or dimension)
+ * which was present in the query but not present in the filter, as while applying filter
+ * some of the blocks where already read and present in chunk holder so not need to
+ * read those blocks again, this is to avoid reading of same blocks which was already read
+ * 4. Set the blocks and filter pages to scanned result
+ *
+ * @param rawBlockletColumnChunks blocklet raw chunk of all columns
+ * @throws FilterUnsupportedException
+ */
+ private BlockletScannedResult executeFilterForPages(
+ RawBlockletColumnChunks rawBlockletColumnChunks)
+ throws FilterUnsupportedException, IOException {
+ long startTime = System.currentTimeMillis();
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ // apply filter on actual data, for each page
+ BitSet pages = this.filterExecuter.prunePages(rawBlockletColumnChunks);
+ // if filter result is empty then return with empty result
+ if (pages.isEmpty()) {
+ CarbonUtil.freeMemory(rawBlockletColumnChunks.getDimensionRawColumnChunks(),
+ rawBlockletColumnChunks.getMeasureRawColumnChunks());
+
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime));
+
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount());
+ return createEmptyResult();
+ }
+
+ BlockletScannedResult scannedResult =
+ new FilterQueryScannedResult(blockExecutionInfo, queryStatisticsModel);
+
+ // valid scanned blocklet
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ // adding statistics for valid number of pages
+ QueryStatistic validPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_PAGE_SCANNED);
+ validPages.addCountStatistic(QueryStatisticsConstants.VALID_PAGE_SCANNED,
+ validPages.getCount() + pages.cardinality());
+ QueryStatistic scannedPages = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.PAGE_SCANNED);
+ scannedPages.addCountStatistic(QueryStatisticsConstants.PAGE_SCANNED,
+ scannedPages.getCount() + pages.cardinality());
+ // get the row indexes from bit set for each page
+ int[] pageFilteredPages = new int[pages.cardinality()];
+ int index = 0;
+ for (int i = pages.nextSetBit(0); i >= 0; i = pages.nextSetBit(i + 1)) {
+ pageFilteredPages[index++] = i;
+ }
+ // count(*) case there would not be any dimensions are measures selected.
+ int[] numberOfRows = new int[pages.cardinality()];
+ for (int i = 0; i < numberOfRows.length; i++) {
+ numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
+ }
+ long dimensionReadTime = System.currentTimeMillis();
+ dimensionReadTime = System.currentTimeMillis() - dimensionReadTime;
+ FileReader fileReader = rawBlockletColumnChunks.getFileReader();
+ DimensionRawColumnChunk[] dimensionRawColumnChunks =
+ new DimensionRawColumnChunk[blockExecutionInfo.getTotalNumberDimensionToRead()];
+ int numDimensionChunks = dimensionRawColumnChunks.length;
+ // read dimension chunk blocks from file which is not present
+ for (int chunkIndex = 0; chunkIndex < numDimensionChunks; chunkIndex++) {
+ dimensionRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ }
+ int[][] allSelectedDimensionColumnIndexRange =
+ blockExecutionInfo.getAllSelectedDimensionColumnIndexRange();
+ DimensionRawColumnChunk[] projectionListDimensionChunk = rawBlockletColumnChunks.getDataBlock()
+ .readDimensionChunks(fileReader, allSelectedDimensionColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedDimensionColumnIndexRange) {
+ System.arraycopy(projectionListDimensionChunk, columnIndexRange[0],
+ dimensionRawColumnChunks, columnIndexRange[0],
+ columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+
+ /*
+ * in case projection if the projected dimension are not loaded in the dimensionColumnDataChunk
+ * then loading them
+ */
+ int[] projectionListDimensionIndexes = blockExecutionInfo.getProjectionListDimensionIndexes();
+ for (int projectionListDimensionIndex : projectionListDimensionIndexes) {
+ if (null == dimensionRawColumnChunks[projectionListDimensionIndex]) {
+ dimensionRawColumnChunks[projectionListDimensionIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ fileReader, projectionListDimensionIndex);
+ }
+ }
+
+ DimensionColumnPage[][] dimensionColumnPages =
+ new DimensionColumnPage[numDimensionChunks][pages.cardinality()];
+ MeasureRawColumnChunk[] measureRawColumnChunks =
+ new MeasureRawColumnChunk[blockExecutionInfo.getTotalNumberOfMeasureToRead()];
+ int numMeasureChunks = measureRawColumnChunks.length;
+
+ // read the measure chunk blocks which is not present
+ for (int chunkIndex = 0; chunkIndex < numMeasureChunks; chunkIndex++) {
+ if (null != rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ measureRawColumnChunks[chunkIndex] =
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
+ }
+ }
+
+ int[][] allSelectedMeasureColumnIndexRange =
+ blockExecutionInfo.getAllSelectedMeasureIndexRange();
+ MeasureRawColumnChunk[] projectionListMeasureChunk = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunks(fileReader, allSelectedMeasureColumnIndexRange);
+ for (int[] columnIndexRange : allSelectedMeasureColumnIndexRange) {
+ System.arraycopy(projectionListMeasureChunk, columnIndexRange[0], measureRawColumnChunks,
+ columnIndexRange[0], columnIndexRange[1] + 1 - columnIndexRange[0]);
+ }
+ /*
+ * in case projection if the projected measure are not loaded in the ColumnPage
+ * then loading them
+ */
+ int[] projectionListMeasureIndexes = blockExecutionInfo.getProjectionListMeasureIndexes();
+ for (int projectionListMeasureIndex : projectionListMeasureIndexes) {
+ if (null == measureRawColumnChunks[projectionListMeasureIndex]) {
+ measureRawColumnChunks[projectionListMeasureIndex] = rawBlockletColumnChunks.getDataBlock()
+ .readMeasureChunk(fileReader, projectionListMeasureIndex);
+ }
+ }
+ ColumnPage[][] measureColumnPages = new ColumnPage[numMeasureChunks][pages.cardinality()];
+ scannedResult.setDimensionColumnPages(dimensionColumnPages);
+ scannedResult.setMeasureColumnPages(measureColumnPages);
+ scannedResult.setDimRawColumnChunks(dimensionRawColumnChunks);
+ scannedResult.setMsrRawColumnChunks(measureRawColumnChunks);
+ scannedResult.setPageFilteredRowCount(numberOfRows);
+ scannedResult.setPageIdFiltered(pageFilteredPages);
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockIdString() + CarbonCommonConstants.FILE_SEPARATOR +
+ rawBlockletColumnChunks.getDataBlock().blockletIndex());
+ // adding statistics for carbon scan time
+ QueryStatistic scanTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.SCAN_BLOCKlET_TIME);
+ scanTime.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKlET_TIME,
+ scanTime.getCount() + (System.currentTimeMillis() - startTime - dimensionReadTime));
+ QueryStatistic readTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.READ_BLOCKlET_TIME);
+ readTime.addCountStatistic(QueryStatisticsConstants.READ_BLOCKlET_TIME,
+ readTime.getCount() + dimensionReadTime);
+ return scannedResult;
+ }
}