You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/07/31 11:49:16 UTC
[2/6] carbondata git commit: Measure Filter implementation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 03fe2b4..20889ea 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -22,13 +22,16 @@ import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl {
protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+ protected MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo;
protected SegmentProperties segmentProperties;
+ protected boolean isMeasure;
/**
* flag to check whether filter values contain the default value applied on the dimension column
@@ -37,11 +40,19 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
protected boolean isDefaultValuePresentInFilterValues;
public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
- SegmentProperties segmentProperties) {
+ MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo,
+ SegmentProperties segmentProperties, boolean isMeasure) {
this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+ this.measureColumnResolvedFilterInfo = measureColumnResolvedFilterInfo;
+ this.isMeasure = isMeasure;
this.segmentProperties = segmentProperties;
- isDefaultValuePresentInFilterValues =
- isDimensionDefaultValuePresentInFilterValues(dimColumnEvaluatorInfo);
+ if (isMeasure) {
+ isDefaultValuePresentInFilterValues =
+ isMeasureDefaultValuePresentInFilterValues(measureColumnResolvedFilterInfo);
+ } else {
+ isDefaultValuePresentInFilterValues =
+ isDimensionDefaultValuePresentInFilterValues(dimColumnEvaluatorInfo);
+ }
}
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 1f97e9b..40065f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -104,17 +104,31 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
Map<Integer, GenericQueryType> complexDimensionInfoMap) {
- this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
this.segmentProperties = segmentProperties;
- this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
- this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
+ if (null == dimColEvaluatorInfoList) {
+ this.dimColEvaluatorInfoList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ } else {
+ this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
+ }
+ if (dimColEvaluatorInfoList.size() > 0) {
+ this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
+ this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+ } else {
+ this.isDimensionPresentInCurrentBlock = new boolean[]{false};
+ this.dimensionBlocksIndex = new int[]{0};
+ }
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
} else {
this.msrColEvalutorInfoList = msrColEvalutorInfoList;
}
- this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
- this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
+ if (msrColEvalutorInfoList.size() > 0) {
+ this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
+ this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+ } else {
+ this.isMeasurePresentInCurrentBlock = new boolean[]{false};
+ this.measureBlocksIndex = new int[] {0};
+ }
this.exp = exp;
this.tableIdentifier = tableIdentifier;
this.complexDimensionInfoMap = complexDimensionInfoMap;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 73352cb..47c854e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -17,25 +17,34 @@
package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
@@ -44,6 +53,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* flag to check whether default values is present in the filter value list
*/
private boolean isDefaultValuePresentInFilter;
+ private int lastDimensionColOrdinal = 0;
public RowLevelRangeGrtThanFiterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -52,8 +62,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
- isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
- && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
+ if (isDimensionPresentInCurrentBlock[0] == true) {
+ isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
+ && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ }
ifDefaultValueMatchesFilter();
}
@@ -61,7 +74,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
* This method will check whether default value is present in the given filter values
*/
private void ifDefaultValueMatchesFilter() {
- if (!this.isDimensionPresentInCurrentBlock[0]) {
+ if (!dimColEvaluatorInfoList.isEmpty() && !isDimensionPresentInCurrentBlock[0]) {
CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
@@ -74,80 +87,205 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
}
}
}
+ } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
+ CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
+ byte[] defaultValue = measure.getDefaultValue();
+ if (null != defaultValue) {
+ for (int k = 0; k < filterRangeValues.length; k++) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ if (maxCompare < 0) {
+ isDefaultValuePresentInFilter = true;
+ break;
+ }
+ }
+ }
}
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
- isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
+ boolean isScanRequired = false;
+ byte[] maxValue = null;
+ if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
+ if (isMeasurePresentInCurrentBlock[0]) {
+ maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ isScanRequired =
+ isScanRequired(maxValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType());
+ } else {
+ maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ isScanRequired = isScanRequired(maxValue, filterRangeValues);
+ }
+ } else {
+ isScanRequired = isDefaultValuePresentInFilter;
+ }
+
if (isScanRequired) {
bitSet.set(0);
}
return bitSet;
}
+
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
boolean isScanRequired = false;
- if (isDimensionPresentInCurrentBlock[0]) {
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
- // if any filter value is in range than this block needs to be
- // scanned means always less than block max range.
- if (maxCompare < 0) {
- isScanRequired = true;
- break;
- }
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare < 0) {
+ isScanRequired = true;
+ break;
}
- } else {
- isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
+ private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+ DataType dataType) {
+ for (int i = 0; i < filterValue.length; i++) {
+ if (filterValue[i].length == 0 || maxValue.length == 0) {
+ return isScanRequired(maxValue, filterValue);
+ }
+ switch (dataType) {
+ case DOUBLE:
+ double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble();
+ double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble();
+ if (filterValueDouble < maxValueDouble) {
+ return true;
+ }
+ break;
+ case INT:
+ case SHORT:
+ case LONG:
+ long maxValueLong = ByteBuffer.wrap(maxValue).getLong();
+ long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+ if (filterValueLong < maxValueLong) {
+ return true;
+ }
+ break;
+ case DECIMAL:
+ BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue);
+ BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]);
+ if (filterDecimal.compareTo(maxDecimal) < 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
- if (!isDimensionPresentInCurrentBlock[0]) {
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
numberOfRows, true);
}
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
- for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
- if (rawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
- int compare = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
- if (compare < 0) {
- BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
- bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare < 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
}
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare < 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
}
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
}
+ return bitSetGroup;
}
- return bitSetGroup;
+ return null;
+ }
+
+ private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ byte[][] filterValues = this.filterRangeValues;
+ DataType msrType = msrColEvalutorInfoList.get(0).getType();
+ Comparator comparator = PartitionFilterUtil.getComparatorByDataTypeForMeasure(msrType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i].length == 0) {
+ BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet();
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.set(j);
+ }
+ continue;
+ }
+ Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType);
+ for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex,
+ msrType, msrColEvalutorInfoList.get(0).getMeasure());
+
+ if (comparator.compare(msrValue, filter) > 0) {
+ // This is a match.
+ bitSet.set(startIndex);
+ }
+ }
+ }
+ CarbonUtil
+ .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet);
+ return bitSet;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -296,6 +434,12 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex = measureBlocksIndex[0];
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 6e8e188..6b665b2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -17,25 +17,34 @@
package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
@@ -45,6 +54,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* flag to check whether default values is present in the filter value list
*/
private boolean isDefaultValuePresentInFilter;
+ private int lastDimensionColOrdinal = 0;
public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
@@ -54,8 +64,12 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
- isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
- && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
+
+ if (isDimensionPresentInCurrentBlock[0] == true) {
+ isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
+ && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ }
ifDefaultValueMatchesFilter();
}
@@ -63,7 +77,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
* This method will check whether default value is present in the given filter values
*/
private void ifDefaultValueMatchesFilter() {
- if (!this.isDimensionPresentInCurrentBlock[0]) {
+ if (!dimColEvaluatorInfoList.isEmpty() && !isDimensionPresentInCurrentBlock[0]) {
CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
@@ -76,13 +90,39 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
}
}
+ } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
+ CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
+ byte[] defaultValue = measure.getDefaultValue();
+ if (null != defaultValue) {
+ for (int k = 0; k < filterRangeValues.length; k++) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ if (maxCompare <= 0) {
+ isDefaultValuePresentInFilter = true;
+ break;
+ }
+ }
+ }
}
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
- isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
+ boolean isScanRequired = false;
+ byte[] maxValue = null;
+ if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
+ if (isMeasurePresentInCurrentBlock[0]) {
+ maxValue = blockMaxValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ isScanRequired =
+ isScanRequired(maxValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType());
+ } else {
+ maxValue = blockMaxValue[dimensionBlocksIndex[0]];
+ isScanRequired = isScanRequired(maxValue, filterRangeValues);
+ }
+ } else {
+ isScanRequired = isDefaultValuePresentInFilter;
+ }
+
if (isScanRequired) {
bitSet.set(0);
}
@@ -91,67 +131,167 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
boolean isScanRequired = false;
- if (isDimensionPresentInCurrentBlock[0]) {
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
- // if any filter value is in range than this block needs to be
- // scanned less than equal to max range.
- if (maxCompare <= 0) {
- isScanRequired = true;
- break;
- }
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare <= 0) {
+ isScanRequired = true;
+ break;
}
- } else {
- isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
+ private boolean isScanRequired(byte[] maxValue, byte[][] filterValue,
+ DataType dataType) {
+ for (int i = 0; i < filterValue.length; i++) {
+ if (filterValue[i].length == 0 || maxValue.length == 0) {
+ return isScanRequired(maxValue, filterValue);
+ }
+ switch (dataType) {
+ case DOUBLE:
+ double maxValueDouble = ByteBuffer.wrap(maxValue).getDouble();
+ double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble();
+ if (filterValueDouble <= maxValueDouble) {
+ return true;
+ }
+ break;
+ case INT:
+ case SHORT:
+ case LONG:
+ long maxValueLong = ByteBuffer.wrap(maxValue).getLong();
+ long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+ if (filterValueLong <= maxValueLong) {
+ return true;
+ }
+ break;
+ case DECIMAL:
+ BigDecimal maxDecimal = DataTypeUtil.byteToBigDecimal(maxValue);
+ BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]);
+ if (filterDecimal.compareTo(maxDecimal) <= 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
- if (!isDimensionPresentInCurrentBlock[0]) {
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
numberOfRows, true);
}
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
- for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
- if (rawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
- int compare = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
- if (compare <= 0) {
- BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
- bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
+
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare <= 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
}
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare <= 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
}
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
}
+ return bitSetGroup;
}
- return bitSetGroup;
+ return null;
}
+ private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ byte[][] filterValues = this.filterRangeValues;
+ DataType msrType = msrColEvalutorInfoList.get(0).getType();
+ Comparator comparator = PartitionFilterUtil.getComparatorByDataTypeForMeasure(msrType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i].length == 0) {
+ BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet();
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.set(j);
+ }
+ continue;
+ }
+ Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType);
+ for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex,
+ msrType, msrColEvalutorInfoList.get(0).getMeasure());
+
+ if (comparator.compare(msrValue, filter) >= 0) {
+ // This is a match.
+ bitSet.set(startIndex);
+ }
+ }
+ }
+ CarbonUtil
+ .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet);
+ return bitSet;
+ }
+
+
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
int numerOfRows) {
BitSet bitSet = null;
@@ -283,6 +423,12 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex = measureBlocksIndex[0];
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index d6f7c86..6d6ca19 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -17,27 +17,36 @@
package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
protected byte[][] filterRangeValues;
@@ -46,6 +55,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
* flag to check whether default values is present in the filter value list
*/
private boolean isDefaultValuePresentInFilter;
+ private int lastDimensionColOrdinal = 0;
public RowLevelRangeLessThanEqualFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -53,17 +63,20 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
SegmentProperties segmentProperties) {
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
+ lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
this.filterRangeValues = filterRangeValues;
ifDefaultValueMatchesFilter();
- isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
- && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ if (isDimensionPresentInCurrentBlock[0] == true) {
+ isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
+ && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ }
}
/**
* This method will check whether default value is present in the given filter values
*/
private void ifDefaultValueMatchesFilter() {
- if (!this.isDimensionPresentInCurrentBlock[0]) {
+ if (!dimColEvaluatorInfoList.isEmpty() && !isDimensionPresentInCurrentBlock[0]) {
CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
@@ -76,13 +89,38 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
}
}
}
+ } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
+ CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
+ byte[] defaultValue = measure.getDefaultValue();
+ if (null != defaultValue) {
+ for (int k = 0; k < filterRangeValues.length; k++) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ if (maxCompare >= 0) {
+ isDefaultValuePresentInFilter = true;
+ break;
+ }
+ }
+ }
}
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
- isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
+ byte[] minValue = null;
+ boolean isScanRequired = false;
+ if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
+ if (isMeasurePresentInCurrentBlock[0]) {
+ minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ isScanRequired =
+ isScanRequired(minValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType());
+ } else {
+ minValue = blockMinValue[dimensionBlocksIndex[0]];
+ isScanRequired = isScanRequired(minValue, filterRangeValues);
+ }
+ } else {
+ isScanRequired = isDefaultValuePresentInFilter;
+ }
if (isScanRequired) {
bitSet.set(0);
}
@@ -91,57 +129,147 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues) {
boolean isScanRequired = false;
- if (isDimensionPresentInCurrentBlock[0]) {
- for (int k = 0; k < filterValues.length; k++) {
- // and filter-min should be positive
- int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+ for (int k = 0; k < filterValues.length; k++) {
+ // and filter-min should be positive
+ int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
- // if any filter applied is not in range of min and max of block
- // then since its a less than equal to fiter validate whether the block
- // min range is less than equal to applied filter member
- if (minCompare >= 0) {
- isScanRequired = true;
- break;
- }
+ // if any filter applied is not in range of min and max of block
+ // then since its a less than equal to fiter validate whether the block
+ // min range is less than equal to applied filter member
+ if (minCompare >= 0) {
+ isScanRequired = true;
+ break;
}
- } else {
- isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
+ private boolean isScanRequired(byte[] minValue, byte[][] filterValue,
+ DataType dataType) {
+ for (int i = 0; i < filterValue.length; i++) {
+ if (filterValue[i].length == 0 || minValue.length == 0) {
+ return isScanRequired(minValue, filterValue);
+ }
+ switch (dataType) {
+ case DOUBLE:
+ double minValueDouble = ByteBuffer.wrap(minValue).getDouble();
+ double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble();
+ if (filterValueDouble >= minValueDouble) {
+ return true;
+ }
+ break;
+ case INT:
+ case SHORT:
+ case LONG:
+ long minValueLong = ByteBuffer.wrap(minValue).getLong();
+ long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+ if (filterValueLong >= minValueLong) {
+ return true;
+ }
+ break;
+ case DECIMAL:
+ BigDecimal minDecimal = DataTypeUtil.byteToBigDecimal(minValue);
+ BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]);
+ if (filterDecimal.compareTo(minDecimal) >= 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
- if (!isDimensionPresentInCurrentBlock[0]) {
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
numberOfRows, true);
}
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
- for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
- if (rawColumnChunk.getMinValues() != null) {
- if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ } else {
BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
+ }
+ return bitSetGroup;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ }
+ return null;
+ }
+
+ private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ byte[][] filterValues = this.filterRangeValues;
+ DataType msrType = msrColEvalutorInfoList.get(0).getType();
+ Comparator comparator = PartitionFilterUtil.getComparatorByDataTypeForMeasure(msrType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i].length == 0) {
+ BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet();
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.set(j);
+ }
+ continue;
+ }
+ Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType);
+ for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex,
+ msrType, msrColEvalutorInfoList.get(0).getMeasure());
+
+ if (comparator.compare(msrValue, filter) <= 0) {
+ // This is a match.
+ bitSet.set(startIndex);
+ }
}
}
- return bitSetGroup;
+ CarbonUtil
+ .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet);
+ return bitSet;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -322,6 +450,12 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex = measureBlocksIndex[0];
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 597ba52..98e4a78 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -17,27 +17,36 @@
package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.BitSet;
+import java.util.Comparator;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.partition.PartitionFilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
@@ -46,7 +55,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
* flag to check whether default values is present in the filter value list
*/
private boolean isDefaultValuePresentInFilter;
-
+ private int lastDimensionColOrdinal = 0;
public RowLevelRangeLessThanFiterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -55,16 +64,19 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
null);
this.filterRangeValues = filterRangeValues;
+ lastDimensionColOrdinal = segmentProperties.getLastDimensionColOrdinal();
ifDefaultValueMatchesFilter();
- isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
- && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ if (isDimensionPresentInCurrentBlock[0] == true) {
+ isNaturalSorted = dimColEvaluatorInfoList.get(0).getDimension().isUseInvertedIndex()
+ && dimColEvaluatorInfoList.get(0).getDimension().isSortColumn();
+ }
}
/**
* This method will check whether default value is present in the given filter values
*/
private void ifDefaultValueMatchesFilter() {
- if (!this.isDimensionPresentInCurrentBlock[0]) {
+ if (!dimColEvaluatorInfoList.isEmpty() && !isDimensionPresentInCurrentBlock[0]) {
CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
@@ -77,72 +89,188 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
}
}
}
+ } else if (!msrColEvalutorInfoList.isEmpty() && !isMeasurePresentInCurrentBlock[0]) {
+ CarbonMeasure measure = this.msrColEvalutorInfoList.get(0).getMeasure();
+ byte[] defaultValue = measure.getDefaultValue();
+ if (null != defaultValue) {
+ for (int k = 0; k < filterRangeValues.length; k++) {
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ if (maxCompare > 0) {
+ isDefaultValuePresentInFilter = true;
+ break;
+ }
+ }
+ }
}
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
- isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
+ byte[] minValue = null;
+ boolean isScanRequired = false;
+ if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
+ if (isMeasurePresentInCurrentBlock[0]) {
+ minValue = blockMinValue[measureBlocksIndex[0] + lastDimensionColOrdinal];
+ isScanRequired =
+ isScanRequired(minValue, filterRangeValues, msrColEvalutorInfoList.get(0).getType());
+ } else {
+ minValue = blockMinValue[dimensionBlocksIndex[0]];
+ isScanRequired = isScanRequired(minValue, filterRangeValues);
+ }
+ } else {
+ isScanRequired = isDefaultValuePresentInFilter;
+ }
if (isScanRequired) {
bitSet.set(0);
}
return bitSet;
}
+
private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues) {
boolean isScanRequired = false;
- if (isDimensionPresentInCurrentBlock[0]) {
- for (int k = 0; k < filterValues.length; k++) {
- // and filter-min should be positive
- int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
+ for (int k = 0; k < filterValues.length; k++) {
+ // and filter-min should be positive
+ int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
- // if any filter applied is not in range of min and max of block
- // then since its a less than fiter validate whether the block
- // min range is less than applied filter member
- if (minCompare > 0) {
- isScanRequired = true;
- break;
- }
+ // if any filter applied is not in range of min and max of block
+ // then since its a less than equal to fiter validate whether the block
+ // min range is less than equal to applied filter member
+ if (minCompare > 0) {
+ isScanRequired = true;
+ break;
}
- } else {
- isScanRequired = isDefaultValuePresentInFilter;
}
return isScanRequired;
}
+ private boolean isScanRequired(byte[] minValue, byte[][] filterValue,
+ DataType dataType) {
+ for (int i = 0; i < filterValue.length; i++) {
+ if (filterValue[i].length == 0 || minValue.length == 0) {
+ return isScanRequired(minValue, filterValue);
+ }
+ switch (dataType) {
+ case DOUBLE:
+ double minValueDouble = ByteBuffer.wrap(minValue).getDouble();
+ double filterValueDouble = ByteBuffer.wrap(filterValue[i]).getDouble();
+ if (filterValueDouble > minValueDouble) {
+ return true;
+ }
+ break;
+ case INT:
+ case SHORT:
+ case LONG:
+ long minValueLong = ByteBuffer.wrap(minValue).getLong();
+ long filterValueLong = ByteBuffer.wrap(filterValue[i]).getLong();
+ if (filterValueLong > minValueLong) {
+ return true;
+ }
+ break;
+ case DECIMAL:
+ BigDecimal minDecimal = DataTypeUtil.byteToBigDecimal(minValue);
+ BigDecimal filterDecimal = DataTypeUtil.byteToBigDecimal(filterValue[i]);
+ if (filterDecimal.compareTo(minDecimal) > 0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
// select all rows if dimension does not exists in the current block
- if (!isDimensionPresentInCurrentBlock[0]) {
+ if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
return FilterUtil
.createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
numberOfRows, true);
}
- int blockIndex =
- segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
- for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
- if (rawColumnChunk.getMinValues() != null) {
- if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
+ if (isDimensionPresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getDimensionOrdinalToBlockMapping().get(dimensionBlocksIndex[0]);
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ } else {
BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
- } else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
- rawColumnChunk.getRowCount()[i]);
- bitSetGroup.setBitSet(bitSet, i);
+ }
+ return bitSetGroup;
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex =
+ segmentProperties.getMeasuresOrdinalToBlockMapping().get(measureBlocksIndex[0]);
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ MeasureRawColumnChunk rawColumnChunk = blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ msrColEvalutorInfoList.get(0).getType())) {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ } else {
+ BitSet bitSet =
+ getFilteredIndexesForMeasures(rawColumnChunk.convertToMeasureColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
+ }
+ return null;
+ }
+
+ private BitSet getFilteredIndexesForMeasures(MeasureColumnDataChunk measureColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ byte[][] filterValues = this.filterRangeValues;
+ DataType msrType = msrColEvalutorInfoList.get(0).getType();
+ Comparator comparator = PartitionFilterUtil.getComparatorByDataTypeForMeasure(msrType);
+ for (int i = 0; i < filterValues.length; i++) {
+ if (filterValues[i].length == 0) {
+ BitSet nullBitSet = measureColumnDataChunk.getNullValueIndexHolder().getBitSet();
+ for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+ bitSet.set(j);
+ }
+ continue;
+ }
+ Object filter = DataTypeUtil.getMeasureObjectFromDataType(filterValues[i], msrType);
+ for (int startIndex = 0; startIndex < numerOfRows; startIndex++) {
+ Object msrValue = DataTypeUtil
+ .getMeasureObjectBasedOnDataType(measureColumnDataChunk.getColumnPage(), startIndex,
+ msrType, msrColEvalutorInfoList.get(0).getMeasure());
+
+ if (comparator.compare(msrValue, filter) < 0) {
+ // This is a match.
+ bitSet.set(startIndex);
+ }
}
}
- return bitSetGroup;
+ CarbonUtil
+ .updateBitSetForNull(measureColumnDataChunk.getNullValueIndexHolder().getBitSet(), bitSet);
+ return bitSet;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -337,6 +465,12 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
+ } else if (isMeasurePresentInCurrentBlock[0]) {
+ int blockIndex = measureBlocksIndex[0];
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
index d040c1b..a056d94 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
@@ -30,6 +30,8 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;
+import org.apache.spark.sql.types.Decimal;
+
public class PartitionFilterUtil {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 609a130..95d3223 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -24,7 +24,6 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
@@ -35,6 +34,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
public class ConditionalFilterResolverImpl implements FilterResolverIntf {
@@ -44,16 +44,22 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
protected boolean isExpressionResolve;
protected boolean isIncludeFilter;
private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
+ private MeasureColumnResolvedFilterInfo msrColResolvedFilterInfo;
private AbsoluteTableIdentifier tableIdentifier;
+ private boolean isMeasure;
public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+ boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier, boolean isMeasure) {
this.exp = exp;
this.isExpressionResolve = isExpressionResolve;
this.isIncludeFilter = isIncludeFilter;
this.tableIdentifier = tableIdentifier;
- this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
-
+ this.isMeasure = isMeasure;
+ if (isMeasure == false) {
+ this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
+ } else {
+ this.msrColResolvedFilterInfo = new MeasureColumnResolvedFilterInfo();
+ }
}
@@ -83,11 +89,11 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
// column expression.
// we need to check if the other expression contains column
// expression or not in depth.
- CarbonDimension dimension = columnExpression.getDimension();
if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
- || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) &&
- dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) && ((
+ (null != columnExpression.getDimension()) && (columnExpression.getDimension()
+ .hasEncoding(Encoding.DICTIONARY) && !columnExpression.getDimension()
+ .hasEncoding(Encoding.DIRECT_DICTIONARY))))) {
isExpressionResolve = true;
} else {
//Visitor pattern is been used in this scenario inorder to populate the
@@ -96,9 +102,16 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
//3 types of visitors custom,direct and no dictionary, all types of visitor populate
//the visitable instance as per its buisness logic which is different for all the
// visitors.
- dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
- metadata);
+ if (columnExpression.isMeasure()) {
+ msrColResolvedFilterInfo.setMeasure(columnExpression.getMeasure());
+ msrColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
+ metadata);
+ } else {
+ dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
+ metadata);
+ }
}
} else if (rightExp instanceof ColumnExpression) {
ColumnExpression columnExpression = (ColumnExpression) rightExp;
@@ -119,10 +132,15 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
isExpressionResolve = true;
} else {
- dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
- metadata);
-
+ if (columnExpression.isMeasure()) {
+ msrColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
+ metadata);
+ } else {
+ dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression, exp),
+ metadata);
+ }
}
}
} else {
@@ -135,24 +153,34 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
metadata.setColumnExpression(columnList.get(0));
metadata.setExpression(exp);
metadata.setIncludeFilter(isIncludeFilter);
- if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) || columnList.get(0)
- .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)
+ if ((null != columnList.get(0).getDimension()) && (
+ !columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) || columnList.get(0)
+ .getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY))
|| (exp instanceof RangeExpression)) {
dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0), exp),
metadata);
- } else if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(
- columnList.get(0).getDimension().getDataType()
- == org.apache.carbondata.core.metadata.datatype.DataType.STRUCT
- || columnList.get(0).getDimension().getDataType()
- == org.apache.carbondata.core.metadata.datatype.DataType.ARRAY)) {
+ } else if ((null != columnList.get(0).getDimension()) && (
+ columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(
+ columnList.get(0).getDimension().getDataType()
+ == org.apache.carbondata.core.metadata.datatype.DataType.STRUCT
+ || columnList.get(0).getDimension().getDataType()
+ == org.apache.carbondata.core.metadata.datatype.DataType.ARRAY))) {
dimColResolvedFilterInfo.setFilterValues(FilterUtil
.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
isIncludeFilter));
dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension());
+ } else if (columnList.get(0).isMeasure()) {
+ msrColResolvedFilterInfo.setMeasure(columnList.get(0).getMeasure());
+ msrColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0), exp),
+ metadata);
+ msrColResolvedFilterInfo.setCarbonColumn(columnList.get(0).getCarbonColumn());
+ msrColResolvedFilterInfo.setColumnIndex(columnList.get(0).getCarbonColumn().getOrdinal());
+ msrColResolvedFilterInfo.setType(columnList.get(0).getCarbonColumn().getDataType());
}
}
@@ -189,6 +217,17 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
return dimColResolvedFilterInfo;
}
+ /**
+ * Method will return the MeasureColumnResolvedFilterInfo instance which consists
+ * the mapping of the respective dimension and its surrogates involved in
+ * filter expression.
+ *
+ * @return DimColumnResolvedFilterInfo
+ */
+ public MeasureColumnResolvedFilterInfo getMsrColResolvedFilterInfo() {
+ return msrColResolvedFilterInfo;
+ }
+
public AbsoluteTableIdentifier getTableIdentifier() {
return tableIdentifier;
}
@@ -198,10 +237,18 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
*/
public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
- FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
- segmentProperties, startKey, startKeyList);
- FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
- segmentProperties, setOfStartKeyByteArray);
+ if (null != dimColResolvedFilterInfo) {
+ FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+ segmentProperties, startKey, startKeyList);
+ FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
+ setOfStartKeyByteArray);
+ }
+// else {
+// FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+// segmentProperties, startKey, startKeyList);
+// FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
+// setOfStartKeyByteArray);
+// }
}
/**
@@ -209,10 +256,12 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
*/
@Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
- FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
- segmentProperties, endKeyList);
- FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
- setOfEndKeyByteArray);
+ if (null != dimColResolvedFilterInfo) {
+ FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
+ segmentProperties, endKeyList);
+ FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo, segmentProperties,
+ setOfEndKeyByteArray);
+ }
}
/**
@@ -258,7 +307,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
*
* @return
*/
- public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
+ public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
if (null != dimColResolvedFilterInfo.getFilterValues() && !dimColResolvedFilterInfo
.getDimension().hasEncoding(Encoding.DICTIONARY)) {
@@ -268,7 +317,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
} else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo
.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(),
- this.dimColResolvedFilterInfo.getDimension(), segmentProperties);
+ this.dimColResolvedFilterInfo.getDimension(), null, segmentProperties);
}
return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
index 746b96d..89a3890 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
public interface FilterResolverIntf extends Serializable {
@@ -66,6 +67,14 @@ public interface FilterResolverIntf extends Serializable {
DimColumnResolvedFilterInfo getDimColResolvedFilterInfo();
/**
+ * API will return the resolved filter instance, this instance will provide
+ * the resolved surrogates based on the applied filter
+ *
+ * @return MeasureColumnResolvedFilterInfo object
+ */
+ MeasureColumnResolvedFilterInfo getMsrColResolvedFilterInfo();
+
+ /**
* API will get the start key based on the filter applied based on the key generator
*
* @param segmentProperties
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
index db35823..368cd9a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
public class LogicalFilterResolverImpl implements FilterResolverIntf {
/**
@@ -84,6 +85,9 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
return null;
}
+ @Override public MeasureColumnResolvedFilterInfo getMsrColResolvedFilterInfo() {
+ return null;
+ }
@Override public void getStartKey(SegmentProperties segmentProperties, long[] startKey,
SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/266c473b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index 919faaf..b399fd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -39,7 +39,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier);
+ super(exp, isExpressionResolve, isIncludeFilter, tableIdentifier, false);
dimColEvaluatorInfoList =
new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
@@ -72,6 +72,7 @@ public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
msrColumnEvalutorInfo.setRowIndex(index++);
msrColumnEvalutorInfo
.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
+ msrColumnEvalutorInfo.setMeasure(columnExpression.getMeasure());
msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
}