You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/02/09 07:58:23 UTC
[21/35] carbondata git commit: [CARBONDATA-2099] Refactor query scan
process to improve readability
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 9391ebd..44f7c07 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -16,20 +16,10 @@
*/
package org.apache.carbondata.core.scan.filter.executer;
-import java.util.ArrayList;
import java.util.BitSet;
-import java.util.List;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
@@ -38,12 +28,6 @@ import org.apache.carbondata.core.util.ByteUtil;
public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
/**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
* @param dimColResolvedFilterInfo
* @param segmentProperties
*/
@@ -53,54 +37,6 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
}
/**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getExcludeFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.flip(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
* Check if scan is required on given block based on min and max value
*/
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
@@ -109,25 +45,4 @@ public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl
return bitSet;
}
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 465bee6..5974666 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -40,13 +40,13 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
- protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ private DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ private DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
private SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -82,48 +82,50 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
- DimensionColumnDataChunk[] dimensionColumnDataChunks =
- dimensionRawColumnChunk.convertToDimColDataChunks();
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
+ DimensionColumnPage[] dimensionColumnPages =
+ dimensionRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
- for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
- BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
+ for (int i = 0; i < dimensionColumnPages.length; i++) {
+ BitSet bitSet = getFilteredIndexes(dimensionColumnPages[i],
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
ColumnPage[] ColumnPages =
- measureRawColumnChunk.convertToColumnPage();
+ measureRawColumnChunk.decodeAllColumnPages();
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < ColumnPages.length; i++) {
BitSet bitSet =
getFilteredIndexesForMeasure(
- measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i],
useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(),
+ rawBlockletColumnChunks.getBitSetGroup(),
i,
msrType);
bitSetGroup.setBitSet(bitSet, i);
@@ -133,7 +135,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return null;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax) {
if (isDimensionPresentInCurrentBlock) {
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
byte[] col = (byte[])value.getVal(dimColEvaluatorInfo.getDimension().getOrdinal());
@@ -275,43 +278,41 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether applying filtered based on previous bitset will be optimal
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getExcludeFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
- numberOfRows);
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter based on previous filtered bitset
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
- * @param numberOfRows
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
- BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
+ BitSetGroup prvBitSetGroup, int pageNumber) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet();
bitSet.or(prvPageBitSet);
@@ -319,11 +320,11 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -336,8 +337,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult != 0) {
bitSet.set(index);
} else {
@@ -351,7 +352,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
int startIndex = 0;
@@ -361,10 +362,10 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.flip(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.flip(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -373,7 +374,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
bitSet.flip(0, numerOfRows);
@@ -390,7 +391,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.flip(j);
@@ -403,14 +404,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.flip(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.flip(j);
}
}
@@ -419,26 +420,30 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
if (isDimensionPresentInCurrentBlock) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 85891dc..aac5e63 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public interface FilterExecuter {
@@ -32,8 +32,8 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException;
+ BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException;
boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException;
@@ -50,7 +50,7 @@ public interface FilterExecuter {
/**
* It just reads necessary block for filter executor, it does not uncompress the data.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
*/
- void readBlocks(BlocksChunkHolder blockChunkHolder)throws IOException;
+ void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks)throws IOException;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index 9c4c7ba..b683fd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -23,7 +23,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -41,26 +41,31 @@ public class ImplicitIncludeFilterExecutorImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeline)
- throws FilterUnsupportedException {
- BitSetGroup bitSetGroup = new BitSetGroup(blockChunkHolder.getDataBlock().numberOfPages());
- for (int i = 0; i < blockChunkHolder.getDataBlock().numberOfPages(); i++) {
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeline) {
+ BitSetGroup bitSetGroup = new BitSetGroup(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages());
+ for (int i = 0; i < rawBlockletColumnChunks.getDataBlock().numberOfPages(); i++) {
bitSetGroup.setBitSet(
- setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().getPageRowCount(i)), i);
+ setBitSetForCompleteDimensionData(
+ rawBlockletColumnChunks.getDataBlock().getPageRowCount(i)), i);
}
return bitSetGroup;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return false;
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
return null;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 1cbc0bc..e4da26f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -24,14 +24,14 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
@@ -59,7 +59,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
/**
* It fills BitSet with row index which matches filter key
*/
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ protected BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
@@ -70,7 +70,7 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
byte[] filterVal = filterValues[i];
for (int rowId = 0; rowId < numerOfRows; rowId++) {
byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ dimensionColumnPage.fillRawData(rowId, 0, colData, keyStructureInfo);
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
bitSet.set(rowId);
}
@@ -85,20 +85,21 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -133,10 +134,10 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
BitSet bitSet = new BitSet(1);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
+ byte[] maxValue = getMinMaxData(cols, blkMaxVal[chunkIndex], columnIndex);
+ byte[] minValue = getMinMaxData(cols, blkMinVal[chunkIndex], columnIndex);
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index fe1421c..05328f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -20,7 +20,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -41,12 +41,12 @@ import org.apache.carbondata.core.util.comparator.SerializableComparator;
public class IncludeFilterExecuterImpl implements FilterExecuter {
protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
- protected MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ private MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo;
+ private MeasureColumnExecuterFilterInfo msrColumnExecutorInfo;
protected SegmentProperties segmentProperties;
- protected boolean isDimensionPresentInCurrentBlock = false;
- protected boolean isMeasurePresentInCurrentBlock = false;
+ private boolean isDimensionPresentInCurrentBlock = false;
+ private boolean isMeasurePresentInCurrentBlock = false;
protected SerializableComparator comparator;
/**
* is dimension column data is natural sorted
@@ -86,44 +86,46 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk dimensionRawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
- BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.decodeColumnPage(i),
dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i);
+ rawBlockletColumnChunks.getBitSetGroup(), i);
bitSetGroup.setBitSet(bitSet, i);
}
}
return bitSetGroup;
} else if (isMeasurePresentInCurrentBlock) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
MeasureRawColumnChunk measureRawColumnChunk =
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex];
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(measureRawColumnChunk.getPagesCount());
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < measureRawColumnChunk.getPagesCount(); i++) {
@@ -132,16 +134,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType())) {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
} else {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+ getFilteredIndexesForMeasure(measureRawColumnChunk.decodeColumnPage(i),
measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);
+ rawBlockletColumnChunks.getBitSetGroup(), i, msrType);
bitSetGroup.setBitSet(bitSet, i);
}
}
@@ -293,43 +295,43 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
/**
* Below method will be used to apply filter on dimension column
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numberOfRows
* @param useBitsetPipeLine
* @param prvBitSetGroup
* @param pageNumber
* @return filtered indexes bitset
*/
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
// check whether previous indexes can be optimal to apply filter on dimension column
if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
dimColumnExecuterInfo.getFilterKeys().length)) {
- return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+ return getFilteredIndexesUisngPrvBitset(dimensionColumnPage, prvBitSetGroup, pageNumber,
numberOfRows);
} else {
- return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+ return getFilteredIndexes(dimensionColumnPage, numberOfRows);
}
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numberOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numberOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numberOfRows);
}
/**
* Below method will be used to apply filter on dimension
* column based on previous filtered indexes
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param prvBitSetGroup
* @param pageNumber
* @param numberOfRows
* @return filtered bitset
*/
- private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnPage dimensionColumnPage,
BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
BitSet bitSet = new BitSet(numberOfRows);
@@ -337,11 +339,11 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
int compareResult = 0;
// if dimension data was natural sorted then get the index from previous bitset
// and use the same in next column data, otherwise use the inverted index reverse
- if (!dimensionColumnDataChunk.isExplicitSorted()) {
+ if (!dimensionColumnPage.isExplicitSorted()) {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1, index);
if (compareResult == 0) {
bitSet.set(index);
}
@@ -350,8 +352,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
for (int index = prvPageBitSet.nextSetBit(0);
index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
compareResult = CarbonUtil
- .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
- dimensionColumnDataChunk.getInvertedReverseIndex(index));
+ .isFilterPresent(filterKeys, dimensionColumnPage, 0, filterKeys.length - 1,
+ dimensionColumnPage.getInvertedReverseIndex(index));
if (compareResult == 0) {
bitSet.set(index);
}
@@ -360,7 +362,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int startIndex = 0;
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -369,10 +371,10 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
if (rangeIndex[1] >= 0) {
startIndex = rangeIndex[1] + 1;
@@ -381,7 +383,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
@@ -394,7 +396,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
int[] rangeIndex = CarbonUtil
- .getRangeIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getRangeIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[i]);
for (int j = rangeIndex[0]; j <= rangeIndex[1]; j++) {
bitSet.set(j);
@@ -407,14 +409,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
if (filterValues.length > 1) {
for (int i = 0; i < numerOfRows; i++) {
int index = CarbonUtil.binarySearch(filterValues, 0, filterValues.length - 1,
- dimensionColumnDataChunk.getChunkData(i));
+ dimensionColumnPage.getChunkData(i));
if (index >= 0) {
bitSet.set(i);
}
}
} else {
for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[0]) == 0) {
+ if (dimensionColumnPage.compareTo(j, filterValues[0]) == 0) {
bitSet.set(j);
}
}
@@ -423,26 +425,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
+ @Override
public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = null;
int columnIndex = 0;
- int blockIndex = 0;
+ int chunkIndex = 0;
boolean isScanRequired = false;
if (isDimensionPresentInCurrentBlock) {
filterValues = dimColumnExecuterInfo.getFilterKeys();
columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- isScanRequired =
- isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues);
-
+ chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(columnIndex);
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues);
} else if (isMeasurePresentInCurrentBlock) {
columnIndex = msrColumnEvaluatorInfo.getColumnIndex();
- blockIndex =
- segmentProperties.getMeasuresOrdinalToBlockMapping().get(columnIndex) + segmentProperties
- .getLastDimensionColOrdinal();
- isScanRequired = isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex],
+ chunkIndex =
+ segmentProperties.getMeasuresOrdinalToChunkMapping().get(columnIndex) +
+ segmentProperties.getLastDimensionColOrdinal();
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
msrColumnExecutorInfo.getFilterKeys(),
msrColumnEvaluatorInfo.getType());
}
@@ -492,20 +493,23 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return false;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
- } else if (isMeasurePresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
+ } else if (isMeasurePresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
- blockChunkHolder.getMeasureRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getMeasureChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readMeasureChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 87273bb..9f0afb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -21,7 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class OrFilterExecuterImpl implements FilterExecuter {
@@ -35,30 +35,33 @@ public class OrFilterExecuterImpl implements FilterExecuter {
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false);
- BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ BitSetGroup leftFilters = leftExecuter.applyFilter(rawBlockletColumnChunks, false);
+ BitSetGroup rightFilters = rightExecuter.applyFilter(rawBlockletColumnChunks, false);
leftFilters.or(rightFilters);
- blockChunkHolder.setBitSetGroup(leftFilters);
+ rawBlockletColumnChunks.setBitSetGroup(leftFilters);
return leftFilters;
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException, IOException {
return leftExecuter.applyFilter(value, dimOrdinalMax) ||
rightExecuter.applyFilter(value, dimOrdinalMax);
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
leftFilters.or(rightFilters);
return leftFilters;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- leftExecuter.readBlocks(blockChunkHolder);
- rightExecuter.readBlocks(blockChunkHolder);
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ leftExecuter.readColumnChunks(rawBlockletColumnChunks);
+ rightExecuter.readColumnChunks(rawBlockletColumnChunks);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index ee373c5..4a9716c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -22,11 +22,10 @@ import java.util.BitSet;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -38,8 +37,7 @@ import org.apache.carbondata.core.scan.expression.conditional.LessThanExpression
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -59,25 +57,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* it has index at which given dimension is stored in file
*/
- protected int dimensionBlocksIndex;
+ private int dimensionChunkIndex;
/**
* flag to check whether the filter dimension is present in current block list of dimensions.
* Applicable for restructure scenarios
*/
- protected boolean isDimensionPresentInCurrentBlock;
- boolean lessThanExp;
- boolean lessThanEqualExp;
- boolean greaterThanExp;
- boolean greaterThanEqualExp;
- boolean startBlockMinIsDefaultStart;
- boolean endBlockMaxisDefaultEnd;
- boolean isRangeFullyCoverBlock;
+ private boolean isDimensionPresentInCurrentBlock;
+ private boolean lessThanExp;
+ private boolean lessThanEqualExp;
+ private boolean greaterThanExp;
+ private boolean greaterThanEqualExp;
+ private boolean startBlockMinIsDefaultStart;
+ private boolean endBlockMaxisDefaultEnd;
+ private boolean isRangeFullyCoverBlock;
public RangeValueFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- MeasureColumnResolvedFilterInfo msrColEvaluatorInfo, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
+ Expression exp, byte[][] filterRangeValues, SegmentProperties segmentProperties) {
this.dimColEvaluatorInfo = dimColEvaluatorInfo;
this.exp = exp;
@@ -90,7 +86,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startBlockMinIsDefaultStart = false;
endBlockMaxisDefaultEnd = false;
isRangeFullyCoverBlock = false;
- initDimensionBlockIndexes();
+ initDimensionChunkIndexes();
ifDefaultValueMatchesFilter();
}
@@ -99,13 +95,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* This method will initialize the dimension info for the current block to be
* used for filtering the data
*/
- private void initDimensionBlockIndexes() {
+ private void initDimensionChunkIndexes() {
// find the dimension in the current block dimensions list
CarbonDimension dimensionFromCurrentBlock =
segmentProperties.getDimensionFromCurrentBlock(dimColEvaluatorInfo.getDimension());
if (null != dimensionFromCurrentBlock) {
dimColEvaluatorInfo.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
- this.dimensionBlocksIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ this.dimensionChunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimensionFromCurrentBlock.getOrdinal());
isDimensionPresentInCurrentBlock = true;
}
@@ -135,14 +131,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to apply the filter.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @return
* @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws FilterUnsupportedException, IOException {
- return applyNoAndDirectFilter(blockChunkHolder);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
+ return applyNoAndDirectFilter(rawBlockletColumnChunks);
}
/**
@@ -243,32 +239,32 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Filter Min <-----------------------------------------------> Filter Max
if (isDimensionPresentInCurrentBlock) {
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || (
- (greaterThanExp == true) && (
+ (greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) {
// completely out of block boundary
isScanRequired = false;
} else {
- if (((greaterThanExp == true) && (
+ if (((greaterThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || (
- (greaterThanEqualExp == true) && (
+ (greaterThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) {
startBlockMinIsDefaultStart = true;
}
- if (((lessThanExp == true) && (
+ if (((lessThanExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || (
- (lessThanEqualExp == true) && (
+ (lessThanEqualExp) && (
ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) {
endBlockMaxisDefaultEnd = true;
}
- if (startBlockMinIsDefaultStart == true && endBlockMaxisDefaultEnd == true) {
+ if (startBlockMinIsDefaultStart && endBlockMaxisDefaultEnd) {
isRangeFullyCoverBlock = true;
}
}
@@ -284,7 +280,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* @param blockMinValue
* @return
*/
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangesValues;
int columnIndex = this.dimColEvaluatorInfo.getColumnIndex();
@@ -300,50 +297,49 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* Method to apply the Range Filter.
* @param blockChunkHolder
* @return
- * @throws FilterUnsupportedException
* @throws IOException
*/
- public BitSetGroup applyNoAndDirectFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
+ private BitSetGroup applyNoAndDirectFilter(RawBlockletColumnChunks blockChunkHolder)
+ throws IOException {
// In case of Alter Table Add and Delete Columns the isDimensionPresentInCurrentBlock can be
// false, in that scenario the default values of the column should be shown.
// select all rows if dimension does not exists in the current block
if (!isDimensionPresentInCurrentBlock) {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, true);
+ int numberOfRows = blockChunkHolder.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ blockChunkHolder.getDataBlock().numberOfPages(), numberOfRows, true);
}
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex]) {
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex] =
+ blockChunkHolder.getDataBlock().readDimensionChunk(
+ blockChunkHolder.getFileReader(), chunkIndex);
}
DimensionRawColumnChunk rawColumnChunk =
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ blockChunkHolder.getDimensionRawColumnChunks()[chunkIndex];
BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
if (isScanRequired(rawColumnChunk.getMinValues()[i], rawColumnChunk.getMaxValues()[i],
this.filterRangesValues)) {
- if (isRangeFullyCoverBlock == true) {
+ if (isRangeFullyCoverBlock) {
// Set all the bits in this case as filter Min Max values cover the whole block.
BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
}
} else {
- BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i),
rawColumnChunk.getRowCount()[i]);
bitSetGroup.setBitSet(bitSet, i);
}
@@ -351,12 +347,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
return bitSetGroup;
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet getFilteredIndexes(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+ if (dimensionColumnPage.isExplicitSorted()) {
+ return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnPage, numerOfRows);
}
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ return setFilterdIndexToBitSet(dimensionColumnPage, numerOfRows);
}
/**
@@ -365,12 +361,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* column is not supported by default so column index mapping will be present for
* accesing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
private BitSet setFilterdIndexToBitSetWithColumnIndex(
- DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ DimensionColumnPage dimensionColumnPage, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
int start = 0;
int startIndex = 0;
@@ -381,13 +377,13 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
// Get the Min Value
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -399,8 +395,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
- .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) {
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage
+ .getChunkData(dimensionColumnPage.getInvertedIndex(start)))) > 0) {
start = start + 1;
}
}
@@ -411,14 +407,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// Get the Max value
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -429,7 +425,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
if ((ByteUtil.compare(filterValues[1],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ dimensionColumnPage.getChunkData(dimensionColumnPage.getInvertedIndex(start)))
< 0)) {
start = start - 1;
}
@@ -440,18 +436,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
for (int j = startMin; j <= endMax; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ bitSet.set(dimensionColumnPage.getInvertedIndex(j));
}
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
return bitSet;
}
- private void updateForNoDictionaryColumn(int start, int end, DimensionColumnDataChunk dataChunk,
+ private void updateForNoDictionaryColumn(int start, int end, DimensionColumnPage dataChunk,
BitSet bitset) {
for (int j = start; j <= end; j++) {
if (dataChunk.compareTo(j, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY) == 0
@@ -467,31 +463,31 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
* be called if the column is sorted default so column index
* mapping will be present for accesaing the members from the block.
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param numerOfRows
* @return BitSet.
*/
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ private BitSet setFilterdIndexToBitSet(DimensionColumnPage dimensionColumnPage,
int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
- // if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ // if (dimensionColumnPage instanceof FixedLengthDimensionColumnPage) {
byte[][] filterValues = this.filterRangesValues;
- if (dimensionColumnDataChunk.isExplicitSorted()) {
+ if (dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int startMin = 0;
int endMax = 0;
int startIndex = 0;
// For Range expression we expect two values. The First is the Min Value and Second is the
// Max value.
- if (startBlockMinIsDefaultStart == false) {
+ if (!startBlockMinIsDefaultStart) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[0], greaterThanExp);
- if (greaterThanExp == true && start >= 0) {
+ if (greaterThanExp && start >= 0) {
start = CarbonUtil
- .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[0],
+ .nextGreaterValueToTarget(start, dimensionColumnPage, filterValues[0],
numerOfRows);
}
@@ -503,7 +499,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnPage.getChunkData(start)))
> 0) {
start = start + 1;
}
@@ -513,14 +509,14 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
startMin = startIndex;
}
- if (endBlockMaxisDefaultEnd == false) {
+ if (!endBlockMaxisDefaultEnd) {
start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ .getFirstIndexUsingBinarySearch(dimensionColumnPage, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
- if (lessThanExp == true && start >= 0) {
+ if (lessThanExp && start >= 0) {
start =
- CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
+ CarbonUtil.nextLesserValueToTarget(start, dimensionColumnPage, filterValues[1]);
}
if (start < 0) {
@@ -530,7 +526,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// In case the start is less than 0, then positive value of start is pointing to the next
// value of the searched key. So move to the previous one.
- if ((ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start))
+ if ((ByteUtil.compare(filterValues[1], dimensionColumnPage.getChunkData(start))
< 0)) {
start = start - 1;
}
@@ -546,8 +542,8 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Binary Search cannot be done on '@NU#LL$!", so need to check and compare for null on
// matching row.
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()) {
- updateForNoDictionaryColumn(startMin, endMax, dimensionColumnDataChunk, bitSet);
+ if (dimensionColumnPage.isNoDicitionaryColumn()) {
+ updateForNoDictionaryColumn(startMin, endMax, dimensionColumnPage, bitSet);
}
} else {
byte[] defaultValue = null;
@@ -556,7 +552,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
.getDirectDictionaryGenerator(dimColEvaluatorInfo.getDimension().getDataType());
int key = directDictionaryGenerator.generateDirectSurrogateKey(null);
CarbonDimension currentBlockDimension =
- segmentProperties.getDimensions().get(dimensionBlocksIndex);
+ segmentProperties.getDimensions().get(dimensionChunkIndex);
if (currentBlockDimension.isSortColumn()) {
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
this.segmentProperties.getSortColumnsGenerator());
@@ -572,55 +568,37 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
}
// evaluate result for lower range value first and then perform and operation in the
// upper range value in order to compute the final result
- bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[0],
+ bitSet = evaluateGreaterThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[0],
numerOfRows);
BitSet upperRangeBitSet =
- evaluateLessThanFilterForUnsortedColumn(dimensionColumnDataChunk, filterValues[1],
+ evaluateLessThanFilterForUnsortedColumn(dimensionColumnPage, filterValues[1],
numerOfRows);
bitSet.and(upperRangeBitSet);
- FilterUtil.removeNullValues(dimensionColumnDataChunk, bitSet, defaultValue);
+ FilterUtil.removeNullValues(dimensionColumnPage, bitSet, defaultValue);
}
return bitSet;
}
/**
- * This method will compare the selected data against null values and
- * flip the bitSet if any null value is found
- *
- * @param dimensionColumnDataChunk
- * @param bitSet
- */
- private void removeNullValues(DimensionColumnDataChunk dimensionColumnDataChunk, BitSet bitSet) {
- if (!bitSet.isEmpty()) {
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- if (dimensionColumnDataChunk.compareTo(i, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY)
- == 0) {
- bitSet.flip(i);
- }
- }
- }
- }
-
- /**
* This method will evaluate the result for filter column based on the lower range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateGreaterThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (greaterThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) > 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) > 0)) {
bitSet.set(i);
}
}
} else if (greaterThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) >= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) >= 0)) {
bitSet.set(i);
}
}
@@ -631,23 +609,23 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* This method will evaluate the result for filter column based on the upper range value
*
- * @param dimensionColumnDataChunk
+ * @param dimensionColumnPage
* @param filterValue
* @param numberOfRows
* @return
*/
private BitSet evaluateLessThanFilterForUnsortedColumn(
- DimensionColumnDataChunk dimensionColumnDataChunk, byte[] filterValue, int numberOfRows) {
+ DimensionColumnPage dimensionColumnPage, byte[] filterValue, int numberOfRows) {
BitSet bitSet = new BitSet(numberOfRows);
if (lessThanExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) < 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) < 0)) {
bitSet.set(i);
}
}
} else if (lessThanEqualExp) {
for (int i = 0; i < numberOfRows; i++) {
- if ((ByteUtil.compare(dimensionColumnDataChunk.getChunkData(i), filterValue) <= 0)) {
+ if ((ByteUtil.compare(dimensionColumnPage.getChunkData(i), filterValue) <= 0)) {
bitSet.set(i);
}
}
@@ -657,16 +635,18 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
/**
* Method to read the blocks.
- * @param blockChunkHolder
+ * @param rawBlockletColumnChunks
* @throws IOException
*/
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) throws IOException {
+ if (isDimensionPresentInCurrentBlock) {
+ int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
+ rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] =
+ rawBlockletColumnChunks.getDataBlock().readDimensionChunk(
+ rawBlockletColumnChunks.getFileReader(), chunkIndex);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 5707eb4..d7bec7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -34,7 +34,7 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -48,26 +48,29 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
+ .createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(),
numberOfRows, !isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureExcludeFilterExecutorImpl on row");
}
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ @Override
+ public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.flip(0, 1);
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4749ca29/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 8bcc53f..c874fc5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks;
import org.apache.carbondata.core.util.BitSetGroup;
public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorImpl {
@@ -33,7 +33,7 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
* flag to check whether filter values contain the default value applied on the dimension column
* which does not exist in the current block
*/
- protected boolean isDefaultValuePresentInFilterValues;
+ private boolean isDefaultValuePresentInFilterValues;
public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
MeasureColumnResolvedFilterInfo measureColumnResolvedFilterInfo, boolean isMeasure) {
@@ -47,15 +47,16 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
}
@Override
- public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
- throws IOException {
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- return FilterUtil
- .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),
- numberOfRows, isDefaultValuePresentInFilterValues);
+ public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
+ boolean useBitsetPipeLine) throws IOException {
+ int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
+ return FilterUtil.createBitSetGroupWithDefaultValue(
+ rawBlockletColumnChunks.getDataBlock().numberOfPages(),
+ numberOfRows, isDefaultValuePresentInFilterValues);
}
- @Override public boolean applyFilter(RowIntf value, int dimOrdinalMax)
+ @Override
+ public boolean applyFilter(RowIntf value, int dimOrdinalMax)
throws FilterUnsupportedException {
throw new FilterUnsupportedException("Unsupported RestructureIncludeFilterExecutorImpl on row");
}
@@ -66,7 +67,8 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
return bitSet;
}
- @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override
+ public void readColumnChunks(RawBlockletColumnChunks rawBlockletColumnChunks) {
}