You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/02/24 05:38:44 UTC
[3/5] incubator-carbondata git commit: WIP Added code for new V3
format to optimize scan
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 66633f3..00a0e01 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -20,8 +20,8 @@ import java.io.IOException;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
import org.apache.carbondata.core.datastore.FileHolder;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
@@ -84,9 +84,9 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @param blockIndexes indexes of the blocks need to be read
* @return dimension data chunks
*/
- @Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader,
+ @Override public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return dimensionChunksReader.readDimensionChunks(fileReader, blockIndexes);
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
}
/**
@@ -96,9 +96,9 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @param blockIndex block index to be read
* @return dimension data chunk
*/
- @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
+ @Override public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader,
int blockIndex) throws IOException {
- return dimensionChunksReader.readDimensionChunk(fileReader, blockIndex);
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndex);
}
/**
@@ -108,9 +108,9 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @param blockIndexes block indexes to be read from file
* @return measure column data chunk
*/
- @Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader,
+ @Override public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader,
int[][] blockIndexes) throws IOException {
- return measureColumnChunkReader.readMeasureChunks(fileReader, blockIndexes);
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
}
/**
@@ -120,8 +120,8 @@ public class BlockletBTreeLeafNode extends AbstractBTreeLeafNode {
* @param blockIndex block index to be read from file
* @return measure data chunk
*/
- @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
throws IOException {
- return measureColumnChunkReader.readMeasureChunk(fileReader, blockIndex);
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index fe64e88..97f3822 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -50,17 +50,21 @@ public class BlockletInfo implements Serializable {
private List<Long> dimensionChunkOffsets;
- private List<Short> dimensionChunksLength;
+ private List<Integer> dimensionChunksLength;
private List<Long> measureChunkOffsets;
- private List<Short> measureChunksLength;
+ private List<Integer> measureChunksLength;
/**
* to store the index like min max and start and end key of each column of the blocklet
*/
private BlockletIndex blockletIndex;
+ private long dimensionOffset;
+
+ private long measureOffsets;
+
/**
* @return the numberOfRows
*/
@@ -125,11 +129,11 @@ public class BlockletInfo implements Serializable {
this.dimensionChunkOffsets = dimensionChunkOffsets;
}
- public List<Short> getDimensionChunksLength() {
+ public List<Integer> getDimensionChunksLength() {
return dimensionChunksLength;
}
- public void setDimensionChunksLength(List<Short> dimensionChunksLength) {
+ public void setDimensionChunksLength(List<Integer> dimensionChunksLength) {
this.dimensionChunksLength = dimensionChunksLength;
}
@@ -141,12 +145,28 @@ public class BlockletInfo implements Serializable {
this.measureChunkOffsets = measureChunkOffsets;
}
- public List<Short> getMeasureChunksLength() {
+ public List<Integer> getMeasureChunksLength() {
return measureChunksLength;
}
- public void setMeasureChunksLength(List<Short> measureChunksLength) {
+ public void setMeasureChunksLength(List<Integer> measureChunksLength) {
this.measureChunksLength = measureChunksLength;
}
+ public long getDimensionOffset() {
+ return dimensionOffset;
+ }
+
+ public void setDimensionOffset(long dimensionOffset) {
+ this.dimensionOffset = dimensionOffset;
+ }
+
+ public long getMeasureOffsets() {
+ return measureOffsets;
+ }
+
+ public void setMeasureOffsets(long measureOffsets) {
+ this.measureOffsets = measureOffsets;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index f622ddc..9bc40c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -108,27 +108,41 @@ public class DictionaryBasedVectorResultCollector extends AbstractScannedResultC
@Override public void collectVectorBatch(AbstractScannedResult scannedResult,
CarbonColumnarBatch columnarBatch) {
- int rowCounter = scannedResult.getRowCounter();
- int availableRows = scannedResult.numberOfOutputRows() - rowCounter;
- int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getActualSize();
- requiredRows = Math.min(requiredRows, availableRows);
- if (requiredRows < 1) {
- return;
- }
- for (int i = 0; i < allColumnInfo.length; i++) {
- allColumnInfo[i].size = requiredRows;
- allColumnInfo[i].offset = rowCounter;
- allColumnInfo[i].vectorOffset = columnarBatch.getRowCounter();
- allColumnInfo[i].vector = columnarBatch.columnVectors[i];
- }
+ int numberOfPages = scannedResult.numberOfpages();
+ while (scannedResult.getCurrentPageCounter() < numberOfPages) {
+ int currentPageRowCount = scannedResult.getCurrentPageRowCount();
+ if (currentPageRowCount == 0) {
+ scannedResult.incrementPageCounter();
+ continue;
+ }
+ int rowCounter = scannedResult.getRowCounter();
+ int availableRows = currentPageRowCount - rowCounter;
+ int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getActualSize();
+ requiredRows = Math.min(requiredRows, availableRows);
+ if (requiredRows < 1) {
+ return;
+ }
+ for (int i = 0; i < allColumnInfo.length; i++) {
+ allColumnInfo[i].size = requiredRows;
+ allColumnInfo[i].offset = rowCounter;
+ allColumnInfo[i].vectorOffset = columnarBatch.getRowCounter();
+ allColumnInfo[i].vector = columnarBatch.columnVectors[i];
+ }
- scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
- scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
- scannedResult.fillColumnarMeasureBatch(measureInfo, measuresOrdinal);
- scannedResult.fillColumnarComplexBatch(complexInfo);
- scannedResult.setRowCounter(rowCounter + requiredRows);
- columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows);
- columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
+ scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
+ scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
+ scannedResult.fillColumnarMeasureBatch(measureInfo, measuresOrdinal);
+ scannedResult.fillColumnarComplexBatch(complexInfo);
+ // it means fetched all data out of page so increment the page counter
+ if (availableRows == requiredRows) {
+ scannedResult.incrementPageCounter();
+ } else {
+ // Or set the row counter.
+ scannedResult.setRowCounter(rowCounter + requiredRows);
+ }
+ columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows);
+ columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
index 4588074..46c6714 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
@@ -21,12 +21,13 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-import org.apache.spark.sql.catalyst.util.*;
-import org.apache.spark.sql.types.*;
+import org.apache.spark.sql.catalyst.util.GenericArrayData;
+import org.apache.spark.sql.types.ArrayType;
+import org.apache.spark.sql.types.DataType;
public class ArrayQueryType extends ComplexQueryType implements GenericQueryType {
@@ -61,11 +62,10 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
}
- public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
+ public void parseBlocksAndReturnComplexColumnByteArray(DimensionRawColumnChunk[] rawColumnChunks,
+ int rowNumber, int pageNumber, DataOutputStream dataOutputStream) throws IOException {
byte[] input = new byte[8];
- copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
+ copyBlockDataChunk(rawColumnChunks, rowNumber, pageNumber, input);
ByteBuffer byteArray = ByteBuffer.wrap(input);
int dataLength = byteArray.getInt();
dataOutputStream.writeInt(dataLength);
@@ -73,7 +73,7 @@ public class ArrayQueryType extends ComplexQueryType implements GenericQueryType
int columnIndex = byteArray.getInt();
for (int i = 0; i < dataLength; i++) {
children
- .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, columnIndex++,
+ .parseBlocksAndReturnComplexColumnByteArray(rawColumnChunks, columnIndex++, pageNumber,
dataOutputStream);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
index 0966508..080d577 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.scan.complextypes;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -45,9 +45,10 @@ public class ComplexQueryType {
* @param rowNumber
* @param input
*/
- protected void copyBlockDataChunk(DimensionColumnDataChunk[] dimensionColumnDataChunks,
- int rowNumber, byte[] input) {
- byte[] data = dimensionColumnDataChunks[blockIndex].getChunkData(rowNumber);
+ protected void copyBlockDataChunk(DimensionRawColumnChunk[] rawColumnChunks,
+ int rowNumber, int pageNumber, byte[] input) {
+ byte[] data =
+ rawColumnChunks[blockIndex].convertToDimColDataChunk(pageNumber).getChunkData(rowNumber);
System.arraycopy(data, 0, input, 0, data.length);
}
@@ -55,8 +56,8 @@ public class ComplexQueryType {
* This method will read the block data chunk from the respective block
*/
protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index 2f693d3..9c9be86 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.Bits;
@@ -92,11 +93,12 @@ public class PrimitiveQueryType extends ComplexQueryType implements GenericQuery
}
@Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
- byte[] currentVal =
- new byte[dimensionDataChunks[blockIndex].getColumnValueSize()];
- copyBlockDataChunk(dimensionDataChunks, rowNumber, currentVal);
+ DimensionRawColumnChunk[] rawColumnChunks, int rowNumber,
+ int pageNumber, DataOutputStream dataOutputStream) throws IOException {
+ DimensionColumnDataChunk dataChunk =
+ rawColumnChunks[blockIndex].convertToDimColDataChunk(pageNumber);
+ byte[] currentVal = new byte[dataChunk.getColumnValueSize()];
+ copyBlockDataChunk(rawColumnChunks, rowNumber, pageNumber, currentVal);
dataOutputStream.write(currentVal);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
index 75b97c7..bb64e92 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
@@ -23,7 +23,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
@@ -82,10 +82,10 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
}
@Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
+ DimensionRawColumnChunk[] dimensionColumnDataChunks, int rowNumber,
+ int pageNumber, DataOutputStream dataOutputStream) throws IOException {
byte[] input = new byte[8];
- copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
+ copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, pageNumber, input);
ByteBuffer byteArray = ByteBuffer.wrap(input);
int childElement = byteArray.getInt();
dataOutputStream.writeInt(childElement);
@@ -93,7 +93,7 @@ public class StructQueryType extends ComplexQueryType implements GenericQueryTyp
for (int i = 0; i < childElement; i++) {
children.get(i)
.parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber,
- dataOutputStream);
+ pageNumber, dataOutputStream);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 82c193f..0c2e8ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Executors;
@@ -103,7 +104,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier()
.getCarbonTableIdentifier().getTableName());
// add executor service for query execution
- queryProperties.executorService = Executors.newFixedThreadPool(1);
+ queryProperties.executorService = Executors.newCachedThreadPool();
// Initializing statistics list to record the query statistics
// creating copy on write to handle concurrent scenario
queryProperties.queryStatisticsRecorder =
@@ -331,7 +332,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// setting all the dimension chunk indexes to be read from file
int numberOfElementToConsider = 0;
// list of dimensions to be projected
- List<Integer> allProjectionListDimensionIdexes = new ArrayList<>();
+ Set<Integer> allProjectionListDimensionIdexes = new LinkedHashSet<>();
int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension,
segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
queryProperties.complexFilterDimension, allProjectionListDimensionIdexes);
@@ -346,10 +347,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
} else {
blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
}
-
+ // list of measures to be projected
+ List<Integer> allProjectionListMeasureIdexes = new ArrayList<>();
int[] measureBlockIndexes = QueryUtil
.getMeasureBlockIndexes(queryModel.getQueryMeasures(), expressionMeasures,
- segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures);
+ segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures,
+ allProjectionListMeasureIdexes);
if (measureBlockIndexes.length > 0) {
numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
@@ -363,6 +366,14 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
} else {
blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
}
+ // setting the indexes of list of dimension in projection list
+ blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(
+ allProjectionListDimensionIdexes
+ .toArray(new Integer[allProjectionListDimensionIdexes.size()])));
+ // setting the indexes of list of measures in projection list
+ blockExecutionInfo.setProjectionListMeasureIndexes(ArrayUtils.toPrimitive(
+ allProjectionListMeasureIdexes
+ .toArray(new Integer[allProjectionListMeasureIdexes.size()])));
// setting the key structure info which will be required
// to update the older block key with new key generator
blockExecutionInfo.setKeyStructureInfo(queryProperties.keyStructureInfo);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index d797126..2dd6721 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -98,6 +98,16 @@ public class BlockExecutionInfo {
private int[][] allSelectedMeasureBlocksIndexes;
/**
+ * list of dimension present in the projection
+ */
+ private int[] projectionListDimensionIndexes;
+
+ /**
+ * list of dimension present in the projection
+ */
+ private int[] projectionListMeasureIndexes;
+
+ /**
* this will be used to update the older block fixed length keys with the
* new block fixed length key
*/
@@ -599,4 +609,21 @@ public class BlockExecutionInfo {
public void setBlockId(String blockId) {
this.blockId = blockId;
}
+
+ public int[] getProjectionListDimensionIndexes() {
+ return projectionListDimensionIndexes;
+ }
+
+ public void setProjectionListDimensionIndexes(int[] projectionListDimensionIndexes) {
+ this.projectionListDimensionIndexes = projectionListDimensionIndexes;
+ }
+
+ public int[] getProjectionListMeasureIndexes() {
+ return projectionListMeasureIndexes;
+ }
+
+ public void setProjectionListMeasureIndexes(int[] projectionListMeasureIndexes) {
+ this.projectionListMeasureIndexes = projectionListMeasureIndexes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index a8338e7..ef6fb8a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -209,7 +209,7 @@ public class QueryUtil {
public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
Map<Integer, Integer> dimensionOrdinalToBlockMapping,
List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
- List<Integer> allProjectionListDimensionIndexes) {
+ Set<Integer> allProjectionListDimensionIndexes) {
// using set as in row group columns will point to same block
Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
@@ -218,7 +218,13 @@ public class QueryUtil {
if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
continue;
}
- allProjectionListDimensionIndexes.add(queryDimensions.get(i).getDimension().getOrdinal());
+
+ allProjectionListDimensionIndexes.add(
+ dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal()));
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ addChildrenBlockIndex(allProjectionListDimensionIndexes,
+ queryDimensions.get(i).getDimension());
+ }
if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
blockIndex =
@@ -394,10 +400,11 @@ public class QueryUtil {
*/
public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
- Set<CarbonMeasure> filterMeasures) {
+ Set<CarbonMeasure> filterMeasures, List<Integer> allProjectionListMeasureIdexes) {
Set<Integer> measureBlockIndex = new HashSet<Integer>();
Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
for (int i = 0; i < queryMeasures.size(); i++) {
+ allProjectionListMeasureIdexes.add(queryMeasures.get(i).getMeasure().getOrdinal());
if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) {
measureBlockIndex
.add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index e933f98..520b460 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -68,7 +68,6 @@ import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
-import org.apache.carbondata.core.scan.filter.executer.RestructureFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
@@ -125,10 +124,6 @@ public final class FilterUtil {
complexDimensionInfoMap),
createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
complexDimensionInfoMap));
- case RESTRUCTURE:
- return new RestructureFilterExecuterImpl(
- filterExpressionResolverTree.getDimColResolvedFilterInfo(),
- segmentProperties);
case ROWLEVEL_LESSTHAN:
case ROWLEVEL_LESSTHAN_EQUALTO:
case ROWLEVEL_GREATERTHAN_EQUALTO:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
index 4518149..3742e7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
@@ -21,7 +21,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.spark.sql.types.DataType;
@@ -40,8 +40,8 @@ public interface GenericQueryType {
int getColsCount();
- void parseBlocksAndReturnComplexColumnByteArray(DimensionColumnDataChunk[] dimensionDataChunks,
- int rowNumber, DataOutputStream dataOutputStream) throws IOException;
+ void parseBlocksAndReturnComplexColumnByteArray(DimensionRawColumnChunk[] rawColumnChunks,
+ int rowNumber, int pageNumber, DataOutputStream dataOutputStream) throws IOException;
DataType getSchemaType();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index 4356511..971f9b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -21,6 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
public class AndFilterExecuterImpl implements FilterExecuter {
@@ -32,13 +33,13 @@ public class AndFilterExecuterImpl implements FilterExecuter {
this.rightExecuter = rightExecuter;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
- BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
if (leftFilters.isEmpty()) {
return leftFilters;
}
- BitSet rightFilter = rightExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder);
if (rightFilter.isEmpty()) {
return rightFilter;
}
@@ -58,4 +59,9 @@ public class AndFilterExecuterImpl implements FilterExecuter {
leftFilters.and(rightFilter);
return leftFilters;
}
+
+ @Override public void readBlocks(BlocksChunkHolder blocksChunkHolder) throws IOException {
+ leftExecuter.readBlocks(blocksChunkHolder);
+ rightExecuter.readBlocks(blocksChunkHolder);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 667474d..8e7a3c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -21,11 +21,13 @@ import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.CarbonUtil;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
@@ -43,15 +45,26 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
+ DimensionRawColumnChunk dimensionRawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ DimensionColumnDataChunk[] dimensionColumnDataChunks =
+ dimensionRawColumnChunk.convertToDimColDataChunks();
+ BitSetGroup bitSetGroup =
+ new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
+ for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
+ BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
+ dimensionRawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+
+ return bitSetGroup;
}
protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
@@ -148,4 +161,13 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
bitSet.flip(0, 1);
return bitSet;
}
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index c476c0a..7182dd5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -21,6 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
public interface FilterExecuter {
@@ -30,7 +31,7 @@ public interface FilterExecuter {
* @return
* @throws FilterUnsupportedException
*/
- BitSet applyFilter(BlocksChunkHolder blocksChunkHolder)
+ BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder)
throws FilterUnsupportedException, IOException;
/**
@@ -42,4 +43,10 @@ public interface FilterExecuter {
* @return BitSet
*/
BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue);
+
+ /**
+ * It just reads necessary block for filter executor, it does not uncompress the data.
+ * @param blockChunkHolder
+ */
+ void readBlocks(BlocksChunkHolder blockChunkHolder)throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index c2a717f..e640d71 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -21,11 +21,13 @@ import java.util.BitSet;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -45,15 +47,31 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
+ DimensionRawColumnChunk dimensionRawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
+ for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
+ if (dimensionRawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
+ dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ dimensionRawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ } else {
+ BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
+ dimensionRawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
}
protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -149,16 +167,25 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ boolean isScanRequired =
+ isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues);
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+ private boolean isScanRequired(byte[] blkMaxVal, byte[] blkMinVal, byte[][] filterValues) {
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
// max>filtervalue>min
// so filter-max should be negative
int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMaxVal[blockIndex]);
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMaxVal);
// and filter-min should be positive
int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMinVal[blockIndex]);
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMinVal);
// if any filter value is in range than this block needs to be
// scanned
@@ -167,10 +194,15 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
break;
}
}
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
+ return isScanRequired;
}
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColumnEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 248bb22..119bda7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -21,6 +21,7 @@ import java.util.BitSet;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
public class OrFilterExecuterImpl implements FilterExecuter {
@@ -32,10 +33,10 @@ public class OrFilterExecuterImpl implements FilterExecuter {
this.rightExecuter = rightExecuter;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
- BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
- BitSet rightFilters = rightExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+ BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder);
leftFilters.or(rightFilters);
return leftFilters;
@@ -48,4 +49,8 @@ public class OrFilterExecuterImpl implements FilterExecuter {
return leftFilters;
}
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ leftExecuter.readBlocks(blockChunkHolder);
+ rightExecuter.readBlocks(blockChunkHolder);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
deleted file mode 100644
index 715c98d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.filter.executer;
-
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
-
-public class RestructureFilterExecuterImpl implements FilterExecuter {
-
- DimColumnExecuterFilterInfo dimColumnExecuterInfo;
-
- public RestructureFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil
- .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), segmentProperties,
- dimColumnResolvedFilterInfo.getDimension(), dimColumnExecuterInfo);
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blocksChunkHolder) {
- BitSet bitSet = new BitSet(blocksChunkHolder.getDataBlock().nodeSize());
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- if (null != filterValues && filterValues.length > 0) {
- bitSet.set(0, blocksChunkHolder.getDataBlock().nodeSize());
- }
- return bitSet;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- bitSet.set(0);
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8538209..7595ab6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -20,8 +20,8 @@ import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.Map;
@@ -31,6 +31,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -50,6 +52,7 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -90,56 +93,48 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
this.complexDimensionInfoMap = complexDimensionInfoMap;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
- for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
- if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
- && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
- if (null == blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]]) {
- blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
- }
- } else {
- GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
- complexType.fillRequiredBlockData(blockChunkHolder);
- }
- }
+ readBlocks(blockChunkHolder);
+ // CHECKSTYLE:ON
- // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_001
- if (null != msrColEvalutorInfoList) {
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo
- .getColumnIndex()]) {
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
- blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
- msrColumnEvalutorInfo.getColumnIndex());
+ int[] numberOfRows = null;
+ int pageNumbers = 0;
+
+ if (dimColEvaluatorInfoList.size() > 0) {
+ pageNumbers = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getPagesCount();
+ numberOfRows = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getRowCount();
+ }
+ if (msrColEvalutorInfoList.size() > 0) {
+ int columnIndex = msrColEvalutorInfoList.get(0).getColumnIndex();
+ pageNumbers = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getPagesCount();
+ numberOfRows = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getRowCount();
+ }
+ BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
+ for (int i = 0; i < pageNumbers; i++) {
+ BitSet set = new BitSet(numberOfRows[i]);
+ RowIntf row = new RowImpl();
+ boolean invalidRowsPresent = false;
+ for (int index = 0; index < numberOfRows[i]; index++) {
+ createRow(blockChunkHolder, row ,i, index);
+ Boolean rslt = false;
+ try {
+ rslt = exp.evaluate(row).getBoolean();
+ }
+ // Any invalid member while evaluation shall be ignored, system will log the
+ // error only once since all rows the evaluation happens so inorder to avoid
+ // too much log inforation only once the log will be printed.
+ catch (FilterIllegalMemberException e) {
+ FilterUtil.logError(e, invalidRowsPresent);
+ }
+ if (null != rslt && rslt) {
+ set.set(index);
}
}
+ bitSetGroup.setBitSet(set, i);
}
- // CHECKSTYLE:ON
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- BitSet set = new BitSet(numberOfRows);
- RowIntf row = new RowImpl();
- boolean invalidRowsPresent = false;
- for (int index = 0; index < numberOfRows; index++) {
- createRow(blockChunkHolder, row, index);
- Boolean rslt = false;
- try {
- rslt = exp.evaluate(row).getBoolean();
- }
- // Any invalid member while evaluation shall be ignored, system will log the
- // error only once since all rows the evaluation happens so inorder to avoid
- // too much log inforation only once the log will be printed.
- catch (FilterIllegalMemberException e) {
- FilterUtil.logError(e, invalidRowsPresent);
- }
- if (null != rslt && rslt) {
- set.set(index);
- }
- }
- return set;
+ return bitSetGroup;
}
/**
@@ -151,7 +146,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param index
* @throws IOException
*/
- private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int index)
+ private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int pageIndex, int index)
throws IOException {
Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
String memberString;
@@ -162,28 +157,28 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
}
+ DimensionColumnDataChunk columnDataChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]
+ .convertToDimColDataChunk(pageIndex);
if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && blockChunkHolder
- .getDimensionDataChunk()[blocksIndex[i]] instanceof VariableLengthDimensionDataChunk) {
+ && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
VariableLengthDimensionDataChunk dimensionColumnDataChunk =
- (VariableLengthDimensionDataChunk) blockChunkHolder
- .getDimensionDataChunk()[blocksIndex[i]];
- memberString = readMemberBasedOnNoDictionaryVal(dimensionColumnDataChunk, index);
- if (null != memberString) {
- if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- memberString = null;
+ (VariableLengthDimensionDataChunk) columnDataChunk;
+ byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
+ if (null != memberBytes) {
+ if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
+ memberBytes = null;
}
record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberString,
+ .getDataBasedOnDataType(memberBytes,
dimColumnEvaluatorInfo.getDimension().getDataType());
} else {
continue;
}
} else {
- int dictionaryValue =
- readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
- blocksIndex[i]);
+ int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex,
+ dimColumnEvaluatorInfo, blocksIndex[i]);
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
memberString =
@@ -204,9 +199,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
- complexType
- .parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionDataChunk(),
- index, dataOutputStream);
+ complexType.parseBlocksAndReturnComplexColumnByteArray(
+ blockChunkHolder.getDimensionRawDataChunk(), index, pageIndex, dataOutputStream);
record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
.getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
byteStream.close();
@@ -232,23 +226,25 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
// if measure doesnt exist then set the default value.
Object msrValue;
+ MeasureColumnDataChunk measureColumnDataChunk =
+ blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .convertToMeasureColDataChunk(pageIndex);
switch (msrType) {
case INT:
case LONG:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableLongValueByIndex(index);
+ msrValue =
+ measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
break;
case DECIMAL:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
+ msrValue = measureColumnDataChunk.getMeasureDataHolder()
+ .getReadableBigDecimalValueByIndex(index);
break;
default:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableDoubleValueByIndex(index);
+ msrValue =
+ measureColumnDataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
}
record[msrColumnEvalutorInfo.getRowIndex()] =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
+ measureColumnDataChunk.getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
}
row.setValues(record);
}
@@ -305,32 +301,32 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @param dimColumnEvaluatorInfo
* @return
*/
- private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index,
+ private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index, int page,
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ DimensionColumnDataChunk dataChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex].convertToDimColDataChunk(page);
if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
- byte[] rawData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ byte[] rawData = dataChunk.getChunkData(index);
ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
return dictionaryValue;
} else {
- return readSurrogatesFromColumnGroupBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
- blockIndex);
+ return readSurrogatesFromColumnGroupBlock(dataChunk, index, dimColumnEvaluatorInfo);
}
}
/**
- * @param blockChunkHolder
* @param index
* @param dimColumnEvaluatorInfo
* @return read surrogate of given row of given column group dimension
*/
- private int readSurrogatesFromColumnGroupBlock(BlocksChunkHolder blockChunkHolder, int index,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ private int readSurrogatesFromColumnGroupBlock(DimensionColumnDataChunk chunk, int index,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
try {
KeyStructureInfo keyStructureInfo =
QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo);
- byte[] colData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ byte[] colData = chunk.getChunkData(index);
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
@@ -343,24 +339,38 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
return 0;
}
- /**
- * Reading the blocks for no dictionary data, in no dictionary case
- * directly the filter data will read, no need to scan the dictionary
- * or read the dictionary value.
- *
- * @param dimensionColumnDataChunk
- * @param index
- * @return
- */
- private String readMemberBasedOnNoDictionaryVal(
- VariableLengthDimensionDataChunk dimensionColumnDataChunk, int index) {
- return new String(dimensionColumnDataChunk.getChunkData(index),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- }
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
bitSet.set(0);
return bitSet;
}
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+ if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
+ && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]] =
+ blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
+ }
+ } else {
+ GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+ complexType.fillRequiredBlockData(blockChunkHolder);
+ }
+ }
+
+ if (null != msrColEvalutorInfoList) {
+ for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+ if (null == blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo
+ .getColumnIndex()]) {
+ blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
+ blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
+ msrColumnEvalutorInfo.getColumnIndex());
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index cd60190..9f28d7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -30,6 +31,7 @@ import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedExc
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -50,13 +52,21 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangeValues;
int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues);
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
// max>filtervalue>min
// so filter-max should be negative
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
// if any filter value is in range than this block needs to be
// scanned means always less than block max range.
if (maxCompare < 0) {
@@ -64,26 +74,45 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
break;
}
}
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
+ return isScanRequired;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare < 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -118,8 +147,9 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
filterValues[i], true);
if (start >= 0) {
- start = CarbonUtil.nextGreaterValueToTarget(start, dimensionColumnDataChunk,
- filterValues[i], numerOfRows);
+ start = CarbonUtil
+ .nextGreaterValueToTarget(start, dimensionColumnDataChunk, filterValues[i],
+ numerOfRows);
}
// Logic will handle the case where the range filter member is not present in block
// in this case the binary search will return the index from where the bit sets will be
@@ -207,4 +237,15 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return bitSet;
}
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ super.readBlocks(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 02666c4..27377c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
@@ -30,6 +31,7 @@ import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedExc
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -51,13 +53,21 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangeValues;
int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues);
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// filter value should be in range of max and min value i.e
// max>filtervalue>min
// so filter-max should be negative
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
// if any filter value is in range than this block needs to be
// scanned less than equal to max range.
if (maxCompare <= 0) {
@@ -65,26 +75,45 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
break;
}
}
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
+ return isScanRequired;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMaxValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
+ if (compare <= 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -194,4 +223,16 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
}
return bitSet;
}
+
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ super.readBlocks(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/72cb415a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 82ca4d9..2bdce8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@ -33,6 +34,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -53,11 +55,18 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangeValues;
int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = isScanRequired(blockMinValue[columnIndex], filterValues);
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+ private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues) {
boolean isScanRequired = false;
for (int k = 0; k < filterValues.length; k++) {
// and filter-min should be positive
- int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue[columnIndex]);
+ int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue);
// if any filter applied is not in range of min and max of block
// then since its a less than equal to fiter validate whether the block
@@ -67,26 +76,45 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
break;
}
}
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
+ return isScanRequired;
}
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
return super.applyFilter(blockChunkHolder);
}
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
+ DimensionRawColumnChunk rawColumnChunk =
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex];
+ BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
+ for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
+ if (rawColumnChunk.getMinValues() != null) {
+ if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues)) {
+ int compare = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterRangeValues[0], rawColumnChunk.getMaxValues()[i]);
+ if (compare >= 0) {
+ BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
+ bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ } else {
+ BitSet bitSet = getFilteredIndexes(rawColumnChunk.convertToDimColDataChunk(i),
+ rawColumnChunk.getRowCount()[i]);
+ bitSetGroup.setBitSet(bitSet, i);
+ }
+ }
+ return bitSetGroup;
}
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
@@ -151,7 +179,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
filterValues[i], true);
if (start < 0) {
start = -(start + 1);
- if (start == numerOfRows) {
+ if (start >= numerOfRows) {
start = start - 1;
}
// Method will compare the tentative index value after binary search, this tentative
@@ -218,7 +246,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
filterValues[k], true);
if (start < 0) {
start = -(start + 1);
- if (start == numerOfRows) {
+ if (start >= numerOfRows) {
start = start - 1;
}
// Method will compare the tentative index value after binary search, this tentative
@@ -243,4 +271,15 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return bitSet;
}
+ @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ super.readBlocks(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
}