You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:21 UTC
[34/50] [abbrv] incubator-carbondata git commit: [Bug] Column Group
filter row level filter and Exclude filter not working(#775)
[Bug] Column Group filter row level filter and Exclude filter not working(#775)
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b5fc5188
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b5fc5188
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b5fc5188
Branch: refs/heads/master
Commit: b5fc5188ef198402fd302b5f0a2e96c7cf077097
Parents: a743b77
Author: ashokblend <as...@gmail.com>
Authored: Wed Jun 29 23:40:45 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Wed Jun 29 23:40:45 2016 +0530
----------------------------------------------------------------------
.../query/carbon/executor/util/QueryUtil.java | 40 ++++
.../executer/ColGroupFilterExecuterImpl.java | 209 -------------------
.../ExcludeColGroupFilterExecuterImpl.java | 137 ++++++++++++
.../executer/ExcludeFilterExecuterImpl.java | 41 ++--
.../IncludeColGroupFilterExecuterImpl.java | 209 +++++++++++++++++++
.../executer/RowLevelFilterExecuterImpl.java | 84 ++++++--
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 4 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 4 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 4 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 4 +-
.../filters/measurefilter/util/FilterUtil.java | 32 ++-
.../ColumnGroupDataTypesTestCase.scala | 18 ++
12 files changed, 518 insertions(+), 268 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index a7d8f05..f03219f 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -63,6 +63,7 @@ import org.carbondata.query.carbon.model.DimensionAggregatorInfo;
import org.carbondata.query.carbon.model.QueryDimension;
import org.carbondata.query.carbon.model.QueryMeasure;
import org.carbondata.query.carbon.model.QueryModel;
+import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.commons.lang3.ArrayUtils;
@@ -892,4 +893,43 @@ public class QueryUtil {
}
return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
}
+
+ /**
+ * It is required for extracting column data from columngroup chunk
+ *
+ * @return
+ * @throws KeyGenException
+ */
+ public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
+ int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+
+ mdKeyOrdinal.add(segmentProperties
+ .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMaskedBytes(maksedByte);
+ return restructureInfos;
+ }
+
+ public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java
deleted file mode 100644
index 98b3ed4..0000000
--- a/core/src/main/java/org/carbondata/query/filter/executer/ColGroupFilterExecuterImpl.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.query.filter.executer;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import org.carbondata.core.keygenerator.KeyGenException;
-import org.carbondata.core.keygenerator.KeyGenerator;
-import org.carbondata.core.util.ByteUtil;
-import org.carbondata.query.carbon.executor.infos.KeyStructureInfo;
-import org.carbondata.query.carbon.executor.util.QueryUtil;
-import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-/**
- * It checks if filter is required on given block and if required, it does
- * linear search on block data and set the bitset.
- */
-public class ColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl {
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ColGroupFilterExecuterImpl.class.getName());
-
- /**
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- */
- public ColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- super(dimColResolvedFilterInfo, segmentProperties);
- }
-
- /**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
-
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.set(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMaskedBytes(maksedByte);
- return restructureInfos;
- }
-
- /**
- * Check if scan is required on given block based on min and max value
- */
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue);
- // and filter-min should be positive
- int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue);
-
- // if any filter value is in range than this block needs to be
- // scanned
- if (maxCompare <= 0 && minCompare >= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
- }
-
- /**
- * It extract min and max data for given column from stored min max value
- *
- * @param colGrpColumns
- * @param minMaxData
- * @param columnIndex
- * @return
- */
- private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) {
- int startIndex = 0;
- int endIndex = 0;
- if (null != colGrpColumns) {
- for (int i = 0; i < colGrpColumns.length; i++) {
- int colGrpId = getColumnGroupId(colGrpColumns[i]);
- int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId);
- int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal);
- int colSize = 0;
- for (int j = byteRange[0]; j <= byteRange[1]; j++) {
- colSize++;
- }
- if (colGrpColumns[i] == columnIndex) {
- endIndex = startIndex + colSize;
- break;
- }
- startIndex += colSize;
- }
- }
- byte[] data = new byte[endIndex - startIndex];
- System.arraycopy(minMaxData, startIndex, data, 0, data.length);
- return data;
- }
-
- /**
- * It returns column groups which have provided column ordinal
- *
- * @param columnIndex
- * @return column group array
- */
- private int[] getAllColumns(int columnIndex) {
- int[][] colGroups = segmentProperties.getColumnGroups();
- for (int i = 0; i < colGroups.length; i++) {
- if (QueryUtil.searchInArray(colGroups[i], columnIndex)) {
- return colGroups[i];
- }
- }
- return null;
- }
-
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/ExcludeColGroupFilterExecuterImpl.java
new file mode 100644
index 0000000..0a73ca9
--- /dev/null
+++ b/core/src/main/java/org/carbondata/query/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.query.filter.executer;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.keygenerator.KeyGenerator;
+import org.carbondata.core.util.ByteUtil;
+import org.carbondata.query.carbon.executor.infos.KeyStructureInfo;
+import org.carbondata.query.carbon.executor.util.QueryUtil;
+import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+/**
+ * It checks if filter is required on given block and if required, it does
+ * linear search on block data and set the bitset.
+ */
+public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
+
+ /**
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ */
+ public ExcludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties) {
+ super(dimColResolvedFilterInfo, segmentProperties);
+ }
+
+ /**
+ * It fills BitSet with row index which matches filter key
+ */
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ try {
+ KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ for (int rowId = 0; rowId < numerOfRows; rowId++) {
+ byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
+ dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
+ bitSet.flip(rowId);
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * It is required for extracting column data from columngroup chunk
+ *
+ * @return
+ * @throws KeyGenException
+ */
+ private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
+ int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMaskedBytes(maksedByte);
+ return restructureInfos;
+ }
+
+ /**
+ * Check if scan is required on given block based on min and max value
+ */
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.flip(0, 1);
+ return bitSet;
+ }
+
+ private int getMdkeyOrdinal(int ordinal, int colGrpId) {
+ return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
+ }
+
+ private int getColumnGroupId(int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ public KeyGenerator getKeyGenerator(int colGrpId) {
+ return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/ExcludeFilterExecuterImpl.java
index f2c6de3..1f620c8 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/ExcludeFilterExecuterImpl.java
@@ -21,10 +21,10 @@ package org.carbondata.query.filter.executer;
import java.util.BitSet;
import java.util.List;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
import org.carbondata.core.carbon.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.carbondata.core.carbon.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.carbondata.core.keygenerator.KeyGenerator;
import org.carbondata.core.util.ByteUtil;
import org.carbondata.core.util.CarbonUtil;
import org.carbondata.query.carbon.processor.BlocksChunkHolder;
@@ -34,37 +34,38 @@ import org.carbondata.query.filters.measurefilter.util.FilterUtil;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
- DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- DimColumnExecuterFilterInfo dimColumnExecuterInfo;
-
- public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo) {
- this.dimColEvaluatorInfo = dimColEvaluatorInfo;
- }
+ protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ protected SegmentProperties segmentProperties;
public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- KeyGenerator blockKeyGenerator) {
- this(dimColEvaluatorInfo);
+ SegmentProperties segmentProperties) {
+ this.dimColEvaluatorInfo = dimColEvaluatorInfo;
dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), blockKeyGenerator,
- dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
+ this.segmentProperties = segmentProperties;
+ FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(),
+ segmentProperties.getDimensionKeyGenerator(), dimColEvaluatorInfo.getDimension(),
+ dimColumnExecuterInfo);
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) {
- if (null == blockChunkHolder.getDimensionDataChunk()[dimColEvaluatorInfo.getColumnIndex()]) {
- blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(),
- dimColEvaluatorInfo.getColumnIndex());
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- if (null == blockChunkHolder.getDimensionDataChunk()[dimColEvaluatorInfo.getColumnIndex()]) {
- blockChunkHolder.getDimensionDataChunk()[dimColEvaluatorInfo.getColumnIndex()] =
- blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(),
- dimColEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
return getFilteredIndexes(
- blockChunkHolder.getDimensionDataChunk()[dimColEvaluatorInfo.getColumnIndex()],
+ blockChunkHolder.getDimensionDataChunk()[blockIndex],
blockChunkHolder.getDataBlock().nodeSize());
}
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk, int numerOfRows) {
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
+ int numerOfRows) {
// For high cardinality dimensions.
if (dimColumnDataChunk.getAttributes().isNoDictionary()
&& dimColumnDataChunk instanceof VariableLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java
new file mode 100644
index 0000000..f230484
--- /dev/null
+++ b/core/src/main/java/org/carbondata/query/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.query.filter.executer;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.carbondata.common.logging.LogService;
+import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.carbondata.core.keygenerator.KeyGenException;
+import org.carbondata.core.keygenerator.KeyGenerator;
+import org.carbondata.core.util.ByteUtil;
+import org.carbondata.query.carbon.executor.infos.KeyStructureInfo;
+import org.carbondata.query.carbon.executor.util.QueryUtil;
+import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+/**
+ * It checks if filter is required on given block and if required, it does
+ * linear search on block data and set the bitset.
+ */
+public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl {
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(IncludeColGroupFilterExecuterImpl.class.getName());
+
+ /**
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ */
+ public IncludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties) {
+ super(dimColResolvedFilterInfo, segmentProperties);
+ }
+
+ /**
+ * It fills BitSet with row index which matches filter key
+ */
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+
+ try {
+ KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ for (int rowId = 0; rowId < numerOfRows; rowId++) {
+ byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
+ dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
+ bitSet.set(rowId);
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * It is required for extracting column data from columngroup chunk
+ *
+ * @return
+ * @throws KeyGenException
+ */
+ private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
+ int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ int[] maksedByte = QueryUtil.getMaskedByte(keyGenerator.getKeySizeInBytes(), maskByteRanges);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMaskedBytes(maksedByte);
+ return restructureInfos;
+ }
+
+ /**
+ * Check if scan is required on given block based on min and max value
+ */
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ int[] cols = getAllColumns(columnIndex);
+ byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
+ byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue);
+ // and filter-min should be positive
+ int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue);
+
+ // if any filter value is in range than this block needs to be
+ // scanned
+ if (maxCompare <= 0 && minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+ /**
+ * It extract min and max data for given column from stored min max value
+ *
+ * @param colGrpColumns
+ * @param minMaxData
+ * @param columnIndex
+ * @return
+ */
+ private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) {
+ int startIndex = 0;
+ int endIndex = 0;
+ if (null != colGrpColumns) {
+ for (int i = 0; i < colGrpColumns.length; i++) {
+ int colGrpId = getColumnGroupId(colGrpColumns[i]);
+ int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId);
+ int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal);
+ int colSize = 0;
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ colSize++;
+ }
+ if (colGrpColumns[i] == columnIndex) {
+ endIndex = startIndex + colSize;
+ break;
+ }
+ startIndex += colSize;
+ }
+ }
+ byte[] data = new byte[endIndex - startIndex];
+ System.arraycopy(minMaxData, startIndex, data, 0, data.length);
+ return data;
+ }
+
+ /**
+ * It returns column groups which have provided column ordinal
+ *
+ * @param columnIndex
+ * @return column group array
+ */
+ private int[] getAllColumns(int columnIndex) {
+ int[][] colGroups = segmentProperties.getColumnGroups();
+ for (int i = 0; i < colGroups.length; i++) {
+ if (QueryUtil.searchInArray(colGroups[i], columnIndex)) {
+ return colGroups[i];
+ }
+ }
+ return null;
+ }
+
+ private int getMdkeyOrdinal(int ordinal, int colGrpId) {
+ return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
+ }
+
+ private int getColumnGroupId(int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ public KeyGenerator getKeyGenerator(int colGrpId) {
+ return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelFilterExecuterImpl.java
index 095f757..f08936b 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelFilterExecuterImpl.java
@@ -28,16 +28,20 @@ import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.cache.dictionary.Dictionary;
import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.carbondata.core.carbon.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.carbondata.core.carbon.metadata.datatype.DataType;
import org.carbondata.core.carbon.metadata.encoder.Encoding;
import org.carbondata.core.constants.CarbonCommonConstants;
+import org.carbondata.core.keygenerator.KeyGenException;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.carbondata.core.util.CarbonUtil;
import org.carbondata.query.aggregator.MeasureAggregator;
import org.carbondata.query.aggregator.util.MeasureAggregatorFactory;
import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
+import org.carbondata.query.carbon.executor.infos.KeyStructureInfo;
+import org.carbondata.query.carbon.executor.util.QueryUtil;
import org.carbondata.query.carbon.processor.BlocksChunkHolder;
import org.carbondata.query.carbon.util.DataTypeUtil;
import org.carbondata.query.carbonfilterinterface.RowImpl;
@@ -58,11 +62,22 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
protected List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
protected Expression exp;
protected AbsoluteTableIdentifier tableIdentifier;
+ protected SegmentProperties segmentProperties;
+ /**
+ * it has index at which given dimension is stored in file
+ */
+ private int[] blocksIndex;
public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier) {
+ AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties) {
this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
+ this.segmentProperties = segmentProperties;
+ this.blocksIndex = new int[dimColEvaluatorInfoList.size()];
+ for (int i=0;i<dimColEvaluatorInfoList.size();i++) {
+ this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(i).getColumnIndex());
+ }
if (null == msrColEvalutorInfoList) {
this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
} else {
@@ -74,18 +89,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException {
- for (DimColumnResolvedFilterInfo dimColumnEvaluatorInfo : dimColEvaluatorInfoList) {
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
&& dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
- if (null == blockChunkHolder.getDimensionDataChunk()[dimColumnEvaluatorInfo
- .getColumnIndex()]) {
- blockChunkHolder.getDimensionDataChunk()[dimColumnEvaluatorInfo.getColumnIndex()] =
- blockChunkHolder.getDataBlock().getDimensionChunk(blockChunkHolder.getFileReader(),
- dimColumnEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]]) {
+ blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
}
} else {
GenericQueryType complexType = dimColumnEvaluatorInfo.getComplexTypesWithBlockStartIndex()
- .get(dimColumnEvaluatorInfo.getColumnIndex());
+ .get(blocksIndex[i]);
complexType.fillRequiredBlockData(blockChunkHolder);
}
}
@@ -143,19 +157,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
throws QueryExecutionException {
Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
String memberString = null;
- for (DimColumnResolvedFilterInfo dimColumnEvaluatorInfo : dimColEvaluatorInfoList) {
+ for (int i=0;i<dimColEvaluatorInfoList.size();i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
&& dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
}
if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && blockChunkHolder.getDimensionDataChunk()[dimColumnEvaluatorInfo
- .getColumnIndex()] instanceof VariableLengthDimensionDataChunk) {
+ && blockChunkHolder
+ .getDimensionDataChunk()[blocksIndex[i]] instanceof VariableLengthDimensionDataChunk) {
VariableLengthDimensionDataChunk dimensionColumnDataChunk =
(VariableLengthDimensionDataChunk) blockChunkHolder
- .getDimensionDataChunk()[dimColumnEvaluatorInfo.getColumnIndex()];
+ .getDimensionDataChunk()[blocksIndex[i]];
if (null != dimensionColumnDataChunk.getCompleteDataChunk()) {
memberString =
readMemberBasedOnNoDictionaryVal(dimColumnEvaluatorInfo, dimensionColumnDataChunk,
@@ -173,7 +188,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
}
} else {
int dictionaryValue =
- readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo);
+ readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
+ blocksIndex[i]);
Dictionary forwardDictionary = null;
if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
&& !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -317,13 +333,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
* @return
*/
private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
- byte[] rawData =
- blockChunkHolder.getDimensionDataChunk()[dimColumnEvaluatorInfo.getColumnIndex()]
- .getChunkData(index);
- ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
- int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
- return dictionaryValue;
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
+ byte[] rawData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
+ return dictionaryValue;
+ } else {
+ return readSurrogatesFromColumnGroupBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
+ blockIndex);
+ }
+
+ }
+
+ /**
+ * @param blockChunkHolder
+ * @param index
+ * @param dimColumnEvaluatorInfo
+ * @return read surrogate of given row of given column group dimension
+ */
+ private int readSurrogatesFromColumnGroupBlock(BlocksChunkHolder blockChunkHolder, int index,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ try {
+ KeyStructureInfo keyStructureInfo =
+ QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo);
+ byte[] colData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
+ int colGroupId =
+ QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ int dictionaryValue = (int) result[segmentProperties
+ .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())];
+ return dictionaryValue;
+ } catch (KeyGenException e) {
+ LOGGER.error(e);
+ }
+ return 0;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 571f046..0efca00 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -36,16 +36,14 @@ import org.carbondata.query.filter.resolver.resolverinfo.MeasureColumnResolvedFi
public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
- private SegmentProperties segmentProperties;
public RowLevelRangeGrtThanFiterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier);
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties);
this.filterRangeValues = filterRangeValues;
- this.segmentProperties = segmentProperties;
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index e390b8d..935c2ce 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -37,16 +37,14 @@ import org.carbondata.query.filter.resolver.resolverinfo.MeasureColumnResolvedFi
public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
protected byte[][] filterRangeValues;
- private SegmentProperties segmentProperties;
public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier);
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties);
this.filterRangeValues = filterRangeValues;
- this.segmentProperties = segmentProperties;
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 1f33c8c..27f7935 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -36,16 +36,14 @@ import org.carbondata.query.filter.resolver.resolverinfo.MeasureColumnResolvedFi
public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
protected byte[][] filterRangeValues;
- private SegmentProperties segmentProperties;
public RowLevelRangeLessThanEqualFilterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier);
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties);
this.filterRangeValues = filterRangeValues;
- this.segmentProperties = segmentProperties;
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 9bb0420..1786553 100644
--- a/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -36,16 +36,14 @@ import org.carbondata.query.filter.resolver.resolverinfo.MeasureColumnResolvedFi
public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
- private SegmentProperties segmentProperties;
public RowLevelRangeLessThanFiterExecuterImpl(
List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier);
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties);
this.filterRangeValues = filterRangeValues;
- this.segmentProperties = segmentProperties;
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java b/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
index ee72f62..8bae3fc 100644
--- a/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
+++ b/core/src/main/java/org/carbondata/query/filters/measurefilter/util/FilterUtil.java
@@ -73,9 +73,10 @@ import org.carbondata.query.expression.LiteralExpression;
import org.carbondata.query.expression.exception.FilterIllegalMemberException;
import org.carbondata.query.expression.exception.FilterUnsupportedException;
import org.carbondata.query.filter.executer.AndFilterExecuterImpl;
-import org.carbondata.query.filter.executer.ColGroupFilterExecuterImpl;
+import org.carbondata.query.filter.executer.ExcludeColGroupFilterExecuterImpl;
import org.carbondata.query.filter.executer.ExcludeFilterExecuterImpl;
import org.carbondata.query.filter.executer.FilterExecuter;
+import org.carbondata.query.filter.executer.IncludeColGroupFilterExecuterImpl;
import org.carbondata.query.filter.executer.IncludeFilterExecuterImpl;
import org.carbondata.query.filter.executer.OrFilterExecuterImpl;
import org.carbondata.query.filter.executer.RestructureFilterExecuterImpl;
@@ -113,9 +114,8 @@ public final class FilterUtil {
return getIncludeFilterExecuter(
filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
case EXCLUDE:
- return new ExcludeFilterExecuterImpl(
- filterExpressionResolverTree.getDimColResolvedFilterInfo(),
- segmentProperties.getDimensionKeyGenerator());
+ return getExcludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
case OR:
return new OrFilterExecuterImpl(
createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties),
@@ -143,7 +143,8 @@ public final class FilterUtil {
((RowLevelFilterResolverImpl) filterExpressionResolverTree)
.getMsrColEvalutorInfoList(),
((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier());
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties);
}
}
@@ -151,7 +152,8 @@ public final class FilterUtil {
((RowLevelFilterResolverImpl) filterExpressionResolverTree).getDimColEvaluatorInfoList(),
((RowLevelFilterResolverImpl) filterExpressionResolverTree).getMsrColEvalutorInfoList(),
((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier());
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties);
}
@@ -168,11 +170,27 @@ public final class FilterUtil {
if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
} else {
- return new ColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
}
}
/**
+ * It gives filter executer based on columnar or column group
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return
+ */
+ private static FilterExecuter getExcludeFilterExecuter(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
+
+ if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+ return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ } else {
+ return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ }
+ }
+ /**
* This method will check if a given expression contains a column expression
* recursively.
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b5fc5188/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
index 57f296a..8098308 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
@@ -104,6 +104,24 @@ class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
sql("select * from colgrp_dictexclude_after where column3='column311' and column4='column42' "),
sql("select * from normal where column3='column311' and column4='column42'"))
}
+ test("ExcludeFilter") {
+ checkAnswer(
+ sql("select * from colgrp where column3 != 'column311'"),
+ sql("select * from normal where column3 != 'column311'"))
+
+ checkAnswer(
+ sql("select * from colgrp where column3 like 'column31%'"),
+ sql("select * from normal where column3 like 'column31%'"))
+ checkAnswer(
+ sql("select * from colgrp where column3 not like 'column31%'"),
+ sql("select * from normal where column3 not like 'column31%'"))
+ }
+ test("RowFilter") {
+ checkAnswer(
+ sql("select * from colgrp where column3 != column4"),
+ sql("select * from normal where column3 != column4"))
+ }
+
override def afterAll {
sql("drop table colgrp")
sql("drop table normal")