You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by vi...@apache.org on 2016/08/31 02:20:01 UTC
[1/2] incubator-carbondata git commit: Fiex Data Mismatch issue in
filter query
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 523d0132a -> be642c972
Fiex Data Mismatch issue in filter query
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fe56ad0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fe56ad0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fe56ad0d
Branch: refs/heads/master
Commit: fe56ad0dd1c30008b032af7796b23ebb0628d1aa
Parents: 523d013
Author: kumarvishal <ku...@gmail.com>
Authored: Mon Aug 29 18:25:46 2016 +0530
Committer: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 07:43:52 2016 +0530
----------------------------------------------------------------------
.../carbondata/scan/filter/FilterUtil.java | 31 +++++++++++++++-----
.../executer/ExcludeFilterExecuterImpl.java | 8 ++---
.../executer/IncludeFilterExecuterImpl.java | 6 ++--
.../executer/RestructureFilterExecuterImpl.java | 7 ++---
.../RowLevelRangeFilterResolverImpl.java | 2 +-
5 files changed, 33 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
index a96b0b2..71ac1bf 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
@@ -111,7 +111,7 @@ public final class FilterUtil {
case RESTRUCTURE:
return new RestructureFilterExecuterImpl(
filterExpressionResolverTree.getDimColResolvedFilterInfo(),
- segmentProperties.getDimensionKeyGenerator());
+ segmentProperties);
case ROWLEVEL_LESSTHAN:
case ROWLEVEL_LESSTHAN_EQUALTO:
case ROWLEVEL_GREATERTHAN_EQUALTO:
@@ -609,12 +609,23 @@ public final class FilterUtil {
return columnFilterInfo;
}
+ /**
+ * Below method will be used to covert the filter surrogate keys
+ * to mdkey
+ *
+ * @param dimColumnFilterInfo
+ * @param carbonDimension
+ * @param segmentProperties
+ * @return
+ */
public static byte[][] getKeyArray(DimColumnFilterInfo dimColumnFilterInfo,
- CarbonDimension carbonDimension, KeyGenerator blockLevelKeyGenerator) {
+ CarbonDimension carbonDimension, SegmentProperties segmentProperties) {
if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
return dimColumnFilterInfo.getNoDictionaryFilterValuesList()
.toArray((new byte[dimColumnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
}
+ KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
+ int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
Arrays.fill(keys, 0);
@@ -623,9 +634,13 @@ public final class FilterUtil {
if (null != dimColumnFilterInfo) {
for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
try {
- keys[carbonDimension.getKeyOrdinal()] = surrogate;
- filterValuesList
- .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+ if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
+ keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ filterValuesList
+ .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+ } else {
+ break;
+ }
} catch (KeyGenException e) {
LOGGER.error(e.getMessage());
}
@@ -979,14 +994,14 @@ public final class FilterUtil {
* API will prepare the Keys from the surrogates of particular filter resolver
*
* @param filterValues
- * @param blockKeyGenerator
+ * @param segmentProperties
* @param dimension
* @param dimColumnExecuterInfo
*/
public static void prepareKeysFromSurrogates(DimColumnFilterInfo filterValues,
- KeyGenerator blockKeyGenerator, CarbonDimension dimension,
+ SegmentProperties segmentProperties, CarbonDimension dimension,
DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
- byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, blockKeyGenerator);
+ byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties);
dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 5ebae79..db4326b 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -42,9 +42,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
this.dimColEvaluatorInfo = dimColEvaluatorInfo;
dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
this.segmentProperties = segmentProperties;
- FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(),
- segmentProperties.getDimensionKeyGenerator(), dimColEvaluatorInfo.getDimension(),
- dimColumnExecuterInfo);
+ FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,
+ dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
}
@Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) {
@@ -58,8 +57,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- return getFilteredIndexes(
- blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
blockChunkHolder.getDataBlock().nodeSize());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
index 8e645b3..f399aa9 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -42,9 +42,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
this.segmentProperties = segmentProperties;
dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
- segmentProperties.getDimensionKeyGenerator(), dimColumnEvaluatorInfo.getDimension(),
- dimColumnExecuterInfo);
+ FilterUtil
+ .prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(), segmentProperties,
+ dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
index 84d7898..343fd03 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
@@ -20,21 +20,20 @@ package org.apache.carbondata.scan.filter.executer;
import java.util.BitSet;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.scan.filter.FilterUtil;
import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
public class RestructureFilterExecuterImpl implements FilterExecuter {
DimColumnExecuterFilterInfo dimColumnExecuterInfo;
public RestructureFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo,
- KeyGenerator blockKeyGenerator) {
+ SegmentProperties segmentProperties) {
dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
FilterUtil
- .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), blockKeyGenerator,
+ .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), segmentProperties,
dimColumnResolvedFilterInfo.getDimension(), dimColumnExecuterInfo);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fe56ad0d/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 0f9b47d..f07fe98 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -88,7 +88,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
this.dimColEvaluatorInfoList.get(0).getDimension(),
- segmentProperties.getDimensionKeyGenerator());
+ segmentProperties);
}
return null;
[2/2] incubator-carbondata git commit: [CARBONDATA-160]Data mismatch
issue in case of multiple segment with different dictionary key size. This
closes #106
Posted by vi...@apache.org.
[CARBONDATA-160]Data mismatch issue in case of multiple segment with different dictionary key size. This closes #106
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/be642c97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/be642c97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/be642c97
Branch: refs/heads/master
Commit: be642c972a15b6560562e31c81cdf9c9fd826cb8
Parents: 523d013 fe56ad0
Author: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Authored: Wed Aug 31 07:49:00 2016 +0530
Committer: Vimal Das Kammath <vk...@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 07:49:00 2016 +0530
----------------------------------------------------------------------
.../carbondata/scan/filter/FilterUtil.java | 31 +++++++++++++++-----
.../executer/ExcludeFilterExecuterImpl.java | 8 ++---
.../executer/IncludeFilterExecuterImpl.java | 6 ++--
.../executer/RestructureFilterExecuterImpl.java | 7 ++---
.../RowLevelRangeFilterResolverImpl.java | 2 +-
5 files changed, 33 insertions(+), 21 deletions(-)
----------------------------------------------------------------------