You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/09/18 13:38:17 UTC
[3/4] carbondata git commit: [CARBONDATA-2896][Refactor] Adaptive
Encoding for Primitive data types
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/ComplexDimensionIndexCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/ComplexDimensionIndexCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/ComplexDimensionIndexCodec.java
index cc044cc..f232652 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/ComplexDimensionIndexCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/ComplexDimensionIndexCodec.java
@@ -21,8 +21,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorage;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -46,7 +46,7 @@ public class ComplexDimensionIndexCodec extends IndexStorageCodec {
return new IndexStorageEncoder() {
@Override
void encodeIndexStorage(ColumnPage inputPage) {
- IndexStorage indexStorage =
+ BlockIndexerStorage<byte[][]> indexStorage =
new BlockIndexerStorageForShort(inputPage.getByteArrayPage(), false, false, false);
byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage());
Compressor compressor = CompressorFactory.getInstance().getCompressor(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DictDimensionIndexCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DictDimensionIndexCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DictDimensionIndexCodec.java
index 66f5f1d..f3475fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DictDimensionIndexCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DictDimensionIndexCodec.java
@@ -21,9 +21,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorage;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -47,7 +47,7 @@ public class DictDimensionIndexCodec extends IndexStorageCodec {
return new IndexStorageEncoder() {
@Override
void encodeIndexStorage(ColumnPage inputPage) {
- IndexStorage indexStorage;
+ BlockIndexerStorage<byte[][]> indexStorage;
byte[][] data = inputPage.getByteArrayPage();
if (isInvertedIndex) {
indexStorage = new BlockIndexerStorageForShort(data, true, false, isSort);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DirectDictDimensionIndexCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DirectDictDimensionIndexCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DirectDictDimensionIndexCodec.java
index a130cbd..15827f8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DirectDictDimensionIndexCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/DirectDictDimensionIndexCodec.java
@@ -21,9 +21,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorage;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -47,7 +47,7 @@ public class DirectDictDimensionIndexCodec extends IndexStorageCodec {
return new IndexStorageEncoder() {
@Override
void encodeIndexStorage(ColumnPage inputPage) {
- IndexStorage indexStorage;
+ BlockIndexerStorage<byte[][]> indexStorage;
byte[][] data = inputPage.getByteArrayPage();
if (isInvertedIndex) {
indexStorage = new BlockIndexerStorageForShort(data, false, false, isSort);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
index bce8523..7a1627c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/HighCardDictDimensionIndexCodec.java
@@ -21,9 +21,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorage;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndexForShort;
import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForShort;
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -54,7 +54,7 @@ public class HighCardDictDimensionIndexCodec extends IndexStorageCodec {
@Override
protected void encodeIndexStorage(ColumnPage input) {
- IndexStorage indexStorage;
+ BlockIndexerStorage<byte[][]> indexStorage;
byte[][] data = input.getByteArrayPage();
boolean isDictionary = input.isLocalDictGeneratedPage();
if (isInvertedIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
index 37d9052..96fcc08 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/dimension/legacy/IndexStorageEncoder.java
@@ -22,7 +22,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorage;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
@@ -31,7 +31,7 @@ import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.SortState;
public abstract class IndexStorageEncoder extends ColumnPageEncoder {
- IndexStorage indexStorage;
+ BlockIndexerStorage indexStorage;
byte[] compressedDataPage;
abstract void encodeIndexStorage(ColumnPage inputPage);
@@ -61,7 +61,9 @@ public abstract class IndexStorageEncoder extends ColumnPageEncoder {
out.writeShort(dataRle);
}
}
- return stream.toByteArray();
+ byte[] result = stream.toByteArray();
+ stream.close();
+ return result;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index c2b722b..609f17e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -61,7 +61,8 @@ public class TablePageKey {
/** update all keys based on the input row */
public void update(int rowId, CarbonRow row, byte[] mdk) {
if (hasNoDictionary) {
- currentNoDictionaryKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+ Object[] noDictAndComplexDimension = WriteStepRowUtil.getNoDictAndComplexDimension(row);
+ currentNoDictionaryKey = new byte[noDictAndComplexDimension.length][0];
}
if (rowId == 0) {
startKey = mdk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/TablePageStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/TablePageStatistics.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/TablePageStatistics.java
index 665860b..29a180b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/TablePageStatistics.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/TablePageStatistics.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastore.page.statistics;
import org.apache.carbondata.core.datastore.page.encoding.EncodedColumnPage;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
// Statistics of dimension and measure column in a TablePage
public class TablePageStatistics {
@@ -58,8 +59,17 @@ public class TablePageStatistics {
private void updateDimensionMinMax(EncodedColumnPage[] dimensions) {
for (int i = 0; i < dimensions.length; i++) {
SimpleStatsResult stats = dimensions[i].getStats();
- dimensionMaxValue[i] = CarbonUtil.getValueAsBytes(stats.getDataType(), stats.getMax());
- dimensionMinValue[i] = CarbonUtil.getValueAsBytes(stats.getDataType(), stats.getMin());
+ Object min = stats.getMin();
+ Object max = stats.getMax();
+ if (CarbonUtil.isEncodedWithMeta(dimensions[i].getPageMetadata().getEncoders())) {
+ dimensionMaxValue[i] = DataTypeUtil
+ .getMinMaxBytesBasedOnDataTypeForNoDictionaryColumn(max, stats.getDataType());
+ dimensionMinValue[i] = DataTypeUtil
+ .getMinMaxBytesBasedOnDataTypeForNoDictionaryColumn(min, stats.getDataType());
+ } else {
+ dimensionMaxValue[i] = CarbonUtil.getValueAsBytes(stats.getDataType(), max);
+ dimensionMinValue[i] = CarbonUtil.getValueAsBytes(stats.getDataType(), min);
+ }
writeMinMaxForDimensions[i] = stats.writeMinMax();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
index 08dd800..3d9de56 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/WriteStepRowUtil.java
@@ -20,7 +20,9 @@ package org.apache.carbondata.core.datastore.row;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.core.util.DataTypeUtil;
// Utility to create and retrieve data from CarbonRow in write step.
public class WriteStepRowUtil {
@@ -34,7 +36,7 @@ public class WriteStepRowUtil {
public static final int NO_DICTIONARY_AND_COMPLEX = 1;
public static final int MEASURE = 2;
- public static CarbonRow fromColumnCategory(int[] dictDimensions, byte[][] noDictAndComplex,
+ public static CarbonRow fromColumnCategory(int[] dictDimensions, Object[] noDictAndComplex,
Object[] measures) {
Object[] row = new Object[3];
row[DICTIONARY_DIMENSION] = dictDimensions;
@@ -43,7 +45,8 @@ public class WriteStepRowUtil {
return new CarbonRow(row);
}
- public static CarbonRow fromMergerRow(Object[] row, SegmentProperties segmentProperties) {
+ public static CarbonRow fromMergerRow(Object[] row, SegmentProperties segmentProperties,
+ CarbonColumn[] noDicAndComplexColumns) {
Object[] converted = new Object[3];
// dictionary dimension
@@ -55,8 +58,23 @@ public class WriteStepRowUtil {
}
converted[DICTIONARY_DIMENSION] = dictDimensions;
+ byte[][] noDictionaryKeys = ((ByteArrayWrapper) row[0]).getNoDictionaryKeys();
+ Object[] noDictKeys = new Object[noDictionaryKeys.length];
+ for (int i = 0; i < noDictionaryKeys.length; i++) {
+ // in case of compaction rows are collected from result collector and are in byte[].
+ // Convert the no dictionary columns to original data,
+ // as load expects the no dictionary column with original data.
+ if (DataTypeUtil.isPrimitiveColumn(noDicAndComplexColumns[i].getDataType())) {
+ noDictKeys[i] = DataTypeUtil
+ .getDataBasedOnDataTypeForNoDictionaryColumn(noDictionaryKeys[i],
+ noDicAndComplexColumns[i].getDataType());
+ } else {
+ noDictKeys[i] = noDictionaryKeys[i];
+ }
+ }
+
// no dictionary and complex dimension
- converted[NO_DICTIONARY_AND_COMPLEX] = ((ByteArrayWrapper) row[0]).getNoDictionaryKeys();
+ converted[NO_DICTIONARY_AND_COMPLEX] = noDictKeys;
// measure
int measureCount = row.length - 1;
@@ -75,8 +93,8 @@ public class WriteStepRowUtil {
return keyGenerator.generateKey(getDictDimension(row));
}
- public static byte[][] getNoDictAndComplexDimension(CarbonRow row) {
- return (byte[][]) row.getData()[NO_DICTIONARY_AND_COMPLEX];
+ public static Object[] getNoDictAndComplexDimension(CarbonRow row) {
+ return (Object[]) row.getData()[NO_DICTIONARY_AND_COMPLEX];
}
public static Object[] getMeasure(CarbonRow row) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 7cc2b09..219e7a2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -58,7 +58,10 @@ import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.model.ProjectionDimension;
import org.apache.carbondata.core.scan.model.ProjectionMeasure;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.commons.lang3.ArrayUtils;
@@ -745,4 +748,37 @@ public class QueryUtil {
}
}
}
+
+ /**
+ * Put the data to vector
+ *
+ * @param vector
+ * @param value
+ * @param vectorRow
+ * @param length
+ */
+ public static void putDataToVector(CarbonColumnVector vector, byte[] value, int vectorRow,
+ int length) {
+ DataType dt = vector.getType();
+ if ((!(dt == DataTypes.STRING) && length == 0) || ByteUtil.UnsafeComparer.INSTANCE
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, 0,
+ CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, value, 0, length)) {
+ vector.putNull(vectorRow);
+ } else {
+ if (dt == DataTypes.STRING) {
+ vector.putBytes(vectorRow, 0, length, value);
+ } else if (dt == DataTypes.BOOLEAN) {
+ vector.putBoolean(vectorRow, ByteUtil.toBoolean(value[0]));
+ } else if (dt == DataTypes.SHORT) {
+ vector.putShort(vectorRow, ByteUtil.toXorShort(value, 0, length));
+ } else if (dt == DataTypes.INT) {
+ vector.putInt(vectorRow, ByteUtil.toXorInt(value, 0, length));
+ } else if (dt == DataTypes.LONG) {
+ vector.putLong(vectorRow, DataTypeUtil
+ .getDataBasedOnRestructuredDataType(value, vector.getBlockDataType(), 0, length));
+ } else if (dt == DataTypes.TIMESTAMP) {
+ vector.putLong(vectorRow, ByteUtil.toXorLong(value, 0, length) * 1000L);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 410f67a..8c0ea56 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -645,8 +645,9 @@ public final class FilterUtil {
continue;
}
- filterValuesList
- .add(DataTypeUtil.getMeasureValueBasedOnDataType(result, dataType, carbonMeasure));
+ filterValuesList.add(DataTypeUtil
+ .getMeasureValueBasedOnDataType(result, dataType, carbonMeasure.getScale(),
+ carbonMeasure.getPrecision()));
}
} catch (Throwable ex) {
@@ -2178,4 +2179,41 @@ public final class FilterUtil {
}
return filterExecuter;
}
+
+ /**
+ * This method is used to compare the filter value with min and max values.
+ * This is used in case of filter queries on no dictionary column.
+ *
+ * @param filterValue
+ * @param minMaxBytes
+ * @param carbonDimension
+ * @param isMin
+ * @return
+ */
+ public static int compareValues(byte[] filterValue, byte[] minMaxBytes,
+ CarbonDimension carbonDimension, boolean isMin) {
+ DataType dataType = carbonDimension.getDataType();
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !carbonDimension
+ .hasEncoding(Encoding.DICTIONARY)) {
+ Object value =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(minMaxBytes, dataType);
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValue, dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ if (isMin) {
+ return comparator.compare(value, data);
+ } else {
+ return comparator.compare(data, value);
+ }
+ } else {
+ if (isMin) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(minMaxBytes, filterValue);
+ } else {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValue, minMaxBytes);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 7017f21..04264f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -397,7 +397,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return bitSet;
}
// binary search can only be applied if column is sorted
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int startIndex = 0;
for (int i = 0; i < filterValues.length; i++) {
if (startIndex >= numerOfRows) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index b9df60c..ddc9751 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.filter.executer;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
@@ -26,6 +27,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -110,9 +112,20 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
boolean isDecoded = false;
for (int i = 0; i < dimensionRawColumnChunk.getPagesCount(); i++) {
if (dimensionRawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
+ boolean scanRequired;
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
+ && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
+ dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys(),
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else {
+ scanRequired = isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys(),
- dimensionRawColumnChunk.getMinMaxFlagArray()[i])) {
+ dimensionRawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ if (scanRequired) {
DimensionColumnPage dimensionColumnPage = dimensionRawColumnChunk.decodeColumnPage(i);
if (!isDecoded) {
filterValues = FilterUtil
@@ -413,7 +426,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
// binary search can only be applied if column is sorted and
// inverted index exists for that column
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int startIndex = 0;
for (int i = 0; i < filterValues.length; i++) {
if (startIndex >= numerOfRows) {
@@ -459,8 +472,17 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
if (isDimensionPresentInCurrentBlock) {
filterValues = dimColumnExecuterInfo.getFilterKeys();
chunkIndex = dimColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray();
- isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues,
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil
+ .isPrimitiveColumn(dimColumnEvaluatorInfo.getDimension().getDataType())
+ && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else {
+ isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex], filterValues,
isMinMaxSet[chunkIndex]);
+ }
} else if (isMeasurePresentInCurrentBlock) {
chunkIndex = msrColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray();
isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
@@ -501,6 +523,34 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return isScanRequired;
}
+ private boolean isScanRequired(byte[] blkMaxVal, byte[] blkMinVal, byte[][] filterValues,
+ DataType dataType) {
+ boolean isScanRequired = false;
+ Object minValue = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blkMinVal, dataType);
+ Object maxValue = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blkMaxVal, dataType);
+ for (int k = 0; k < filterValues.length; k++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) {
+ return true;
+ }
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ int maxCompare = comparator.compare(data, maxValue);
+ int minCompare = comparator.compare(data, minValue);
+ // if any filter value is in range than this block needs to be
+ // scanned
+ if (maxCompare <= 0 && minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ return isScanRequired;
+ }
+
private boolean isScanRequired(byte[] maxValue, byte[] minValue, Object[] filterValue,
DataType dataType) {
Object maxObject = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 7a0a386..b9729db 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -121,13 +121,13 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter {
CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterRangesValues[0]);
int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangesValues[1], defaultValue);
+ FilterUtil.compareValues(filterRangesValues[0], defaultValue, dimension, true);
+ int maxCompare =
+ FilterUtil.compareValues(filterRangesValues[1], defaultValue, dimension, false);
- if (((greaterThanExp && maxCompare > 0) || (greaterThanEqualExp && maxCompare >= 0))
- && ((lessThanExp && minCompare > 0) || (lessThanEqualExp && minCompare >= 0))) {
+ if (((greaterThanExp && maxCompare > 0) || (greaterThanEqualExp && maxCompare >= 0)) && (
+ (lessThanExp && minCompare > 0) || (lessThanEqualExp && minCompare >= 0))) {
isDefaultValuePresentInFilter = true;
}
}
@@ -275,30 +275,35 @@ public class RangeValueFilterExecuterImpl implements FilterExecuter {
// Case D: Filter Values Completely overlaps Block Min and Max then all bits are set.
// Block Min <-----------------------> Block Max
// Filter Min <-----------------------------------------------> Filter Max
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
if (isDimensionPresentInCurrentBlock) {
+ CarbonDimension carbonDimension = dimColEvaluatorInfo.getDimension();
if (((lessThanExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) >= 0)) || (
- (lessThanEqualExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[1]) > 0)) || (
+ FilterUtil.compareValues(filterValues[1], blockMinValue, carbonDimension, true) >= 0))
+ || ((lessThanEqualExp) && (
+ FilterUtil.compareValues(filterValues[1], blockMinValue, carbonDimension, true) > 0)) || (
(greaterThanExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) >= 0)) || (
- (greaterThanEqualExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[0], blockMaxValue) > 0))) {
+ FilterUtil.compareValues(filterValues[0], blockMaxValue, carbonDimension, false)
+ >= 0)) || ((greaterThanEqualExp) && (
+ FilterUtil.compareValues(filterValues[0], blockMaxValue, carbonDimension, false) > 0))) {
// completely out of block boundary
isScanRequired = false;
} else {
if (((greaterThanExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) > 0)) || (
- (greaterThanEqualExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(blockMinValue, filterValues[0]) >= 0))) {
+ FilterUtil.compareValues(filterValues[0], blockMinValue, carbonDimension, true) > 0))
+ || ((greaterThanEqualExp) && (
+ FilterUtil.compareValues(filterValues[0], blockMinValue, carbonDimension, true)
+ >= 0))) {
startBlockMinIsDefaultStart = true;
}
if (((lessThanExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) > 0)) || (
- (lessThanEqualExp) && (
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[1], blockMaxValue) >= 0))) {
+ FilterUtil.compareValues(filterValues[1], blockMaxValue, carbonDimension, false) > 0))
+ || ((lessThanEqualExp) && (
+ FilterUtil.compareValues(filterValues[1], blockMaxValue, carbonDimension, false)
+ >= 0))) {
endBlockMaxisDefaultEnd = true;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
index f901238..c403846 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureEvaluatorImpl.java
@@ -29,9 +29,9 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
import org.apache.carbondata.core.scan.filter.ColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;
@@ -65,7 +65,7 @@ public abstract class RestructureEvaluatorImpl implements FilterExecuter {
}
List<byte[]> noDictionaryFilterValuesList = filterValues.getNoDictionaryFilterValuesList();
for (byte[] filterValue : noDictionaryFilterValuesList) {
- int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue);
+ int compare = FilterUtil.compareValues(filterValue, defaultValue, dimension, true);
if (compare == 0) {
isDefaultValuePresentInFilterValues = true;
break;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index d5f4ee2..a5faacc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -36,6 +36,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionColumnPage;
+import org.apache.carbondata.core.datastore.chunk.store.ColumnPageWrapper;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
@@ -406,12 +407,11 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
DimensionColumnPage columnDataChunk =
blockChunkHolder.getDimensionRawColumnChunks()[dimensionChunkIndex[i]]
.decodeColumnPage(pageIndex);
- if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && columnDataChunk instanceof VariableLengthDimensionColumnPage) {
+ if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && (
+ columnDataChunk instanceof VariableLengthDimensionColumnPage
+ || columnDataChunk instanceof ColumnPageWrapper)) {
- VariableLengthDimensionColumnPage dimensionColumnDataChunk =
- (VariableLengthDimensionColumnPage) columnDataChunk;
- byte[] memberBytes = dimensionColumnDataChunk.getChunkData(index);
+ byte[] memberBytes = columnDataChunk.getChunkData(index);
if (null != memberBytes) {
if (Arrays.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, memberBytes)) {
memberBytes = null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 31ab42e..63a5976 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -126,8 +126,16 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
maxValue = blockMaxValue[dimensionChunkIndex[0]];
- isScanRequired =
- isScanRequired(maxValue, filterRangeValues, isMinMaxSet[dimensionChunkIndex[0]]);
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ isScanRequired = isScanRequired(maxValue, filterRangeValues, dataType);
+ } else {
+ isScanRequired =
+ isScanRequired(maxValue, filterRangeValues, isMinMaxSet[dimensionChunkIndex[0]]);
+ }
}
} else {
isScanRequired = isDefaultValuePresentInFilter;
@@ -161,6 +169,32 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
return isScanRequired;
}
+ private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, DataType dataType) {
+ boolean isScanRequired = false;
+ Object maxValue =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMaxValue, dataType);
+ for (int k = 0; k < filterValues.length; k++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) {
+ return true;
+ }
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ int maxCompare = comparator.compare(data, maxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare <= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ return isScanRequired;
+ }
+
private boolean isScanRequired(byte[] maxValue, Object[] filterValue,
DataType dataType) {
Object value = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType);
@@ -201,8 +235,19 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(rawColumnChunk.getMaxValues()[i],
- this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i])) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i],
+ this.filterRangeValues, rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ if (scanRequired) {
int compare = ByteUtil.UnsafeComparer.INSTANCE
.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
if (compare < 0) {
@@ -423,7 +468,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
// binary search can only be applied if column is sorted
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int last = 0;
int startIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 8944a51..0f9cfae 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -86,7 +86,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
if (null != defaultValue) {
for (int k = 0; k < filterRangeValues.length; k++) {
int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+ FilterUtil.compareValues(filterRangeValues[k], defaultValue, dimension, false);
if (maxCompare <= 0) {
isDefaultValuePresentInFilter = true;
break;
@@ -124,8 +124,16 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
maxValue = blockMaxValue[dimensionChunkIndex[0]];
- isScanRequired =
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ isScanRequired = isScanRequired(maxValue, filterRangeValues, dataType);
+ } else {
+ isScanRequired =
isScanRequired(maxValue, filterRangeValues, isMinMaxSet[dimensionChunkIndex[0]]);
+ }
}
} else {
isScanRequired = isDefaultValuePresentInFilter;
@@ -158,6 +166,32 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
return isScanRequired;
}
+ private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, DataType dataType) {
+ boolean isScanRequired = false;
+ Object maxValue =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMaxValue, dataType);
+ for (int k = 0; k < filterValues.length; k++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) {
+ return true;
+ }
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ int maxCompare = comparator.compare(data, maxValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare <= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ return isScanRequired;
+ }
+
private boolean isScanRequired(byte[] maxValue, Object[] filterValue,
DataType dataType) {
Object value = DataTypeUtil.getMeasureObjectFromDataType(maxValue, dataType);
@@ -199,8 +233,19 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMaxValues() != null) {
- if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i])) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ if (scanRequired) {
int compare = ByteUtil.UnsafeComparer.INSTANCE
.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
if (compare <= 0) {
@@ -411,7 +456,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
// binary search can only be applied if column is sorted
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int last = 0;
int startIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 038d50b..eff6509 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -127,8 +127,16 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
minValue = blockMinValue[dimensionChunkIndex[0]];
- isScanRequired =
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ isScanRequired = isScanRequired(minValue, filterRangeValues, dataType);
+ } else {
+ isScanRequired =
isScanRequired(minValue, filterRangeValues, isMinMaxSet[dimensionChunkIndex[0]]);
+ }
}
} else {
isScanRequired = isDefaultValuePresentInFilter;
@@ -160,6 +168,32 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
return isScanRequired;
}
+ private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues, DataType dataType) {
+ boolean isScanRequired = false;
+ Object minValue =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMinValue, dataType);
+ for (int k = 0; k < filterValues.length; k++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) {
+ return true;
+ }
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ int minCompare = comparator.compare(data, minValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ return isScanRequired;
+ }
+
private boolean isScanRequired(byte[] minValue, Object[] filterValue,
DataType dataType) {
Object value =
@@ -201,8 +235,19 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
- if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i])) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ if (scanRequired) {
BitSet bitSet;
DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i);
if (null != rawColumnChunk.getLocalDictionary()) {
@@ -433,7 +478,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
// binary search can only be applied if column is sorted
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int last = 0;
int startIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
index fddb4ff..f0773d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -127,8 +127,16 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
} else {
minValue = blockMinValue[dimensionChunkIndex[0]];
- isScanRequired =
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ isScanRequired = isScanRequired(minValue, filterRangeValues, dataType);
+ } else {
+ isScanRequired =
isScanRequired(minValue, filterRangeValues, isMinMaxSet[dimensionChunkIndex[0]]);
+ }
}
} else {
isScanRequired = isDefaultValuePresentInFilter;
@@ -161,6 +169,32 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
return isScanRequired;
}
+ private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues, DataType dataType) {
+ boolean isScanRequired = false;
+ Object minValue =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMinValue, dataType);
+ for (int k = 0; k < filterValues.length; k++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) {
+ return true;
+ }
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ Object data =
+ DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType);
+ SerializableComparator comparator = Comparator.getComparator(dataType);
+ int minCompare = comparator.compare(data, minValue);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ return isScanRequired;
+ }
+
private boolean isScanRequired(byte[] minValue, Object[] filterValue,
DataType dataType) {
Object value = DataTypeUtil.getMeasureObjectFromDataType(minValue, dataType);
@@ -201,8 +235,19 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
boolean isExclude = false;
for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
if (rawColumnChunk.getMinValues() != null) {
- if (isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
- rawColumnChunk.getMinMaxFlagArray()[i])) {
+ boolean scanRequired;
+ DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
+ // for no dictionary measure column comparison can be done
+ // on the original data as like measure column
+ if (DataTypeUtil.isPrimitiveColumn(dataType) && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ scanRequired =
+ isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues, dataType);
+ } else {
+ scanRequired = isScanRequired(rawColumnChunk.getMinValues()[i], this.filterRangeValues,
+ rawColumnChunk.getMinMaxFlagArray()[i]);
+ }
+ if (scanRequired) {
BitSet bitSet;
DimensionColumnPage dimensionColumnPage = rawColumnChunk.decodeColumnPage(i);
if (null != rawColumnChunk.getLocalDictionary()) {
@@ -441,7 +486,7 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
BitSet bitSet = new BitSet(numerOfRows);
byte[][] filterValues = this.filterRangeValues;
// binary search can only be applied if column is sorted
- if (isNaturalSorted) {
+ if (isNaturalSorted && dimensionColumnPage.isExplicitSorted()) {
int start = 0;
int last = 0;
int startIndex = 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2bc73c5..4a713d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -210,7 +210,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
continue;
}
filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(),
- result.getDataType(), carbonMeasure));
+ result.getDataType(), carbonMeasure.getScale(), carbonMeasure.getPrecision()));
} catch (FilterIllegalMemberException e) {
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index daf9a91..0f9ba22 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -240,10 +240,9 @@ public abstract class BlockletScannedResult {
* Fill the column data to vector
*/
public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
- column = dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter]
- .fillVector(vectorInfo, column);
+ dimensionColumnPages[noDictionaryColumnChunkIndexes[i]][pageCounter]
+ .fillVector(vectorInfo, i);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
index 26b1135..1b83110 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
@@ -110,12 +110,9 @@ public class FilterQueryScannedResult extends BlockletScannedResult {
* Fill the column data to vector
*/
public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int chunkIndex : this.noDictionaryColumnChunkIndexes) {
- column = dimensionColumnPages[chunkIndex][pageCounter].fillVector(
- pageFilteredRowId[pageCounter],
- vectorInfo,
- column);
+ for (int index = 0; index < this.noDictionaryColumnChunkIndexes.length; index++) {
+ dimensionColumnPages[noDictionaryColumnChunkIndexes[index]][pageCounter]
+ .fillVector(pageFilteredRowId[pageCounter], vectorInfo, index);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/util/CarbonUnsafeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUnsafeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUnsafeUtil.java
new file mode 100644
index 0000000..00e7dee
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUnsafeUtil.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+public class CarbonUnsafeUtil {
+
+ /**
+ * Put the data to unsafe memory
+ *
+ * @param dataType
+ * @param data
+ * @param baseObject
+ * @param address
+ * @param size
+ * @param sizeInBytes
+ */
+ public static void putDataToUnsafe(DataType dataType, Object data, Object baseObject,
+ long address, int size, int sizeInBytes) {
+ dataType = DataTypeUtil.valueOf(dataType.getName());
+ if (dataType == DataTypes.BOOLEAN) {
+ CarbonUnsafe.getUnsafe().putBoolean(baseObject, address + size, (boolean) data);
+ } else if (dataType == DataTypes.BYTE) {
+ CarbonUnsafe.getUnsafe().putByte(baseObject, address + size, (byte) data);
+ } else if (dataType == DataTypes.SHORT) {
+ CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, (short) data);
+ } else if (dataType == DataTypes.INT) {
+ CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, (int) data);
+ } else if (dataType == DataTypes.LONG) {
+ CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, (long) data);
+ } else if (DataTypes.isDecimal(dataType) || dataType == DataTypes.DOUBLE) {
+ CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, (double) data);
+ } else if (dataType == DataTypes.FLOAT) {
+ CarbonUnsafe.getUnsafe().putFloat(baseObject, address + size, (float) data);
+ } else if (dataType == DataTypes.BYTE_ARRAY) {
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, address + size,
+ sizeInBytes);
+ }
+ }
+
+ /**
+ * Retrieve/Get the data from unsafe memory
+ *
+ * @param dataType
+ * @param baseObject
+ * @param address
+ * @param size
+ * @param sizeInBytes
+ * @return
+ */
+ public static Object getDataFromUnsafe(DataType dataType, Object baseObject, long address,
+ int size, int sizeInBytes) {
+ dataType = DataTypeUtil.valueOf(dataType.getName());
+ Object data = new Object();
+ if (dataType == DataTypes.BOOLEAN) {
+ data = CarbonUnsafe.getUnsafe().getBoolean(baseObject, address + size);
+ } else if (dataType == DataTypes.BYTE) {
+ data = CarbonUnsafe.getUnsafe().getByte(baseObject, address + size);
+ } else if (dataType == DataTypes.SHORT) {
+ data = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size);
+ } else if (dataType == DataTypes.INT) {
+ data = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size);
+ } else if (dataType == DataTypes.LONG) {
+ data = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size);
+ } else if (DataTypes.isDecimal(dataType) || dataType == DataTypes.DOUBLE) {
+ data = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
+ } else if (dataType == DataTypes.FLOAT) {
+ data = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size);
+ } else if (dataType == DataTypes.BYTE_ARRAY) {
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(baseObject, address + size, data, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ sizeInBytes);
+ }
+ return data;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index dc03944..9ab875c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -3243,6 +3243,28 @@ public final class CarbonUtil {
}
/**
+ * Check if the page is adaptive encoded
+ *
+ * @param encodings
+ * @return
+ */
+ public static boolean isEncodedWithMeta(List<org.apache.carbondata.format.Encoding> encodings) {
+ if (encodings != null && !encodings.isEmpty()) {
+ org.apache.carbondata.format.Encoding encoding = encodings.get(0);
+ switch (encoding) {
+ case DIRECT_COMPRESS:
+ case DIRECT_STRING:
+ case ADAPTIVE_INTEGRAL:
+ case ADAPTIVE_DELTA_INTEGRAL:
+ case ADAPTIVE_FLOATING:
+ case ADAPTIVE_DELTA_FLOATING:
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
* Check whether it is standard table means tablepath has Fact/Part0/Segment_ tail present with
* all carbon files. In other cases carbon files present directly under tablepath or
* tablepath/partition folder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index fa08df9..612e17c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -82,8 +82,8 @@ public final class DataTypeUtil {
* @return
*/
public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
- CarbonMeasure carbonMeasure) {
- return getMeasureValueBasedOnDataType(msrValue, dataType,carbonMeasure, false);
+ int scale, int precision) {
+ return getMeasureValueBasedOnDataType(msrValue, dataType, scale, precision, false);
}
/**
@@ -95,13 +95,13 @@ public final class DataTypeUtil {
* @return
*/
public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
- CarbonMeasure carbonMeasure, boolean useConverter) {
+ int scale, int precision, boolean useConverter) {
if (dataType == DataTypes.BOOLEAN) {
return BooleanConvert.parseBoolean(msrValue);
} else if (DataTypes.isDecimal(dataType)) {
BigDecimal bigDecimal =
- new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- BigDecimal decimal = normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
+ new BigDecimal(msrValue).setScale(scale, RoundingMode.HALF_UP);
+ BigDecimal decimal = normalizeDecimalValue(bigDecimal, precision);
if (useConverter) {
return converter.convertFromBigDecimalToDecimal(decimal);
} else {
@@ -415,6 +415,38 @@ public final class DataTypeUtil {
}
/**
+ * Convert the min/max values to bytes for no dictionary column
+ *
+ * @param dimensionValue
+ * @param actualDataType
+ * @return
+ */
+ public static byte[] getMinMaxBytesBasedOnDataTypeForNoDictionaryColumn(Object dimensionValue,
+ DataType actualDataType) {
+ if (dimensionValue == null) {
+ if (actualDataType == DataTypes.STRING) {
+ return CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
+ } else {
+ return new byte[0];
+ }
+ }
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBytes(Boolean.valueOf(ByteUtil.toBoolean((byte) dimensionValue)));
+ } else if (actualDataType == DataTypes.SHORT) {
+ return ByteUtil.toXorBytes((Short) dimensionValue);
+ } else if (actualDataType == DataTypes.INT) {
+ return ByteUtil.toXorBytes((Integer) dimensionValue);
+ } else if (actualDataType == DataTypes.LONG) {
+ return ByteUtil.toXorBytes((Long) dimensionValue);
+ } else if (actualDataType == DataTypes.TIMESTAMP) {
+ return ByteUtil.toXorBytes((Long)dimensionValue);
+ } else {
+ // Default action for String/Varchar
+ return ByteUtil.toBytes(dimensionValue.toString());
+ }
+ }
+
+ /**
* Returns true for fixed length DataTypes.
* @param dataType
* @return
@@ -976,4 +1008,20 @@ public final class DataTypeUtil {
return value;
}
+ /**
+ * Check if the column is a no dictionary primitive column
+ *
+ * @param dataType
+ * @return
+ */
+ public static boolean isPrimitiveColumn(DataType dataType) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE || dataType == DataTypes.SHORT
+ || dataType == DataTypes.INT || dataType == DataTypes.LONG || DataTypes.isDecimal(dataType)
+ || dataType == DataTypes.FLOAT || dataType == DataTypes.DOUBLE
+ || dataType == DataTypes.BYTE_ARRAY) {
+ return true;
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
index fca1244..a51d6d1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NonDictionaryUtil.java
@@ -116,7 +116,7 @@ public class NonDictionaryUtil {
return measures[index];
}
- public static void prepareOutObj(Object[] out, int[] dimArray, byte[][] byteBufferArr,
+ public static void prepareOutObj(Object[] out, int[] dimArray, Object[] byteBufferArr,
Object[] measureArray) {
out[WriteStepRowUtil.DICTIONARY_DIMENSION] = dimArray;
out[WriteStepRowUtil.NO_DICTIONARY_AND_COMPLEX] = byteBufferArr;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/TestEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/TestEncodingFactory.java b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/TestEncodingFactory.java
index 0b8bcc7..3f1c819 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/TestEncodingFactory.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/TestEncodingFactory.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.datastore.page.encoding;
+import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec;
import org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec;
import org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec;
@@ -38,25 +39,25 @@ public class TestEncodingFactory extends TestCase {
// for Byte
primitivePageStatsCollector.update((long) Byte.MAX_VALUE);
ColumnPageCodec columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveIntegralCodec);
assert (DataTypes.BYTE == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
// for Short
primitivePageStatsCollector.update((long) Short.MAX_VALUE);
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveIntegralCodec);
assert (DataTypes.SHORT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
// for int
primitivePageStatsCollector.update((long) Integer.MAX_VALUE);
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveIntegralCodec);
assert (DataTypes.INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
// for long
primitivePageStatsCollector.update(Long.MAX_VALUE);
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof DirectCompressCodec);
assert ("DirectCompressCodec".equals(columnPageCodec.getName()));
}
@@ -67,25 +68,25 @@ public class TestEncodingFactory extends TestCase {
// for Byte
primitivePageStatsCollector.update((long) 200);
ColumnPageCodec columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveDeltaIntegralCodec);
assert (DataTypes.BYTE == ((AdaptiveDeltaIntegralCodec) columnPageCodec).getTargetDataType());
// for Short
primitivePageStatsCollector.update((long) 634767);
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveIntegralCodec);
assert (DataTypes.SHORT_INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
// for int
primitivePageStatsCollector.update((long) (Integer.MAX_VALUE + 200));
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof AdaptiveIntegralCodec);
assert (DataTypes.INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
// for int
primitivePageStatsCollector.update(Long.MAX_VALUE);
columnPageCodec =
- DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false);
+ DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
assert (columnPageCodec instanceof DirectCompressCodec);
assert ("DirectCompressCodec".equals(columnPageCodec.getName()));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index 720e954..e69fa9e 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -62,13 +62,13 @@ public class DataTypeUtilTest {
@Test public void testGetMeasureValueBasedOnDataType() {
ColumnSchema columnSchema = new ColumnSchema();
CarbonMeasure carbonMeasure = new CarbonMeasure(columnSchema, 1);
- Object resultInt = getMeasureValueBasedOnDataType("1", DataTypes.INT, carbonMeasure);
+ Object resultInt = getMeasureValueBasedOnDataType("1", DataTypes.INT, carbonMeasure.getScale(), carbonMeasure.getPrecision());
Object expectedInt = Double.valueOf(1).intValue();
assertEquals(expectedInt, resultInt);
- Object resultLong = getMeasureValueBasedOnDataType("1", DataTypes.LONG, carbonMeasure);
+ Object resultLong = getMeasureValueBasedOnDataType("1", DataTypes.LONG, carbonMeasure.getScale(), carbonMeasure.getPrecision());
Object expectedLong = Long.valueOf(1);
assertEquals(expectedLong, resultLong);
- Object resultDefault = getMeasureValueBasedOnDataType("1", DataTypes.DOUBLE, carbonMeasure);
+ Object resultDefault = getMeasureValueBasedOnDataType("1", DataTypes.DOUBLE, carbonMeasure.getScale(), carbonMeasure.getPrecision());
Double expectedDefault = Double.valueOf(1);
assertEquals(expectedDefault, resultDefault);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 3fdce4e..ee74bf2 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -17,10 +17,13 @@
package org.apache.carbondata.core.util;
+import java.io.IOException;
import java.util.Arrays;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
@@ -37,9 +40,14 @@ import org.apache.carbondata.core.scan.expression.logical.TrueExpression;
import org.apache.carbondata.core.scan.filter.executer.RangeValueFilterExecuterImpl;
import org.apache.carbondata.core.scan.filter.intf.FilterOptimizer;
import org.apache.carbondata.core.scan.filter.optimizer.RangeFilterOptmizer;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import mockit.Deencapsulation;
+import mockit.Mock;
import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -47,7 +55,15 @@ import org.junit.Test;
/* Test Cases for Range Filter */
public class RangeFilterProcessorTest {
+
+ public static DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo =
+ new DimColumnResolvedFilterInfo();
+
@BeforeClass public static void setUp() throws Exception {
+ ColumnSchema columnSchema = new ColumnSchema();
+ columnSchema.setDataType(DataTypes.STRING);
+ CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0);
+ dimColumnResolvedFilterInfo.setDimension(carbonDimension);
}
public boolean checkBothTrees(Expression a, Expression b) {
@@ -320,6 +336,8 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
+
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
Assert.assertFalse(result);
}
@@ -336,6 +354,7 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
Assert.assertFalse(result);
}
@@ -352,6 +371,7 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
Assert.assertTrue(result);
}
@@ -369,6 +389,7 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
rangeCovered = Deencapsulation.getField(range, "isRangeFullyCoverBlock");
@@ -389,6 +410,7 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
startBlockMinIsDefaultStart = Deencapsulation.getField(range, "startBlockMinIsDefaultStart");
@@ -409,6 +431,7 @@ public class RangeFilterProcessorTest {
Deencapsulation.setField(range, "isDimensionPresentInCurrentBlock", true);
Deencapsulation.setField(range, "lessThanExp", true);
Deencapsulation.setField(range, "greaterThanExp", true);
+ Deencapsulation.setField(range, "dimColEvaluatorInfo", dimColumnResolvedFilterInfo);
result = range.isScanRequired(BlockMin, BlockMax, filterMinMax, true);
endBlockMaxisDefaultEnd = Deencapsulation.getField(range, "endBlockMaxisDefaultEnd");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
index 3823460..4734abd 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/AbstractBloomDataMapWriter.java
@@ -144,7 +144,7 @@ public abstract class AbstractBloomDataMapWriter extends DataMapWriter {
|| indexColumns.get(indexColIdx).hasEncoding(Encoding.DIRECT_DICTIONARY)) {
indexValue = convertDictionaryValue(indexColIdx, value);
} else {
- indexValue = convertNonDictionaryValue(indexColIdx, (byte[]) value);
+ indexValue = convertNonDictionaryValue(indexColIdx, value);
}
}
if (indexValue.length == 0) {
@@ -155,7 +155,7 @@ public abstract class AbstractBloomDataMapWriter extends DataMapWriter {
protected abstract byte[] convertDictionaryValue(int indexColIdx, Object value);
- protected abstract byte[] convertNonDictionaryValue(int indexColIdx, byte[] value);
+ protected abstract byte[] convertNonDictionaryValue(int indexColIdx, Object value);
private void initDataMapFile() throws IOException {
if (!FileFactory.isFileExist(dataMapPath)) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c8f70630/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index 27911ca..2e2d94b 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -62,6 +62,7 @@ import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.loading.DataField;
import org.apache.carbondata.processing.loading.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.loading.converter.FieldConverter;
@@ -343,8 +344,18 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
// for dictionary/date columns, convert the surrogate key to bytes
internalFilterValue = CarbonUtil.getValueAsBytes(DataTypes.INT, convertedValue);
} else {
- // for non dictionary dimensions, is already bytes,
- internalFilterValue = (byte[]) convertedValue;
+ // for non dictionary dimensions, numeric columns will be of original data,
+ // so convert the data to bytes
+ if (DataTypeUtil.isPrimitiveColumn(carbonColumn.getDataType())) {
+ if (convertedValue == null) {
+ convertedValue = DataConvertUtil.getNullValueForMeasure(carbonColumn.getDataType(),
+ carbonColumn.getColumnSchema().getScale());
+ }
+ internalFilterValue =
+ CarbonUtil.getValueAsBytes(carbonColumn.getDataType(), convertedValue);
+ } else {
+ internalFilterValue = (byte[]) convertedValue;
+ }
}
if (internalFilterValue.length == 0) {
internalFilterValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;