You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/12/23 13:08:43 UTC
[1/2] incubator-carbondata git commit: code cleanup for bigint
compression
Repository: incubator-carbondata
Updated Branches:
refs/heads/master a992940a7 -> 956eb3348
code cleanup for bigint compression
minor fix
style fix
address comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b0b6bcbe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b0b6bcbe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b0b6bcbe
Branch: refs/heads/master
Commit: b0b6bcbe5116a9f7fc70ee59a5b2008c8c025955
Parents: a992940
Author: Jihong Ma <ji...@Jihongs-MacBook-Pro.local>
Authored: Wed Dec 21 11:59:45 2016 -0800
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Dec 23 18:36:58 2016 +0530
----------------------------------------------------------------------
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 4 +-
.../core/compression/BigDecimalCompressor.java | 14 +--
.../core/compression/BigIntCompressor.java | 124 +++++++++----------
.../core/compression/DoubleCompressor.java | 15 +--
.../core/compression/ValueCompressor.java | 30 ++---
.../store/compression/ReaderCompressModel.java | 10 +-
.../store/compression/WriterCompressModel.java | 14 +--
.../type/UnCompressBigDecimalByte.java | 4 +-
...yCompressedDoubleArrayDataInMemoryStore.java | 6 +-
.../core/util/BigDecimalCompressionFinder.java | 26 ++--
.../carbondata/core/util/CompressionFinder.java | 20 +--
.../core/util/ValueCompressionUtil.java | 20 +--
13 files changed, 142 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 1455030..8bd9245 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -90,7 +90,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
measureColumnChunks.get(blockIndex).getDataPageLength()));
// get the data holder after uncompressing
CarbonReadDataHolder measureDataHolder =
- values.uncompress(compressModel.getChangedDataType())
+ values.uncompress(compressModel.getConvertedDataType())
.getValues(compressModel.getMantissa(), compressModel.getMaxValue());
// create and set the data chunk
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 9cba57f..49c9373 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -168,7 +168,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
values.setValue(dataPage);
// get the data holder after uncompressing
CarbonReadDataHolder measureDataHolder =
- values.uncompress(compressionModel.getChangedDataType()[0])
+ values.uncompress(compressionModel.getConvertedDataType()[0])
.getValues(compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0]);
// set the data chunk
datChunk.setMeasureDataHolder(measureDataHolder);
@@ -222,7 +222,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
values.setValue(dataPage);
// get the data holder after uncompressing
CarbonReadDataHolder measureDataHolder =
- values.uncompress(compressionModel.getChangedDataType()[0])
+ values.uncompress(compressionModel.getConvertedDataType()[0])
.getValues(compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0]);
// set the data chunk
dataChunk.setMeasureDataHolder(measureDataHolder);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
index 85c9927..96ca2e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
@@ -24,7 +24,7 @@ import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
- * Bigdecimal data type compressor
+ * Big decimal data type compressor
*
*/
public class BigDecimalCompressor extends BigIntCompressor {
@@ -39,16 +39,16 @@ public class BigDecimalCompressor extends BigIntCompressor {
Long[] maxValues = (Long[]) maxValue;
Object leftCompressedValue = getCompressedValues(
bigdCompressionFinder.getLeftCompType(), dataHolder,
- bigdCompressionFinder.getLeftChangedDataType(), maxValues[0], 0);
+ bigdCompressionFinder.getLeftConvertedDataType(), maxValues[0], 0);
readLeft = false;
Object rightCompressedValue = getCompressedValues(
bigdCompressionFinder.getRightCompType(), dataHolder,
- bigdCompressionFinder.getRightChangedDataType(), maxValues[1], 0);
+ bigdCompressionFinder.getRightConvertedDataType(), maxValues[1], 0);
return new Object[] { leftCompressedValue, rightCompressedValue };
}
@Override
- protected Object compressMaxMin(DataType changedDataType,
+ protected Object compressMaxMin(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, Object max) {
long maxValue = (long) max;
long[][] writableBigDValues = dataHolder.getWritableBigDecimalValues();
@@ -58,11 +58,11 @@ public class BigDecimalCompressor extends BigIntCompressor {
} else {
value = writableBigDValues[1];
}
- return compressMaxMin(changedDataType, maxValue, value);
+ return compressValue(convertedDataType, value, maxValue, true);
}
@Override
- protected Object compressNone(DataType changedDataType,
+ protected Object compressAdaptive(DataType convertedDataType,
CarbonWriteDataHolder dataHolder) {
long[][] writableBigDValues = dataHolder.getWritableBigDecimalValues();
long[] value = null;
@@ -71,6 +71,6 @@ public class BigDecimalCompressor extends BigIntCompressor {
} else {
value = writableBigDValues[1];
}
- return compressNone(changedDataType, value);
+ return compressValue(convertedDataType, value, 0, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index 315b28f..c27887d 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -26,110 +26,106 @@ import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
*/
public class BigIntCompressor extends ValueCompressor {
- @Override protected Object compressNonDecimalMaxMin(DataType changedDataType,
+ @Override protected Object compressNonDecimalMaxMin(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, int decimal, Object max) {
// in case if bigint, decimal will be 0
- return compressMaxMin(changedDataType, dataHolder, max);
+ return compressMaxMin(convertedDataType, dataHolder, max);
}
@Override
- protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ protected Object compressNonDecimal(DataType convertedDataType, CarbonWriteDataHolder dataHolder,
int decimal) {
// in case if bigint, decimal will be 0
- return compressNone(changedDataType, dataHolder);
+ return compressAdaptive(convertedDataType, dataHolder);
}
+ /**
+ * 1. It gets delta value i.e difference of maximum value and actual value
+ * 2. Convert the delta value computed above to convertedDatatype if it can
+ * be stored with less byte
+ * @param convertedDataType
+ * @param dataHolder
+ * @param max
+ * @return
+ */
@Override
- protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ protected Object compressMaxMin(DataType convertedDataType, CarbonWriteDataHolder dataHolder,
Object max) {
long maxValue = (long) max;
long[] value = dataHolder.getWritableLongValues();
- return compressMaxMin(changedDataType, maxValue, value);
+ return compressValue(convertedDataType, value, maxValue, true);
}
/**
- * 1. It gets delta value i.e difference of maximum value and actual value
- * 2. Convert the delta value computed above to changedDatatype
- * @param changedDataType
- * @param maxValue
- * @param value
+ * 1. It converts actual value to converted data type if it can be
+ * stored with less bytes.
+ * @param convertedDataType
+ * @param dataHolder
* @return
*/
- protected Object compressMaxMin(DataType changedDataType, long maxValue,
- long[] value) {
- int i = 0;
- switch (changedDataType) {
- case DATA_BYTE:
- byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- result[i] = (byte) (maxValue - value[j]);
- i++;
- }
- return result;
- case DATA_SHORT:
- short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- shortResult[i] = (short) (maxValue - value[j]);
- i++;
- }
- return shortResult;
- case DATA_INT:
- int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- intResult[i] = (int) (maxValue - value[j]);
- i++;
- }
- return intResult;
- default:
- long[] defaultResult = new long[value.length];
- for (int j = 0; j < value.length; j++) {
- defaultResult[i] = (long) (maxValue - value[j]);
- i++;
- }
- return defaultResult;
- }
- }
-
@Override
- protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) {
+ protected Object compressAdaptive(DataType convertedDataType, CarbonWriteDataHolder dataHolder) {
long[] value = dataHolder.getWritableLongValues();
- return compressNone(changedDataType, value);
+ return compressValue(convertedDataType, value, 0, false);
}
/**
- * It convert the value to changed datatype.
- * Changed datatype is computed based list of values it has.
+ * It convert the value or delta value (max - actual) to converted data type.
+ * Converted data type is computed based list of values it has.
* for instance if value is 2,10,12,45
- * these value can be easily fit in byte and hence below method convert to byte and store it.
- * @param changedDataType
+ * these value can be easily fitted in byte value,
+ * hence it will be converted into byte to store.
+ * @param convertedDataType
* @param value
+ * @param maxValue
+ * @param isMinMax
* @return
*/
- protected Object compressNone(DataType changedDataType, long[] value) {
- int i = 0;
- switch (changedDataType) {
+ protected Object compressValue(DataType convertedDataType, long[] value, long maxValue,
+ boolean isMinMax) {
+ switch (convertedDataType) {
case DATA_BYTE:
byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- result[i] = (byte) value[j];
- i++;
+ if (isMinMax) {
+ for (int j = 0; j < value.length; j++) {
+ result[j] = (byte) (maxValue - value[j]);
+ }
+ } else {
+ for (int j = 0; j < value.length; j++) {
+ result[j] = (byte) value[j];
+ }
}
return result;
case DATA_SHORT:
short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- shortResult[i] = (short) value[j];
- i++;
+ if (isMinMax) {
+ for (int j = 0; j < value.length; j++) {
+ shortResult[j] = (short) (maxValue - value[j]);
+ }
+ } else {
+ for (int j = 0; j < value.length; j++) {
+ shortResult[j] = (short) value[j];
+ }
}
return shortResult;
case DATA_INT:
int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- intResult[i] = (int) value[j];
- i++;
+ if (isMinMax) {
+ for (int j = 0; j < value.length; j++) {
+ intResult[j] = (int) (maxValue - value[j]);
+ }
+ } else {
+ for (int j=0; j < value.length; j++) {
+ intResult[j] = (int) value[j];
+ }
}
return intResult;
default:
+ if (isMinMax) {
+ for (int j = 0; j < value.length; j++) {
+ value[j] = (maxValue - value[j]);
+ }
+ }
return value;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 8d20f18..909369d 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -28,12 +28,13 @@ import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
*/
public class DoubleCompressor extends ValueCompressor {
- @Override protected Object compressNonDecimalMaxMin(DataType changedDataType,
+
+ @Override protected Object compressNonDecimalMaxMin(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, int decimal, Object maxValue) {
int i = 0;
BigDecimal max = BigDecimal.valueOf((double)maxValue);
double[] value = dataHolder.getWritableDoubleValues();
- switch (changedDataType) {
+ switch (convertedDataType) {
case DATA_BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
@@ -92,11 +93,11 @@ public class DoubleCompressor extends ValueCompressor {
}
@Override
- protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ protected Object compressNonDecimal(DataType convertedDataType, CarbonWriteDataHolder dataHolder,
int decimal) {
int i = 0;
double[] value = dataHolder.getWritableDoubleValues();
- switch (changedDataType) {
+ switch (convertedDataType) {
case DATA_BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
@@ -143,12 +144,12 @@ public class DoubleCompressor extends ValueCompressor {
}
@Override
- protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ protected Object compressMaxMin(DataType convertedDataType, CarbonWriteDataHolder dataHolder,
Object max) {
double maxValue = (double) max;
double[] value = dataHolder.getWritableDoubleValues();
int i = 0;
- switch (changedDataType) {
+ switch (convertedDataType) {
case DATA_BYTE:
byte[] result = new byte[value.length];
for (int j = 0; j < value.length; j++) {
@@ -195,7 +196,7 @@ public class DoubleCompressor extends ValueCompressor {
}
@Override
- protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) {
+ protected Object compressAdaptive(DataType changedDataType, CarbonWriteDataHolder dataHolder) {
double[] value = dataHolder.getWritableDoubleValues();
int i = 0;
switch (changedDataType) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index 9e7862c..31f86c0 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -32,7 +32,7 @@ public abstract class ValueCompressor {
CarbonWriteDataHolder dataHolder, Object maxValue, int decimal) {
return getCompressedValues(compressionFinder.getCompType(),
dataHolder,
- compressionFinder.getChangedDataType(),
+ compressionFinder.getConvertedDataType(),
maxValue, decimal);
}
@@ -40,62 +40,62 @@ public abstract class ValueCompressor {
*
* @param compType
* @param dataHolder
- * @param changedDataType
+ * @param convertedDataType
* @param maxValue
* @param decimal
* @return compressed data
*/
public Object getCompressedValues(COMPRESSION_TYPE compType, CarbonWriteDataHolder dataHolder,
- DataType changedDataType, Object maxValue, int decimal) {
+ DataType convertedDataType, Object maxValue, int decimal) {
switch (compType) {
case ADAPTIVE:
- return compressNone(changedDataType, dataHolder);
+ return compressAdaptive(convertedDataType, dataHolder);
case DELTA_DOUBLE:
- return compressMaxMin(changedDataType, dataHolder, maxValue);
+ return compressMaxMin(convertedDataType, dataHolder, maxValue);
case BIGINT:
- return compressNonDecimal(changedDataType, dataHolder, decimal);
+ return compressNonDecimal(convertedDataType, dataHolder, decimal);
default:
- return compressNonDecimalMaxMin(changedDataType, dataHolder, decimal, maxValue);
+ return compressNonDecimalMaxMin(convertedDataType, dataHolder, decimal, maxValue);
}
}
/**
*
- * @param changedDataType
+ * @param convertedDataType
* @param dataHolder
* @param decimal
* @param maxValue
* @return compressed data
*/
- protected abstract Object compressNonDecimalMaxMin(DataType changedDataType,
+ protected abstract Object compressNonDecimalMaxMin(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, int decimal, Object maxValue);
/**
*
- * @param changedDataType
+ * @param convertedDataType
* @param dataHolder
* @param decimal
* @return compressed data
*/
- protected abstract Object compressNonDecimal(DataType changedDataType,
+ protected abstract Object compressNonDecimal(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, int decimal);
/**
*
- * @param changedDataType
+ * @param convertedDataType
* @param dataHolder
* @param maxValue
* @return compressed data
*/
- protected abstract Object compressMaxMin(DataType changedDataType,
+ protected abstract Object compressMaxMin(DataType convertedDataType,
CarbonWriteDataHolder dataHolder, Object maxValue);
/**
*
- * @param changedDataType
+ * @param convertedDataType
* @param dataHolder
* @return compressed data
*/
- protected abstract Object compressNone(DataType changedDataType,
+ protected abstract Object compressAdaptive(DataType convertedDataType,
CarbonWriteDataHolder dataHolder);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
index ad242f0..8981aaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.util.ValueCompressionUtil;
public class ReaderCompressModel {
private ValueEncoderMeta valueEncoderMeta;
- private ValueCompressionUtil.DataType changedDataType;
+ private ValueCompressionUtil.DataType convertedDataType;
private ValueCompressonHolder.UnCompressValue unCompressValues;
@@ -34,12 +34,12 @@ public class ReaderCompressModel {
this.valueEncoderMeta = valueEncoderMeta;
}
- public ValueCompressionUtil.DataType getChangedDataType() {
- return changedDataType;
+ public ValueCompressionUtil.DataType getConvertedDataType() {
+ return convertedDataType;
}
- public void setChangedDataType(ValueCompressionUtil.DataType changedDataType) {
- this.changedDataType = changedDataType;
+ public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+ this.convertedDataType = convertedDataType;
}
public Object getMaxValue() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
index a9c2666..e68a9fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
@@ -31,7 +31,7 @@ public class WriterCompressModel {
/**
* DataType[] variable.
*/
- private ValueCompressionUtil.DataType[] changedDataType;
+ private ValueCompressionUtil.DataType[] convertedDataType;
/**
* DataType[] variable.
*/
@@ -86,17 +86,17 @@ public class WriterCompressModel {
}
/**
- * @return the changedDataType
+ * @return the convertedDataType
*/
- public ValueCompressionUtil.DataType[] getChangedDataType() {
- return changedDataType;
+ public ValueCompressionUtil.DataType[] getConvertedDataType() {
+ return convertedDataType;
}
/**
- * @param changedDataType the changedDataType to set
+ * @param convertedDataType the convertedDataType to set
*/
- public void setChangedDataType(ValueCompressionUtil.DataType[] changedDataType) {
- this.changedDataType = changedDataType;
+ public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
+ this.convertedDataType = convertedDataType;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java
index 97e0d38..98990d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressBigDecimalByte.java
@@ -87,9 +87,9 @@ public class UnCompressBigDecimalByte<T> implements UnCompressValue<T> {
@Override
public UnCompressValue uncompress(DataType dataType) {
ValueCompressonHolder.UnCompressValue left = leftPart
- .uncompress(compressionFinder.getLeftChangedDataType());
+ .uncompress(compressionFinder.getLeftConvertedDataType());
ValueCompressonHolder.UnCompressValue right = rightPart
- .uncompress(compressionFinder.getRightChangedDataType());
+ .uncompress(compressionFinder.getRightConvertedDataType());
return new UnCompressBigDecimalByte<>(compressionFinder, left, right);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
index c4733f1..b2a99c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
@@ -39,14 +39,14 @@ public class HeavyCompressedDoubleArrayDataInMemoryStore
CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
if (cols != null) {
for (int i = 0; i < cols.length; i++) {
- vals[cols[i]] = values[cols[i]].uncompress(compressionModel.getChangedDataType()[cols[i]])
+ vals[cols[i]] = values[cols[i]].uncompress(compressionModel.getConvertedDataType()[cols[i]])
.getValues(compressionModel.getMantissa()[cols[i]],
compressionModel.getMaxValue()[cols[i]]);
}
} else {
for (int i = 0; i < vals.length; i++) {
- vals[i] = values[i].uncompress(compressionModel.getChangedDataType()[i])
+ vals[i] = values[i].uncompress(compressionModel.getConvertedDataType()[i])
.getValues(compressionModel.getMantissa()[i], compressionModel.getMaxValue()[i]);
}
}
@@ -58,7 +58,7 @@ public class HeavyCompressedDoubleArrayDataInMemoryStore
return null;
}
CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
- vals[cols] = values[cols].uncompress(compressionModel.getChangedDataType()[cols])
+ vals[cols] = values[cols].uncompress(compressionModel.getConvertedDataType()[cols])
.getValues(compressionModel.getMantissa()[cols], compressionModel.getMaxValue()[cols]);
return new CompressedDataMeasureDataWrapper(vals);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java
index 9a7ae13..7f74960 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/BigDecimalCompressionFinder.java
@@ -44,29 +44,29 @@ public class BigDecimalCompressionFinder extends CompressionFinder {
private DataType rightActualDataType;
/**
- * non decimal changed data type
+ * non decimal converted data type
*/
- private DataType leftChangedDataType;
+ private DataType leftConvertedDataType;
/**
- * decimal changed data type
+ * decimal converted data type
*/
- private DataType rightChangedDataType;
+ private DataType rightConvertedDataType;
public BigDecimalCompressionFinder(COMPRESSION_TYPE compType,
- DataType actualDataType, DataType changedDataType, char measureStoreType) {
- super(compType, actualDataType, changedDataType, measureStoreType);
+ DataType actualDataType, DataType convertedDataType, char measureStoreType) {
+ super(compType, actualDataType, convertedDataType, measureStoreType);
}
public BigDecimalCompressionFinder(COMPRESSION_TYPE[] compType,
- DataType[] actualDataType, DataType[] changedDataType, char measureStoreType) {
+ DataType[] actualDataType, DataType[] convertedDataType, char measureStoreType) {
super(null, null, null, measureStoreType);
this.leftCompType = compType[0];
this.rightCompType = compType[1];
this.leftActualDataType = actualDataType[0];
this.rightActualDataType = actualDataType[1];
- this.leftChangedDataType = changedDataType[0];
- this.rightChangedDataType = changedDataType[1];
+ this.leftConvertedDataType = convertedDataType[0];
+ this.rightConvertedDataType = convertedDataType[1];
}
public COMPRESSION_TYPE getLeftCompType() {
@@ -85,12 +85,10 @@ public class BigDecimalCompressionFinder extends CompressionFinder {
return rightActualDataType;
}
- public DataType getLeftChangedDataType() {
- return leftChangedDataType;
- }
+ public DataType getLeftConvertedDataType() { return leftConvertedDataType; }
- public DataType getRightChangedDataType() {
- return rightChangedDataType;
+ public DataType getRightConvertedDataType() {
+ return rightConvertedDataType;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
index b642a20..a65dafb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
@@ -13,7 +13,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
private DataType actualDataType;
- private DataType changedDataType;
+ private DataType convertedDataType;
/**
* the size of changed data
*/
@@ -28,14 +28,14 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
*
* @param compType
* @param actualDataType
- * @param changedDataType
+ * @param convertedDataType
*/
CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType,
- DataType changedDataType, char measureStoreType) {
+ DataType convertedDataType, char measureStoreType) {
super();
this.compType = compType;
this.actualDataType = actualDataType;
- this.changedDataType = changedDataType;
+ this.convertedDataType = convertedDataType;
this.measureStoreType = measureStoreType;
}
@@ -44,16 +44,16 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
*
* @param compType
* @param actualDataType
- * @param changedDataType
+ * @param convertedDataType
* @param priority
*/
- CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType,
+ CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType convertedDataType,
PRIORITY priority, char measureStoreType) {
super();
this.actualDataType = actualDataType;
- this.changedDataType = changedDataType;
- this.size = ValueCompressionUtil.getSize(changedDataType);
+ this.convertedDataType = convertedDataType;
+ this.size = ValueCompressionUtil.getSize(convertedDataType);
this.priority = priority;
this.compType = compType;
this.measureStoreType = measureStoreType;
@@ -127,8 +127,8 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
return actualDataType;
}
- public DataType getChangedDataType() {
- return changedDataType;
+ public DataType getConvertedDataType() {
+ return convertedDataType;
}
public int getSize() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b0b6bcbe/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 6210c30..98094c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -219,8 +219,8 @@ public final class ValueCompressionUtil {
actualDataTypes[0] = leftCompressionFinder.getActualDataType();
actualDataTypes[1] = rightCompressionFinder.getActualDataType();
- changedDataTypes[0] = leftCompressionFinder.getChangedDataType();
- changedDataTypes[1] = rightCompressionFinder.getChangedDataType();
+ changedDataTypes[0] = leftCompressionFinder.getConvertedDataType();
+ changedDataTypes[1] = rightCompressionFinder.getConvertedDataType();
CompressionFinder bigdCompressionFinder = new BigDecimalCompressionFinder(
compressionTypes, actualDataTypes, changedDataTypes, measureStoreType);
@@ -271,7 +271,7 @@ public final class ValueCompressionUtil {
/**
* It returns Compressor for given datatype
- * @param msrType
+ * @param compressorFinder
* @return compressor based on actualdatatype
*/
public static ValueCompressor getValueCompressor(CompressionFinder compressorFinder) {
@@ -303,9 +303,9 @@ public final class ValueCompressionUtil {
private static UnCompressValue getUncompressedValue(
BigDecimalCompressionFinder compressionFinder) {
UnCompressValue leftPart = getUncompressedValue(compressionFinder.getLeftCompType(),
- compressionFinder.getLeftActualDataType(), compressionFinder.getLeftChangedDataType());
+ compressionFinder.getLeftActualDataType(), compressionFinder.getLeftConvertedDataType());
UnCompressValue rightPart = getUncompressedValue(compressionFinder.getRightCompType(),
- compressionFinder.getRightActualDataType(), compressionFinder.getRightChangedDataType());
+ compressionFinder.getRightActualDataType(), compressionFinder.getRightConvertedDataType());
return new UnCompressBigDecimal<>(compressionFinder, leftPart, rightPart);
}
@@ -322,7 +322,7 @@ public final class ValueCompressionUtil {
(BigDecimalCompressionFinder) compressionFinder);
default:
return getUncompressedValue(compressionFinder.getCompType(),
- compressionFinder.getActualDataType(), compressionFinder.getChangedDataType());
+ compressionFinder.getActualDataType(), compressionFinder.getConvertedDataType());
}
}
@@ -727,7 +727,7 @@ public final class ValueCompressionUtil {
byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
WriterCompressModel compressionModel = new WriterCompressModel();
DataType[] actualType = new DataType[measureCount];
- DataType[] changedType = new DataType[measureCount];
+ DataType[] convertedType = new DataType[measureCount];
COMPRESSION_TYPE[] compType = new COMPRESSION_TYPE[measureCount];
CompressionFinder[] compressionFinders = new CompressionFinder[measureCount];
for (int i = 0; i < measureCount; i++) {
@@ -736,13 +736,13 @@ public final class ValueCompressionUtil {
minValue[i], mantissa[i], type[i], dataTypeSelected[i]);
compressionFinders[i] = compresssionFinder;
actualType[i] = compresssionFinder.getActualDataType();
- changedType[i] = compresssionFinder.getChangedDataType();
+ convertedType[i] = compresssionFinder.getConvertedDataType();
compType[i] = compresssionFinder.getCompType();
}
compressionModel.setCompressionFinders(compressionFinders);
compressionModel.setMaxValue(maxValue);
compressionModel.setMantissa(mantissa);
- compressionModel.setChangedDataType(changedType);
+ compressionModel.setConvertedDataType(convertedType);
compressionModel.setCompType(compType);
compressionModel.setActualDataType(actualType);
compressionModel.setMinValue(minValue);
@@ -765,7 +765,7 @@ public final class ValueCompressionUtil {
meta.getType(), meta.getDataTypeSelected());
compressModel.setUnCompressValues(
ValueCompressionUtil.getUncompressedValue(compressFinder));
- compressModel.setChangedDataType(compressFinder.getChangedDataType());
+ compressModel.setConvertedDataType(compressFinder.getConvertedDataType());
compressModel.setValueEncoderMeta(meta);
return compressModel;
}
[2/2] incubator-carbondata git commit: [CARBONDATA-549] code
improvements for bigint, decimal value compression This closes #453
Posted by ra...@apache.org.
[CARBONDATA-549] code improvements for bigint, decimal value compression This closes #453
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/956eb334
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/956eb334
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/956eb334
Branch: refs/heads/master
Commit: 956eb3348094d4deb327c3c4560c368a4dc7fb4e
Parents: a992940 b0b6bcb
Author: ravipesala <ra...@gmail.com>
Authored: Fri Dec 23 18:38:09 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Dec 23 18:38:09 2016 +0530
----------------------------------------------------------------------
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 4 +-
.../core/compression/BigDecimalCompressor.java | 14 +--
.../core/compression/BigIntCompressor.java | 124 +++++++++----------
.../core/compression/DoubleCompressor.java | 15 +--
.../core/compression/ValueCompressor.java | 30 ++---
.../store/compression/ReaderCompressModel.java | 10 +-
.../store/compression/WriterCompressModel.java | 14 +--
.../type/UnCompressBigDecimalByte.java | 4 +-
...yCompressedDoubleArrayDataInMemoryStore.java | 6 +-
.../core/util/BigDecimalCompressionFinder.java | 26 ++--
.../carbondata/core/util/CompressionFinder.java | 20 +--
.../core/util/ValueCompressionUtil.java | 20 +--
13 files changed, 142 insertions(+), 147 deletions(-)
----------------------------------------------------------------------