You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/01 11:43:06 UTC
[1/2] incubator-carbondata git commit: Implement BigInt value
compression
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 7213ac057 -> c95e565e4
Implement BigInt value compression
fix for review comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/20af74ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/20af74ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/20af74ba
Branch: refs/heads/master
Commit: 20af74ba719141002c68bc5e82bc05131e47a2d9
Parents: 7213ac0
Author: Ashok Kumar <as...@gmail.com>
Authored: Sun Jul 24 00:43:58 2016 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 1 19:38:55 2016 +0800
----------------------------------------------------------------------
.../core/compression/BigIntCompressor.java | 110 +++++++++
.../core/compression/DoubleCompressor.java | 242 +++++++++++++++++++
.../core/compression/ValueCompressor.java | 103 ++++++++
.../compression/type/UnCompressDefaultLong.java | 11 +-
.../compression/type/UnCompressMaxMinByte.java | 39 ++-
.../type/UnCompressMaxMinByteForLong.java | 8 +-
.../type/UnCompressMaxMinDefault.java | 13 +-
.../type/UnCompressMaxMinDefaultLong.java | 9 +-
.../compression/type/UnCompressMaxMinFloat.java | 10 +-
.../compression/type/UnCompressMaxMinInt.java | 36 ++-
.../compression/type/UnCompressMaxMinLong.java | 37 ++-
.../compression/type/UnCompressMaxMinShort.java | 36 ++-
.../compression/type/UnCompressNoneByte.java | 39 ++-
.../compression/type/UnCompressNoneDefault.java | 10 +-
.../compression/type/UnCompressNoneFloat.java | 11 +-
.../compression/type/UnCompressNoneInt.java | 29 ++-
.../compression/type/UnCompressNoneLong.java | 39 ++-
.../compression/type/UnCompressNoneShort.java | 38 ++-
...ractHeavyCompressedDoubleArrayDataStore.java | 17 +-
.../core/util/ValueCompressionUtil.java | 80 ++++--
.../core/util/ValueCompressionUtilTest.java | 8 +-
.../store/CarbonFactDataHandlerColumnar.java | 2 -
22 files changed, 844 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
new file mode 100644
index 0000000..7b9e52f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.compression;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * It compresses big int data
+ */
+public class BigIntCompressor extends ValueCompressor {
+
+ @Override protected Object compressNonDecimalMaxMin(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder, int decimal, Object max) {
+ // in case if bigint, decimal will be 0
+ return compressMaxMin(changedDataType, dataHolder, max);
+ }
+
+ @Override
+ protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ int decimal) {
+ // in case if bigint, decimal will be 0
+ return compressNone(changedDataType, dataHolder);
+ }
+
+ @Override
+ protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ Object max) {
+ long maxValue = (long) max;
+ long[] value = dataHolder.getWritableLongValues();
+ int i = 0;
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ result[i] = (byte) (maxValue - value[j]);
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ shortResult[i] = (short) (maxValue - value[j]);
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ intResult[i] = (int) (maxValue - value[j]);
+ i++;
+ }
+ return intResult;
+ default:
+ long[] defaultResult = new long[value.length];
+ for (int j = 0; j < value.length; j++) {
+ defaultResult[i] = (long) (maxValue - value[j]);
+ i++;
+ }
+ return defaultResult;
+ }
+ }
+
+ @Override
+ protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) {
+ long[] value = dataHolder.getWritableLongValues();
+ int i = 0;
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ result[i] = (byte) value[j];
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ shortResult[i] = (short) value[j];
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ intResult[i] = (int) value[j];
+ i++;
+ }
+ return intResult;
+ default:
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
new file mode 100644
index 0000000..8d20f18
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Double compressor
+ */
+public class DoubleCompressor extends ValueCompressor {
+
+ @Override protected Object compressNonDecimalMaxMin(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder, int decimal, Object maxValue) {
+ int i = 0;
+ BigDecimal max = BigDecimal.valueOf((double)maxValue);
+ double[] value = dataHolder.getWritableDoubleValues();
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ result[i] = (byte) (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ shortResult[i] = (short) (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ intResult[i] = (int) (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return intResult;
+ case DATA_LONG:
+ long[] longResult = new long[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ longResult[i] = (long) (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return longResult;
+ case DATA_FLOAT:
+ float[] floatResult = new float[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ floatResult[i] = (float) (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return floatResult;
+ default:
+ double[] defaultResult = new double[value.length];
+ for (int j = 0; j < value.length; j++) {
+ BigDecimal val = BigDecimal.valueOf(value[j]);
+ double diff = max.subtract(val).doubleValue();
+ defaultResult[i] = (Math.round(diff * Math.pow(10, decimal)));
+ i++;
+ }
+ return defaultResult;
+ }
+ }
+
+ @Override
+ protected Object compressNonDecimal(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ int decimal) {
+ int i = 0;
+ double[] value = dataHolder.getWritableDoubleValues();
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ result[i] = (byte) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ intResult[i] = (int) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return intResult;
+ case DATA_LONG:
+ long[] longResult = new long[value.length];
+ for (int j = 0; j < value.length; j++) {
+ longResult[i] = (long) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return longResult;
+ case DATA_FLOAT:
+ float[] floatResult = new float[value.length];
+ for (int j = 0; j < value.length; j++) {
+ floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return floatResult;
+ default:
+ double[] defaultResult = new double[value.length];
+ for (int j = 0; j < value.length; j++) {
+ defaultResult[i] = (double) (Math.round(Math.pow(10, decimal) * value[j]));
+ i++;
+ }
+ return defaultResult;
+ }
+ }
+
+ @Override
+ protected Object compressMaxMin(DataType changedDataType, CarbonWriteDataHolder dataHolder,
+ Object max) {
+ double maxValue = (double) max;
+ double[] value = dataHolder.getWritableDoubleValues();
+ int i = 0;
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ result[i] = (byte) (maxValue - value[j]);
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ shortResult[i] = (short) (maxValue - value[j]);
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ intResult[i] = (int) (maxValue - value[j]);
+ i++;
+ }
+ return intResult;
+ case DATA_LONG:
+ long[] longResult = new long[value.length];
+ for (int j = 0; j < value.length; j++) {
+ longResult[i] = (long) (maxValue - value[j]);
+ i++;
+ }
+ return longResult;
+ case DATA_FLOAT:
+ float[] floatResult = new float[value.length];
+ for (int j = 0; j < value.length; j++) {
+ floatResult[i] = (float) (maxValue - value[j]);
+ i++;
+ }
+ return floatResult;
+ default:
+ double[] defaultResult = new double[value.length];
+ for (int j = 0; j < value.length; j++) {
+ defaultResult[i] = (double) (maxValue - value[j]);
+ i++;
+ }
+ return defaultResult;
+ }
+ }
+
+ @Override
+ protected Object compressNone(DataType changedDataType, CarbonWriteDataHolder dataHolder) {
+ double[] value = dataHolder.getWritableDoubleValues();
+ int i = 0;
+ switch (changedDataType) {
+ case DATA_BYTE:
+ byte[] result = new byte[value.length];
+ for (int j = 0; j < value.length; j++) {
+ result[i] = (byte) value[j];
+ i++;
+ }
+ return result;
+ case DATA_SHORT:
+ short[] shortResult = new short[value.length];
+ for (int j = 0; j < value.length; j++) {
+ shortResult[i] = (short) value[j];
+ i++;
+ }
+ return shortResult;
+ case DATA_INT:
+ int[] intResult = new int[value.length];
+ for (int j = 0; j < value.length; j++) {
+ intResult[i] = (int) value[j];
+ i++;
+ }
+ return intResult;
+ case DATA_LONG:
+ case DATA_BIGINT:
+ long[] longResult = new long[value.length];
+ for (int j = 0; j < value.length; j++) {
+ longResult[i] = (long) value[j];
+ i++;
+ }
+ return longResult;
+ case DATA_FLOAT:
+ float[] floatResult = new float[value.length];
+ for (int j = 0; j < value.length; j++) {
+ floatResult[i] = (float) value[j];
+ i++;
+ }
+ return floatResult;
+ default:
+ return value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
new file mode 100644
index 0000000..5ecb90e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.compression;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Measure compressor
+ */
+public abstract class ValueCompressor {
+
+ /**
+ *
+ * @param compType
+ * @param dataHolder
+ * @param changedDataType
+ * @param maxValue
+ * @param decimal
+ * @return compressed data
+ */
+ public Object getCompressedValues(COMPRESSION_TYPE compType, CarbonWriteDataHolder dataHolder,
+ DataType changedDataType, Object maxValue, int decimal) {
+ Object o;
+ switch (compType) {
+ case NONE:
+
+ o = compressNone(changedDataType, dataHolder);
+ return o;
+
+ case MAX_MIN:
+
+ o = compressMaxMin(changedDataType, dataHolder, maxValue);
+ return o;
+
+ case NON_DECIMAL_CONVERT:
+
+ o = compressNonDecimal(changedDataType, dataHolder, decimal);
+ return o;
+
+ default:
+ o = compressNonDecimalMaxMin(changedDataType, dataHolder, decimal, maxValue);
+ return o;
+ }
+ }
+
+ /**
+ *
+ * @param changedDataType
+ * @param dataHolder
+ * @param decimal
+ * @param maxValue
+ * @return compressed data
+ */
+ protected abstract Object compressNonDecimalMaxMin(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder, int decimal, Object maxValue);
+
+ /**
+ *
+ * @param changedDataType
+ * @param dataHolder
+ * @param decimal
+ * @return compressed data
+ */
+ protected abstract Object compressNonDecimal(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder, int decimal);
+
+ /**
+ *
+ * @param changedDataType
+ * @param dataHolder
+ * @param maxValue
+ * @return compressed data
+ */
+ protected abstract Object compressMaxMin(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder, Object maxValue);
+
+ /**
+ *
+ * @param changedDataType
+ * @param dataHolder
+ * @return compressed data
+ */
+ protected abstract Object compressNone(DataType changedDataType,
+ CarbonWriteDataHolder dataHolder);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java
index a4d3b96..444fb5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressDefaultLong.java
@@ -22,18 +22,21 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressDefaultLong extends UnCompressNoneLong {
+ private static final LogService LOGGER = LogServiceFactory
+ .getLogService(UnCompressDefaultLong.class.getName());
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressDefaultLong.class.getName());
+ public UnCompressDefaultLong(DataType actualDataType) {
+ super(actualDataType);
+ }
public ValueCompressonHolder.UnCompressValue getNew() {
try {
return (ValueCompressonHolder.UnCompressValue) clone();
} catch (CloneNotSupportedException clnNotSupportedExc) {
- LOGGER.error(clnNotSupportedExc,
- clnNotSupportedExc.getMessage());
+ LOGGER.error(clnNotSupportedExc, clnNotSupportedExc.getMessage());
}
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java
index e6486c2..db7f085 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByte.java
@@ -46,8 +46,17 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
*/
protected byte[] value;
+ /**
+ * actual data type
+ */
+ protected DataType actualDataType;
+
//TODO SIMIAN
+ public UnCompressMaxMinByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(byte[] value) {
this.value = value;
@@ -64,13 +73,13 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
@Override public UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
byte1.setValue(byteCompressor.compress(value));
return byte1;
}
@Override public UnCompressValue uncompress(DataType dataType) {
- UnCompressValue byte1 = ValueCompressionUtil.unCompressMaxMin(dataType, dataType);
+ UnCompressValue byte1 = ValueCompressionUtil.unCompressMaxMin(dataType, actualDataType);
ValueCompressonHolder.unCompress(dataType, byte1, value);
return byte1;
}
@@ -87,10 +96,34 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong(decimal, maxValueObject);
+ default:
+ return unCompressDouble(decimal, maxValueObject);
+ }
+ }
+
+ private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) {
+ long maxValue = (long) maxValueObject;
+ long[] vals = new long[value.length];
+ CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ for (int i = 0; i < vals.length; i++) {
+ if (value[i] == 0) {
+ vals[i] = maxValue;
+ } else {
+ vals[i] = maxValue - value[i];
+ }
+ }
+ dataHolder.setReadableLongValues(vals);
+ return dataHolder;
+ }
+
+ private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) {
double maxValue = (double) maxValueObject;
double[] vals = new double[value.length];
CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java
index c265a44..b5d42b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinByteForLong.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte {
@@ -33,6 +34,9 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte {
private static Compressor<byte[]> byteCompressor =
SnappyCompression.SnappyByteCompression.INSTANCE;
+ public UnCompressMaxMinByteForLong(DataType actualDataType) {
+ super(actualDataType);
+ }
@Override public ValueCompressonHolder.UnCompressValue getNew() {
try {
return (ValueCompressonHolder.UnCompressValue) clone();
@@ -44,7 +48,7 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte {
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong();
+ UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(actualDataType);
byte1.setValue(byteCompressor.compress(value));
return byte1;
}
@@ -58,7 +62,7 @@ public class UnCompressMaxMinByteForLong extends UnCompressMaxMinByte {
}
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByteForLong();
+ return new UnCompressMaxMinByteForLong(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java
index df72c61..dea9858 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefault.java
@@ -48,6 +48,15 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
*/
private double[] value;
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ public UnCompressMaxMinDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(double[] value) {
this.value = (double[]) value;
@@ -63,7 +72,7 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
byte1.setValue(doubleCompressor.compress(value));
return byte1;
}
@@ -85,7 +94,7 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
//TODO SIMIAN
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java
index 57a25eb..376b487 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinDefaultLong.java
@@ -25,6 +25,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong {
@@ -33,6 +34,10 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong {
private static Compressor<long[]> longCompressor =
SnappyCompression.SnappyLongCompression.INSTANCE;
+ public UnCompressMaxMinDefaultLong(DataType actualDataType) {
+ super(actualDataType);
+ // TODO Auto-generated constructor stub
+ }
@Override public ValueCompressonHolder.UnCompressValue getNew() {
try {
return (ValueCompressonHolder.UnCompressValue) clone();
@@ -43,7 +48,7 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong {
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong();
+ UnCompressMaxMinByteForLong byte1 = new UnCompressMaxMinByteForLong(actualDataType);
byte1.setValue(longCompressor.compress(value));
return byte1;
}
@@ -53,7 +58,7 @@ public class UnCompressMaxMinDefaultLong extends UnCompressMaxMinLong {
}
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByteForLong();
+ return new UnCompressMaxMinByteForLong(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java
index 396c5c0..0958069 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinFloat.java
@@ -47,6 +47,12 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> {
*/
private float[] value;
+ private DataType actualDataType;
+
+ public UnCompressMaxMinFloat(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(float[] value) {
this.value = (float[]) value;
@@ -63,7 +69,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> {
@Override public UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
byte1.setValue(floatCompressor.compress(value));
return byte1;
}
@@ -85,7 +91,7 @@ public class UnCompressMaxMinFloat implements UnCompressValue<float[]> {
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java
index a2af61d..c25f75b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinInt.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValue<int[]> {
/**
@@ -45,6 +46,12 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
*/
private int[] value;
+ private DataType actualDataType;
+
+ public UnCompressMaxMinInt(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(int[] value) {
this.value = value;
@@ -60,7 +67,7 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
byte1.setValue(intCompressor.compress(value));
return byte1;
}
@@ -83,10 +90,19 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decVal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong(decVal, maxValueObject);
+ default:
+ return unCompressDouble(decVal, maxValueObject);
+ }
+ }
+
+ private CarbonReadDataHolder unCompressDouble(int decVal, Object maxValueObject) {
double maxValue = (double) maxValueObject;
double[] vals = new double[value.length];
CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
@@ -102,4 +118,20 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
return dataHolder;
}
+ private CarbonReadDataHolder unCompressLong(int decVal, Object maxValueObject) {
+ long maxValue = (long) maxValueObject;
+ long[] vals = new long[value.length];
+ CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ for (int i = 0; i < vals.length; i++) {
+ if (value[i] == 0) {
+ vals[i] = maxValue;
+ } else {
+ vals[i] = maxValue - value[i];
+ }
+
+ }
+ dataHolder.setReadableLongValues(vals);
+ return dataHolder;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java
index cab3aa2..0dc1c5b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinLong.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressValue<long[]> {
/**
@@ -45,6 +46,12 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
*/
protected long[] value;
+ protected DataType actualDataType;
+
+ public UnCompressMaxMinLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public ValueCompressonHolder.UnCompressValue getNew() {
try {
return (ValueCompressonHolder.UnCompressValue) clone();
@@ -55,7 +62,7 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByte unCompressByte = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte unCompressByte = new UnCompressMaxMinByte(actualDataType);
unCompressByte.setValue(longCompressor.compress(value));
return unCompressByte;
}
@@ -83,10 +90,20 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong(decimal, maxValueObject);
+ default:
+ return unCompressDouble(decimal, maxValueObject);
+ }
+
+ }
+
+ private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) {
double maxValue = (double) maxValueObject;
double[] vals = new double[value.length];
CarbonReadDataHolder data = new CarbonReadDataHolder();
@@ -102,4 +119,20 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
return data;
}
+ private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) {
+ long maxValue = (long) maxValueObject;
+ long[] vals = new long[value.length];
+ CarbonReadDataHolder data = new CarbonReadDataHolder();
+ for (int i = 0; i < vals.length; i++) {
+ if (value[i] == 0) {
+ vals[i] = maxValue;
+ } else {
+ vals[i] = maxValue - value[i];
+ }
+
+ }
+ data.setReadableLongValues(vals);
+ return data;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java
index 884c430..faca753 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressMaxMinShort.java
@@ -46,6 +46,12 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
*/
private short[] value;
+ private DataType actualDataType;
+
+ public UnCompressMaxMinShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(short[] value) {
this.value = value;
@@ -70,7 +76,7 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte();
+ UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
byte1.setValue(shortCompressor.compress(value));
return byte1;
}
@@ -84,10 +90,20 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong(decimal, maxValueObject);
+ default:
+ return unCompressDouble(decimal, maxValueObject);
+ }
+
+ }
+
+ private CarbonReadDataHolder unCompressDouble(int decimal, Object maxValueObject) {
double maxValue = (double) maxValueObject;
double[] vals = new double[value.length];
CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder();
@@ -103,4 +119,20 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
return carbonDataHolderObj;
}
+ private CarbonReadDataHolder unCompressLong(int decimal, Object maxValueObject) {
+ long maxValue = (long) maxValueObject;
+ long[] vals = new long[value.length];
+ CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder();
+ for (int i = 0; i < vals.length; i++) {
+ if (value[i] == 0) {
+ vals[i] = maxValue;
+ } else {
+ vals[i] = maxValue - value[i];
+ }
+
+ }
+ carbonDataHolderObj.setReadableLongValues(vals);
+ return carbonDataHolderObj;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java
index d689f93..91959a2 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneByte.java
@@ -47,6 +47,15 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> {
*/
private byte[] value;
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ public UnCompressNoneByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public UnCompressValue getNew() {
try {
return (UnCompressValue) clone();
@@ -61,13 +70,13 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> {
}
@Override public UnCompressValue uncompress(DataType dataType) {
- UnCompressValue byte1 = ValueCompressionUtil.unCompressNone(dataType, dataType);
+ UnCompressValue byte1 = ValueCompressionUtil.unCompressNone(dataType, actualDataType);
ValueCompressonHolder.unCompress(dataType, byte1, value);
return byte1;
}
@Override public UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
byte1.setValue(byteCompressor.compress(value));
return byte1;
}
@@ -84,12 +93,24 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> {
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong();
+ default:
+ return unCompressDouble();
+ }
+
+ }
+
+ private CarbonReadDataHolder unCompressDouble() {
CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
double[] vals = new double[value.length];
+
for (int i = 0; i < vals.length; i++) {
vals[i] = value[i];
}
@@ -97,4 +118,16 @@ public class UnCompressNoneByte implements UnCompressValue<byte[]> {
return dataHldr;
}
+ private CarbonReadDataHolder unCompressLong() {
+ CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
+ long[] vals = new long[value.length];
+
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = value[i];
+ }
+ dataHldr.setReadableLongValues(vals);
+ return dataHldr;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java
index c1395b4..abf6ae3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneDefault.java
@@ -46,6 +46,12 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> {
*/
private double[] value;
+ private DataType actualDataType;
+
+ public UnCompressNoneDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(double[] value) {
this.value = value;
@@ -61,7 +67,7 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> {
}
@Override public UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
byte1.setValue(doubleCompressor.compress(value));
return byte1;
@@ -75,7 +81,7 @@ public class UnCompressNoneDefault implements UnCompressValue<double[]> {
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(actualDataType);
}
@Override public byte[] getBackArrayData() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
index 103b1c9..6167c54 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValue<float[]> {
/**
@@ -45,6 +46,12 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu
*/
private float[] value;
+ private DataType actualDataType;
+
+ public UnCompressNoneFloat(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(float[] value) {
this.value = value;
@@ -60,7 +67,7 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
byte1.setValue(floatCompressor.compress(value));
return byte1;
@@ -76,7 +83,7 @@ public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValu
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(this.actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
index ca5593d..d12ebe1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
@@ -45,6 +45,12 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
*/
private int[] value;
+ private DataType actualDataType;
+
+ public UnCompressNoneInt(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(int[] value) {
this.value = value;
@@ -64,7 +70,7 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
byte1.setValue(intCompressor.compress(value));
return byte1;
@@ -84,10 +90,19 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(this.actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong();
+ default:
+ return unCompressDouble();
+ }
+ }
+
+ private CarbonReadDataHolder unCompressDouble() {
CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
double[] vals = new double[value.length];
for (int i = 0; i < vals.length; i++) {
@@ -98,4 +113,14 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
return dataHolderInfoObj;
}
+ private CarbonReadDataHolder unCompressLong() {
+ CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
+ long[] vals = new long[value.length];
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = value[i];
+ }
+
+ dataHolderInfoObj.setReadableLongValues(vals);
+ return dataHolderInfoObj;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
index 47220c5..4661b7a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
@@ -28,6 +28,7 @@ import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressio
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue<long[]> {
/**
@@ -45,6 +46,12 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
*/
protected long[] value;
+ private DataType actualDataType;
+
+ public UnCompressNoneLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(long[] value) {
this.value = value;
@@ -61,7 +68,7 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
byte1.setValue(longCompressor.compress(value));
return byte1;
@@ -85,17 +92,39 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(this.actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong();
+ default:
+ return unCompressDouble();
+ }
+ }
+
+ private CarbonReadDataHolder unCompressDouble() {
+ CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
double[] vals = new double[value.length];
+
for (int i = 0; i < vals.length; i++) {
vals[i] = value[i];
}
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ dataHldr.setReadableDoubleValues(vals);
+ return dataHldr;
}
+ private CarbonReadDataHolder unCompressLong() {
+ CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
+ long[] vals = new long[value.length];
+
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = value[i];
+ }
+ dataHldr.setReadableLongValues(vals);
+ return dataHldr;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
index ed41ca5..2fdf514 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
@@ -48,6 +48,12 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
*/
private short[] shortValue;
+ private DataType actualDataType;
+
+ public UnCompressNoneShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
@Override public void setValue(short[] shortValue) {
this.shortValue = shortValue;
@@ -64,7 +70,7 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte();
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
byte1.setValue(shortCompressor.compress(shortValue));
return byte1;
@@ -88,17 +94,39 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
* @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
*/
@Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(this.actualDataType);
}
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return unCompressLong();
+ default:
+ return unCompressDouble();
+ }
+ }
+
+ private CarbonReadDataHolder unCompressDouble() {
+ CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
double[] vals = new double[shortValue.length];
+
for (int i = 0; i < vals.length; i++) {
vals[i] = shortValue[i];
}
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ dataHldr.setReadableDoubleValues(vals);
+ return dataHldr;
}
+ private CarbonReadDataHolder unCompressLong() {
+ CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
+
+ long[] vals = new long[shortValue.length];
+
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = shortValue[i];
+ }
+ dataHldr.setReadableLongValues(vals);
+ return dataHldr;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
index 21c2a60..6051736 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -64,17 +64,12 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
values[i] = compressionModel.getUnCompressValues()[i].getNew();
if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE
&& type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- values[i].setValue(ValueCompressionUtil
- .getCompressedValues(compressionModel.getCompType()[i],
- dataHolder[i].getWritableLongValues(), compressionModel.getChangedDataType()[i],
- (long) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i]));
- } else {
- values[i].setValue(ValueCompressionUtil
- .getCompressedValues(compressionModel.getCompType()[i],
- dataHolder[i].getWritableDoubleValues(), compressionModel.getChangedDataType()[i],
- (double) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i]));
- }
+
+ values[i].setValue(
+ ValueCompressionUtil.getValueCompressor(compressionModel.getActualDataType()[i])
+ .getCompressedValues(compressionModel.getCompType()[i], dataHolder[i],
+ compressionModel.getChangedDataType()[i], compressionModel.getMaxValue()[i],
+ compressionModel.getDecimal()[i]));
} else {
values[i].setValue(dataHolder[i].getWritableByteArrayValues());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index d3448f1..5e68411 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -25,11 +25,13 @@ import java.util.Arrays;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.compression.BigIntCompressor;
+import org.apache.carbondata.core.compression.DoubleCompressor;
+import org.apache.carbondata.core.compression.ValueCompressor;
import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressByteArray;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressDefaultLong;
import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinByte;
import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinDefault;
import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinFloat;
@@ -149,8 +151,8 @@ public final class ValueCompressionUtil {
return new CompressionFinder(COMPRESSION_TYPE.CUSTOM_BIGDECIMAL, DataType.DATA_BYTE,
DataType.DATA_BYTE);
case 'l':
- return new CompressionFinder(COMPRESSION_TYPE.NONE,
- DataType.DATA_BIGINT, DataType.DATA_BIGINT);
+ return getLongCompressorFinder(maxValue, minValue, decimal, aggregatorType,
+ dataTypeSelected);
default:
break;
}
@@ -199,6 +201,22 @@ public final class ValueCompressionUtil {
}
}
+ private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue,
+ int decimal, char aggregatorType, byte dataTypeSelected) {
+ if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) > getSize(
+ getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) {
+ return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_BIGINT,
+ getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected));
+ } else if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) < getSize(
+ getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) {
+ return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT,
+ getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected));
+ } else {
+ return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT,
+ getDataType((long) maxValue, decimal, dataTypeSelected));
+ }
+ }
+
/**
* @param compType : compression type
* @param values : the data of one measure
@@ -243,6 +261,20 @@ public final class ValueCompressionUtil {
}
}
+ /**
+ * It returns Compressor for given datatype
+ * @param actualDataType
+ * @return compressor based on actualdatatype
+ */
+ public static ValueCompressor getValueCompressor(DataType actualDataType) {
+ switch (actualDataType) {
+ case DATA_BIGINT:
+ return new BigIntCompressor();
+ default:
+ return new DoubleCompressor();
+ }
+ }
+
private static ValueCompressonHolder.UnCompressValue[] getUncompressedValues(
COMPRESSION_TYPE[] compType, DataType[] actualDataType, DataType[] changedDataType) {
@@ -569,34 +601,30 @@ public final class ValueCompressionUtil {
*/
public static ValueCompressonHolder.UnCompressValue unCompressNone(DataType compDataType,
DataType actualDataType) {
- if (actualDataType == DataType.DATA_BIGINT) {
- return new UnCompressDefaultLong();
- } else {
- switch (compDataType) {
- case DATA_BYTE:
+ switch (compDataType) {
+ case DATA_BYTE:
- return new UnCompressNoneByte();
+ return new UnCompressNoneByte(actualDataType);
- case DATA_SHORT:
+ case DATA_SHORT:
- return new UnCompressNoneShort();
+ return new UnCompressNoneShort(actualDataType);
- case DATA_INT:
+ case DATA_INT:
- return new UnCompressNoneInt();
+ return new UnCompressNoneInt(actualDataType);
- case DATA_LONG:
+ case DATA_LONG:
- return new UnCompressNoneLong();
+ return new UnCompressNoneLong(actualDataType);
- case DATA_FLOAT:
+ case DATA_FLOAT:
- return new UnCompressNoneFloat();
+ return new UnCompressNoneFloat(actualDataType);
- default:
+ default:
- return new UnCompressNoneDefault();
- }
+ return new UnCompressNoneDefault(actualDataType);
}
}
@@ -608,27 +636,27 @@ public final class ValueCompressionUtil {
switch (compDataType) {
case DATA_BYTE:
- return new UnCompressMaxMinByte();
+ return new UnCompressMaxMinByte(actualDataType);
case DATA_SHORT:
- return new UnCompressMaxMinShort();
+ return new UnCompressMaxMinShort(actualDataType);
case DATA_INT:
- return new UnCompressMaxMinInt();
+ return new UnCompressMaxMinInt(actualDataType);
case DATA_LONG:
- return new UnCompressMaxMinLong();
+ return new UnCompressMaxMinLong(actualDataType);
case DATA_FLOAT:
- return new UnCompressMaxMinFloat();
+ return new UnCompressMaxMinFloat(actualDataType);
default:
- return new UnCompressMaxMinDefault();
+ return new UnCompressMaxMinDefault(actualDataType);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index db64011..25d1426 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -320,7 +320,7 @@ public class ValueCompressionUtilTest {
@Test public void testToUnCompressNone() {
ValueCompressonHolder.UnCompressValue result =
ValueCompressionUtil.unCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT);
- assertEquals(result.getClass(), UnCompressDefaultLong.class);
+ assertEquals(result.getClass(), UnCompressNoneDefault.class);
}
@Test public void testToUnCompressNoneForByte() {
@@ -563,10 +563,10 @@ public class ValueCompressionUtilTest {
}
@Test public void testToGetValueCompressionModel() {
- Object[] maxValues = { 10, 20, 30 };
- Object[] minValues = { 1, 2, 3 };
+ Object[] maxValues = { 10, 20, 30l };
+ Object[] minValues = { 1, 2, 3l };
int[] decimalLength = { 0, 0, 0 };
- Object[] uniqueValues = { 5, 3, 2 };
+ Object[] uniqueValues = { 5, 3, 2l };
char[] types = { 'c', 'b', 'l' };
byte[] dataTypeSelected = { 1, 2, 4 };
MeasureMetaDataModel measureMetaDataModel =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/20af74ba/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 1fa14f0..e560784 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -1195,8 +1195,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
long minVal = (long) min[count];
max[count] = (maxVal > value ? max[count] : value);
min[count] = (minVal < value ? min[count] : value);
- int num = getDecimalCount(value);
- decimal[count] = (decimal[count] > num ? decimal[count] : num);
} else if (type[count] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
byte[] buff = null;
// in compaction flow the measure with decimal type will come as spark decimal.
[2/2] incubator-carbondata git commit: [CARBONDATA-100]Implement
BigInt value compression This closes #338
Posted by ja...@apache.org.
[CARBONDATA-100]Implement BigInt value compression This closes #338
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c95e565e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c95e565e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c95e565e
Branch: refs/heads/master
Commit: c95e565e471dc1352a169368cc7d06e68abc5599
Parents: 7213ac0 20af74b
Author: jackylk <ja...@huawei.com>
Authored: Thu Dec 1 19:42:52 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Dec 1 19:42:52 2016 +0800
----------------------------------------------------------------------
.../core/compression/BigIntCompressor.java | 110 +++++++++
.../core/compression/DoubleCompressor.java | 242 +++++++++++++++++++
.../core/compression/ValueCompressor.java | 103 ++++++++
.../compression/type/UnCompressDefaultLong.java | 11 +-
.../compression/type/UnCompressMaxMinByte.java | 39 ++-
.../type/UnCompressMaxMinByteForLong.java | 8 +-
.../type/UnCompressMaxMinDefault.java | 13 +-
.../type/UnCompressMaxMinDefaultLong.java | 9 +-
.../compression/type/UnCompressMaxMinFloat.java | 10 +-
.../compression/type/UnCompressMaxMinInt.java | 36 ++-
.../compression/type/UnCompressMaxMinLong.java | 37 ++-
.../compression/type/UnCompressMaxMinShort.java | 36 ++-
.../compression/type/UnCompressNoneByte.java | 39 ++-
.../compression/type/UnCompressNoneDefault.java | 10 +-
.../compression/type/UnCompressNoneFloat.java | 11 +-
.../compression/type/UnCompressNoneInt.java | 29 ++-
.../compression/type/UnCompressNoneLong.java | 39 ++-
.../compression/type/UnCompressNoneShort.java | 38 ++-
...ractHeavyCompressedDoubleArrayDataStore.java | 17 +-
.../core/util/ValueCompressionUtil.java | 80 ++++--
.../core/util/ValueCompressionUtilTest.java | 8 +-
.../store/CarbonFactDataHandlerColumnar.java | 2 -
22 files changed, 844 insertions(+), 83 deletions(-)
----------------------------------------------------------------------