You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/01/08 14:30:22 UTC
[4/7] incubator-carbondata git commit: Added support for offheap
storage in query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
index d9b627f..62d3b53 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
@@ -20,19 +20,38 @@
package org.apache.carbondata.core.datastorage.store.compression;
import java.io.IOException;
+import java.lang.reflect.Field;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
public class SnappyCompressor implements Compressor {
private static final LogService LOGGER =
LogServiceFactory.getLogService(SnappyCompressor.class.getName());
- @Override
- public byte[] compressByte(byte[] unCompInput) {
+ private final SnappyNative snappyNative;
+
+ public SnappyCompressor() {
+ Snappy snappy = new Snappy();
+ Field privateField = null;
+ try {
+ privateField = snappy.getClass().getDeclaredField("impl");
+ } catch (NoSuchFieldException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
+ privateField.setAccessible(true);
+ try {
+ snappyNative = (SnappyNative) privateField.get(snappy);
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput) {
try {
return Snappy.rawCompress(unCompInput, unCompInput.length);
} catch (IOException e) {
@@ -41,8 +60,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public byte[] unCompressByte(byte[] compInput) {
+ @Override public byte[] unCompressByte(byte[] compInput) {
try {
return Snappy.uncompress(compInput);
} catch (IOException e) {
@@ -51,8 +69,20 @@ public class SnappyCompressor implements Compressor {
return compInput;
}
- @Override
- public byte[] compressShort(short[] unCompInput) {
+ @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+ int uncompressedLength = 0;
+ byte[] data = null;
+ try {
+ uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ data = new byte[uncompressedLength];
+ Snappy.uncompress(compInput, offset, length, data, 0);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return data;
+ }
+
+ @Override public byte[] compressShort(short[] unCompInput) {
try {
return Snappy.compress(unCompInput);
} catch (IOException e) {
@@ -61,8 +91,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public short[] unCompressShort(byte[] compInput) {
+ @Override public short[] unCompressShort(byte[] compInput) {
try {
return Snappy.uncompressShortArray(compInput);
} catch (IOException e) {
@@ -71,8 +100,16 @@ public class SnappyCompressor implements Compressor {
return null;
}
- @Override
- public byte[] compressInt(int[] unCompInput) {
+ @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+ try {
+ return Snappy.uncompressShortArray(compInput, offset, lenght);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressInt(int[] unCompInput) {
try {
return Snappy.compress(unCompInput);
} catch (IOException e) {
@@ -81,8 +118,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public int[] unCompressInt(byte[] compInput) {
+ @Override public int[] unCompressInt(byte[] compInput) {
try {
return Snappy.uncompressIntArray(compInput);
} catch (IOException e) {
@@ -91,8 +127,16 @@ public class SnappyCompressor implements Compressor {
return null;
}
- @Override
- public byte[] compressLong(long[] unCompInput) {
+ @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressIntArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressLong(long[] unCompInput) {
try {
return Snappy.compress(unCompInput);
} catch (IOException e) {
@@ -101,8 +145,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public long[] unCompressLong(byte[] compInput) {
+ @Override public long[] unCompressLong(byte[] compInput) {
try {
return Snappy.uncompressLongArray(compInput);
} catch (IOException e) {
@@ -111,8 +154,16 @@ public class SnappyCompressor implements Compressor {
return null;
}
- @Override
- public byte[] compressFloat(float[] unCompInput) {
+ @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressLongArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressFloat(float[] unCompInput) {
try {
return Snappy.compress(unCompInput);
} catch (IOException e) {
@@ -121,8 +172,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public float[] unCompressFloat(byte[] compInput) {
+ @Override public float[] unCompressFloat(byte[] compInput) {
try {
return Snappy.uncompressFloatArray(compInput);
} catch (IOException e) {
@@ -131,8 +181,16 @@ public class SnappyCompressor implements Compressor {
return null;
}
- @Override
- public byte[] compressDouble(double[] unCompInput) {
+ @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressFloatArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressDouble(double[] unCompInput) {
try {
return Snappy.compress(unCompInput);
} catch (IOException e) {
@@ -141,8 +199,7 @@ public class SnappyCompressor implements Compressor {
}
}
- @Override
- public double[] unCompressDouble(byte[] compInput) {
+ @Override public double[] unCompressDouble(byte[] compInput) {
try {
return Snappy.uncompressDoubleArray(compInput);
} catch (IOException e) {
@@ -150,4 +207,16 @@ public class SnappyCompressor implements Compressor {
}
return null;
}
+
+ @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+ try {
+ int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ double[] result = new double[uncompressedLength / 8];
+ snappyNative.rawUncompress(compInput, offset, length, result, 0);
+ return result;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressonHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressonHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressonHolder.java
index f161b9a..eca12ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressonHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressonHolder.java
@@ -19,7 +19,8 @@
package org.apache.carbondata.core.datastorage.store.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import java.math.BigDecimal;
+
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
@@ -30,7 +31,7 @@ public final class ValueCompressonHolder {
/**
* byteCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
private ValueCompressonHolder() {
@@ -41,26 +42,33 @@ public final class ValueCompressonHolder {
* @param value
* @param data
*/
- public static void unCompress(DataType dataType, UnCompressValue value, byte[] data) {
+ public static void unCompress(DataType dataType, UnCompressValue value, byte[] data, int offset,
+ int length, int decimal, Object maxValueObject) {
switch (dataType) {
case DATA_BYTE:
- value.setValue(compressor.unCompressByte(data));
+ value.setUncompressValues(compressor.unCompressByte(data, offset, length), decimal,
+ maxValueObject);
break;
case DATA_SHORT:
- value.setValue(compressor.unCompressShort(data));
+ value.setUncompressValues(compressor.unCompressShort(data, offset, length), decimal,
+ maxValueObject);
break;
case DATA_INT:
- value.setValue(compressor.unCompressInt(data));
+ value.setUncompressValues(compressor.unCompressInt(data, offset, length), decimal,
+ maxValueObject);
break;
case DATA_LONG:
case DATA_BIGINT:
- value.setValue(compressor.unCompressLong(data));
+ value.setUncompressValues(compressor.unCompressLong(data, offset, length), decimal,
+ maxValueObject);
break;
case DATA_FLOAT:
- value.setValue(compressor.unCompressFloat(data));
+ value.setUncompressValues(compressor.unCompressFloat(data, offset, length), decimal,
+ maxValueObject);
break;
default:
- value.setValue(compressor.unCompressDouble(data));
+ value.setUncompressValues(compressor.unCompressDouble(data, offset, length), decimal,
+ maxValueObject);
break;
}
}
@@ -78,13 +86,22 @@ public final class ValueCompressonHolder {
UnCompressValue compress();
- UnCompressValue uncompress(DataType dataType);
+ UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimal, Object maxValueObject);
+
+ void setUncompressValues(T data, int decimal, Object maxValueObject);
byte[] getBackArrayData();
UnCompressValue getCompressorObject();
- CarbonReadDataHolder getValues(int decimal, Object maxValue);
+ long getLongValue(int index);
+
+ double getDoubleValue(int index);
+
+ BigDecimal getBigDecimalValue(int index);
+
+ void freeMemory();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
deleted file mode 100644
index 1b9b163..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressByteArray.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class UnCompressByteArray implements ValueCompressonHolder.UnCompressValue<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressMaxMinByte.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance();
-
- private ByteArrayType arrayType;
- /**
- * value.
- */
- private byte[] value;
-
- public UnCompressByteArray(ByteArrayType type) {
- if (type == ByteArrayType.BYTE_ARRAY) {
- arrayType = ByteArrayType.BYTE_ARRAY;
- } else {
- arrayType = ByteArrayType.BIG_DECIMAL;
- }
-
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
-
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
-
- }
-
- @Override public ValueCompressonHolder.UnCompressValue<byte[]> getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressByteArray byte1 = new UnCompressByteArray(arrayType);
- byte1.setValue(compressor.compressByte(value));
- return byte1;
- }
-
- @Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dataType) {
- ValueCompressonHolder.UnCompressValue byte1 = new UnCompressByteArray(arrayType);
- byte1.setValue(compressor.unCompressByte(value));
- return byte1;
- }
-
- @Override public byte[] getBackArrayData() {
- return this.value;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressByteArray(arrayType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- List<byte[]> valsList = new ArrayList<byte[]>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- ByteBuffer buffer = ByteBuffer.wrap(value);
- buffer.rewind();
- int length = 0;
- byte[] actualValue = null;
- //CHECKSTYLE:OFF Approval No:Approval-367
- while (buffer.hasRemaining()) {//CHECKSTYLE:ON
- length = buffer.getInt();
- actualValue = new byte[length];
- buffer.get(actualValue);
- valsList.add(actualValue);
-
- }
- CarbonReadDataHolder holder = new CarbonReadDataHolder();
- byte[][] value = new byte[valsList.size()][];
- valsList.toArray(value);
- if (arrayType == ByteArrayType.BIG_DECIMAL) {
- BigDecimal[] bigDecimalValues = new BigDecimal[value.length];
- for (int i = 0; i < value.length; i++) {
- bigDecimalValues[i] = DataTypeUtil.byteToBigDecimal(value[i]);
- }
- holder.setReadableBigDecimalValues(bigDecimalValues);
- return holder;
- }
- holder.setReadableByteValues(value);
- return holder;
- }
-
- public enum ByteArrayType {
- BYTE_ARRAY,
- BIG_DECIMAL
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinByte.java
index 50caa62..637e35b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinByte.java
@@ -1,5 +1,5 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one
+3 * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
@@ -19,13 +19,16 @@
package org.apache.carbondata.core.datastorage.store.compression.decimal;
+import java.math.BigDecimal;
+
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -39,17 +42,21 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
protected byte[] value;
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
/**
* actual data type
*/
protected DataType actualDataType;
+ private double maxValue;
+
//TODO SIMIAN
public UnCompressMaxMinByte(DataType actualDataType) {
@@ -76,10 +83,13 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
return byte1;
}
- @Override public UnCompressValue uncompress(DataType dataType) {
- UnCompressValue byte1 = ValueCompressionUtil.getUnCompressDecimalMaxMin(dataType,
- actualDataType);
- ValueCompressonHolder.unCompress(dataType, byte1, value);
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ UnCompressValue byte1 =
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(dataType, actualDataType);
+ ValueCompressonHolder
+ .unCompress(dataType, byte1, compressData, offset, length, decimalPlaces, maxValueObject);
return byte1;
}
@@ -98,38 +108,40 @@ public class UnCompressMaxMinByte implements UnCompressValue<byte[]> {
return new UnCompressMaxMinByte(actualDataType);
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong(maxValueObject);
- default:
- return unCompressDouble(maxValueObject);
+ @Override public long getLongValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ if (byteValue == (byte) 0) {
+ return (long) maxValue;
+ } else {
+ return (short) (maxValue - byteValue);
}
}
- private CarbonReadDataHolder unCompressLong(Object maxValueObject) {
- long maxValue = (long) maxValueObject;
- long[] vals = new long[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- if (value[i] == 0) {
- vals[i] = maxValue;
- } else {
- vals[i] = maxValue - value[i];
- }
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ if (byteValue == (byte) 0) {
+ return maxValue;
+ } else {
+ return (short) (maxValue - byteValue);
}
- dataHolder.setReadableLongValues(vals);
- return dataHolder;
}
- private CarbonReadDataHolder unCompressDouble(Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override public void setUncompressValues(byte[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
}
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinDefault.java
index dc03694..5d5c057 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinDefault.java
@@ -19,14 +19,17 @@
package org.apache.carbondata.core.datastorage.store.compression.decimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -41,17 +44,21 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private double[] value;
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
/**
* actual data type
*/
private DataType actualDataType;
+ private double maxValue;
+
public UnCompressMaxMinDefault(DataType actualDataType) {
this.actualDataType = actualDataType;
}
@@ -76,7 +83,9 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
return byte1;
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -96,16 +105,36 @@ public class UnCompressMaxMinDefault implements ValueCompressonHolder.UnCompress
return new UnCompressMaxMinByte(actualDataType);
}
- //TODO SIMIAN
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ if (doubleValue == 0) {
+ return maxValue;
+ } else {
+ return maxValue - doubleValue;
+ }
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override
+ public void setUncompressValues(double[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
}
- dataHolderInfoObj.setReadableDoubleValues(vals);
- return dataHolderInfoObj;
}
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
deleted file mode 100644
index 44bb60d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinFloat.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class UnCompressMaxMinFloat implements UnCompressValue<float[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressMaxMinFloat.class.getName());
- /**
- * floatCompressor
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
- private float[] value;
-
- private DataType actualDataType;
-
- public UnCompressMaxMinFloat(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(float[] value) {
- this.value = value;
-
- }
-
- @Override public UnCompressValue getNew() {
- try {
- return (UnCompressValue) clone();
- } catch (CloneNotSupportedException ex4) {
- LOGGER.error(ex4, ex4.getMessage());
- }
- return null;
- }
-
- @Override public UnCompressValue compress() {
- UnCompressMaxMinByte byte1 = new UnCompressMaxMinByte(actualDataType);
- byte1.setValue(compressor.compressFloat(value));
- return byte1;
- }
-
- @Override public UnCompressValue uncompress(DataType dTypeVal) {
- return null;
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToFloatArray(buffer, value.length);
- }
-
- @Override public UnCompressValue getCompressorObject() {
- return new UnCompressMaxMinByte(actualDataType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolderVal = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
- }
- dataHolderVal.setReadableDoubleValues(vals);
- return dataHolderVal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinInt.java
index 8e42c64..f58f8a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinInt.java
@@ -19,14 +19,17 @@
package org.apache.carbondata.core.datastorage.store.compression.decimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,7 +43,10 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
/**
* value.
*/
@@ -48,6 +54,8 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
private DataType actualDataType;
+ private double maxValue;
+
public UnCompressMaxMinInt(DataType actualDataType) {
this.actualDataType = actualDataType;
}
@@ -72,8 +80,9 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
return byte1;
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(
- ValueCompressionUtil.DataType dataTypeValue) {
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -93,35 +102,40 @@ public class UnCompressMaxMinInt implements ValueCompressonHolder.UnCompressValu
return new UnCompressMaxMinByte(actualDataType);
}
- @Override public CarbonReadDataHolder getValues(int decVal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong(maxValueObject);
- default:
- return unCompressDouble(maxValueObject);
+ @Override public long getLongValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ if (intValue == 0) {
+ return (long) maxValue;
+ } else {
+ return (long) maxValue - intValue;
}
}
- private CarbonReadDataHolder unCompressDouble(Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ if (intValue == 0) {
+ return maxValue;
+ } else {
+ return maxValue - intValue;
}
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
}
- private CarbonReadDataHolder unCompressLong(Object maxValueObject) {
- long maxValue = (long) maxValueObject;
- long[] vals = new long[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override public void setUncompressValues(int[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
}
- dataHolder.setReadableLongValues(vals);
- return dataHolder;
}
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinLong.java
index 5a266fa..6d64c73 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinLong.java
@@ -19,14 +19,17 @@
package org.apache.carbondata.core.datastorage.store.compression.decimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -39,7 +42,9 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
/**
* value.
*/
@@ -47,6 +52,8 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
protected DataType actualDataType;
+ private double maxValue;
+
public UnCompressMaxMinLong(DataType actualDataType) {
this.actualDataType = actualDataType;
}
@@ -72,7 +79,8 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
}
@Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dataType) {
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -92,35 +100,40 @@ public class UnCompressMaxMinLong implements ValueCompressonHolder.UnCompressVal
return new UnCompressMaxMinByte(actualDataType);
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong(maxValueObject);
- default:
- return unCompressDouble(maxValueObject);
+ @Override public long getLongValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ if (longValue == 0) {
+ return (long) maxValue;
+ } else {
+ return (long) maxValue - longValue;
}
}
- private CarbonReadDataHolder unCompressDouble(Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder data = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public double getDoubleValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ if (longValue == 0) {
+ return maxValue;
+ } else {
+ return maxValue - longValue;
}
- data.setReadableDoubleValues(vals);
- return data;
}
- private CarbonReadDataHolder unCompressLong(Object maxValueObject) {
- long maxValue = (long) maxValueObject;
- long[] vals = new long[value.length];
- CarbonReadDataHolder data = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override public void setUncompressValues(long[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
}
- data.setReadableLongValues(vals);
- return data;
}
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinShort.java
index edade84..a138379 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/UnCompressMaxMinShort.java
@@ -19,14 +19,17 @@
package org.apache.carbondata.core.datastorage.store.compression.decimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -39,13 +42,16 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
/**
* shortCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
/**
* value.
*/
private short[] value;
private DataType actualDataType;
+ private double maxValue;
public UnCompressMaxMinShort(DataType actualDataType) {
this.actualDataType = actualDataType;
@@ -56,7 +62,9 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -91,36 +99,41 @@ public class UnCompressMaxMinShort implements ValueCompressonHolder.UnCompressVa
return new UnCompressMaxMinByte(actualDataType);
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong(maxValueObject);
- default:
- return unCompressDouble(maxValueObject);
+ @Override public long getLongValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ if (shortValue == 0) {
+ return (long) maxValue;
+ } else {
+ return (long) maxValue - shortValue;
}
-
}
- private CarbonReadDataHolder unCompressDouble(Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public double getDoubleValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ if (shortValue == 0) {
+ return maxValue;
+ } else {
+ return maxValue - shortValue;
}
- carbonDataHolderObj.setReadableDoubleValues(vals);
- return carbonDataHolderObj;
}
- private CarbonReadDataHolder unCompressLong(Object maxValueObject) {
- long maxValue = (long) maxValueObject;
- long[] vals = new long[value.length];
- CarbonReadDataHolder carbonDataHolderObj = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = maxValue - value[i];
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override
+ public void setUncompressValues(short[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
}
- carbonDataHolderObj.setReadableLongValues(vals);
- return carbonDataHolderObj;
}
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalByte.java
index 9718609..da11443 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalByte.java
@@ -19,12 +19,15 @@
package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
+import java.math.BigDecimal;
+
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -37,12 +40,16 @@ public class UnCompressNonDecimalByte implements ValueCompressonHolder.UnCompres
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private byte[] value;
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private double divisionFactory;
+
@Override public void setValue(byte[] value) {
this.value = value;
}
@@ -62,10 +69,13 @@ public class UnCompressNonDecimalByte implements ValueCompressonHolder.UnCompres
return byte1;
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType, byte[] compressData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
ValueCompressonHolder.UnCompressValue byte1 =
ValueCompressionUtil.getUnCompressNonDecimal(dataType);
- ValueCompressonHolder.unCompress(dataType, byte1, value);
+ ValueCompressonHolder
+ .unCompress(dataType, byte1, compressData, offset, length, decimalPlaces, maxValueObject);
return byte1;
}
@@ -84,13 +94,26 @@ public class UnCompressNonDecimalByte implements ValueCompressonHolder.UnCompres
return new UnCompressNonDecimalByte();
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimal);
- }
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getByte(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override public void setUncompressValues(byte[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalDefault.java
index b345700..d5c08be 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalDefault.java
@@ -19,14 +19,17 @@
package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,12 +43,15 @@ public class UnCompressNonDecimalDefault
/**
* doubleCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private double[] value;
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+ private double divisionFactory;
+
@Override public ValueCompressonHolder.UnCompressValue getNew() {
try {
return (ValueCompressonHolder.UnCompressValue) clone();
@@ -61,7 +67,9 @@ public class UnCompressNonDecimalDefault
return byte1;
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -83,14 +91,27 @@ public class UnCompressNonDecimalDefault
return new UnCompressNonDecimalByte();
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double[] dblVals = new double[value.length];
- for (int i = 0; i < dblVals.length; i++) {
- dblVals[i] = value[i] / Math.pow(10, decimal);
- }
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- dataHolder.setReadableDoubleValues(dblVals);
- return dataHolder;
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
}
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getDouble(index) / divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override
+ public void setUncompressValues(double[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalFloat.java
deleted file mode 100644
index ef126ff..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalFloat.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class UnCompressNonDecimalFloat implements ValueCompressonHolder.UnCompressValue<float[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNonDecimalFloat.class.getName());
- /**
- * floatCompressor
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
-
- private float[] value;
-
- @Override public void setValue(float[] value) {
- this.value = value;
-
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException cnsexception) {
- LOGGER
- .error(cnsexception, cnsexception.getMessage());
- }
- return null;
- }
-
- public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNonDecimalByte byte1 = new UnCompressNonDecimalByte();
- byte1.setValue(compressor.compressFloat(value));
- return byte1;
- }
-
- @Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dataType) {
- return null;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToFloatArray(buffer, value.length);
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNonDecimalByte();
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double[] vals = new double[value.length];
- for (int m = 0; m < vals.length; m++) {
- vals[m] = value[m] / Math.pow(10, decimal);
- }
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalInt.java
index cd47bff..c81aef1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalInt.java
@@ -19,14 +19,16 @@
package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -39,12 +41,16 @@ public class UnCompressNonDecimalInt implements UnCompressValue<int[]> {
/**
* intCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private int[] value;
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ private double divisionFactory;
+
@Override public void setValue(int[] value) {
this.value = value;
}
@@ -77,18 +83,33 @@ public class UnCompressNonDecimalInt implements UnCompressValue<int[]> {
return new UnCompressNonDecimalByte();
}
- @Override public CarbonReadDataHolder getValues(int mantissa, Object maxValueObject) {
- double[] vals = new double[value.length];
- for (int k = 0; k < vals.length; k++) {
- vals[k] = value[k] / Math.pow(10, mantissa);
- }
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ return null;
}
- @Override public UnCompressValue uncompress(DataType dataType) {
- return null;
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getInt(index) / this.divisionFactory);
}
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimalPlaces value is not supported");
+ }
+
+ @Override public void setUncompressValues(int[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalLong.java
index c717846..f9935cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalLong.java
@@ -19,14 +19,16 @@
package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,13 +42,17 @@ public class UnCompressNonDecimalLong implements UnCompressValue<long[]> {
/**
* longCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private long[] value;
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ private double divisionFactory;
+
@Override public void setValue(long[] value) {
this.value = value;
}
@@ -66,7 +72,9 @@ public class UnCompressNonDecimalLong implements UnCompressValue<long[]> {
return null;
}
- @Override public UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public UnCompressValue uncompress(DataType dataType, byte[] compressData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
return null;
}
@@ -83,14 +91,26 @@ public class UnCompressNonDecimalLong implements UnCompressValue<long[]> {
return new UnCompressNonDecimalByte();
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double[] vals = new double[value.length];
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimal);
- }
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getLong(index) / this.divisionFactory);
}
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimalPlaces value is not supported");
+ }
+
+ @Override public void setUncompressValues(long[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinByte.java
index 42629ff..dcc7850 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinByte.java
@@ -23,10 +23,11 @@ import java.math.BigDecimal;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,18 +41,23 @@ public class UnCompressNonDecimalMaxMinByte
/**
* compressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private byte[] value;
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
@Override public ValueCompressonHolder.UnCompressValue<byte[]> getNew() {
try {
return (ValueCompressonHolder.UnCompressValue<byte[]>) clone();
} catch (CloneNotSupportedException cloneNotSupportedException) {
- LOGGER.error(cloneNotSupportedException,
- cloneNotSupportedException.getMessage());
+ LOGGER.error(cloneNotSupportedException, cloneNotSupportedException.getMessage());
}
return null;
}
@@ -62,10 +68,13 @@ public class UnCompressNonDecimalMaxMinByte
return byte1;
}
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType, byte[] compressData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
ValueCompressonHolder.UnCompressValue byte1 =
ValueCompressionUtil.getUnCompressNonDecimalMaxMin(dataType);
- ValueCompressonHolder.unCompress(dataType, byte1, value);
+ ValueCompressonHolder
+ .unCompress(dataType, byte1, compressData, offset, length, decimalPlaces, maxValueObject);
return byte1;
}
@@ -84,28 +93,37 @@ public class UnCompressNonDecimalMaxMinByte
this.value = value;
}
- @Override public CarbonReadDataHolder getValues(int decimalVal, Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimalVal);
-
- if (value[i] == 0) {
- vals[i] = maxValue;
- } else {
- BigDecimal diff = BigDecimal.valueOf(value[i] / Math.pow(10, decimalVal));
- BigDecimal max = BigDecimal.valueOf(maxValue);
- vals[i] = max.subtract(diff).doubleValue();
- }
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ if (byteValue == 0) {
+ return this.maxValue.doubleValue();
+ } else {
+ BigDecimal diff = BigDecimal.valueOf(byteValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
}
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
}
- @Override public void setValue(byte[] value) {
- this.value = value;
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get bigdecimal value is not supported");
}
+ @Override public void setUncompressValues(byte[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinDefault.java
index 190adbf..d720889 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinDefault.java
@@ -24,10 +24,12 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,12 +42,16 @@ public class UnCompressNonDecimalMaxMinDefault implements UnCompressValue<double
/**
* doubleCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private double[] value;
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+ private BigDecimal maxValue;
+ private double divisionFactor;
+
@Override public void setValue(double[] value) {
this.value = value;
}
@@ -78,28 +84,40 @@ public class UnCompressNonDecimalMaxMinDefault implements UnCompressValue<double
return new UnCompressNonDecimalMaxMinByte();
}
- @Override public UnCompressValue uncompress(DataType dataType) {
+ @Override
+ public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType, byte[] compressData,
+ int offset, int length, int mantissa, Object maxValueObject) {
return null;
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double maxVal = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder holder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimal);
-
- if (value[i] == 0) {
- vals[i] = maxVal;
- } else {
- BigDecimal diff = BigDecimal.valueOf(value[i] / Math.pow(10, decimal));
- BigDecimal max = BigDecimal.valueOf(maxVal);
- vals[i] = max.subtract(diff).doubleValue();
- }
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ if (doubleValue == 0) {
+ return maxValue.doubleValue();
+ } else {
+ BigDecimal diff = BigDecimal.valueOf(doubleValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
}
- holder.setReadableDoubleValues(vals);
- return holder;
}
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ @Override
+ public void setUncompressValues(double[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinFloat.java
deleted file mode 100644
index 54ca7b4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinFloat.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class UnCompressNonDecimalMaxMinFloat
- implements ValueCompressonHolder.UnCompressValue<float[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNonDecimalMaxMinFloat.class.getName());
- /**
- * floatCompressor
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
- private float[] value;
-
- @Override public void setValue(float[] value) {
- this.value = value;
-
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException exc1) {
- LOGGER.error(exc1, exc1.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
-
- UnCompressNonDecimalMaxMinByte byte1 = new UnCompressNonDecimalMaxMinByte();
- byte1.setValue(compressor.compressFloat(value));
- return byte1;
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
- @Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dataType) {
- return null;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToFloatArray(buffer, value.length);
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNonDecimalMaxMinByte();
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder holder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimal);
-
- if (value[i] == 0) {
- vals[i] = maxValue;
- } else {
- BigDecimal diff = BigDecimal.valueOf(value[i] / Math.pow(10, decimal));
- BigDecimal max = BigDecimal.valueOf(maxValue);
- vals[i] = max.subtract(diff).doubleValue();
- }
- }
- holder.setReadableDoubleValues(vals);
- return holder;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinInt.java
index 3e1b241..d834f6a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/UnCompressNonDecimalMaxMinInt.java
@@ -24,10 +24,12 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
@@ -40,12 +42,16 @@ public class UnCompressNonDecimalMaxMinInt implements UnCompressValue<int[]> {
/**
* intCompressor.
*/
- private static Compressor compressor = CompressorFactory.getInstance();
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
/**
* value.
*/
private int[] value;
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+ private double divisionFactor;
+ private BigDecimal maxValue;
+
@Override public void setValue(int[] value) {
this.value = value;
@@ -81,28 +87,40 @@ public class UnCompressNonDecimalMaxMinInt implements UnCompressValue<int[]> {
return new UnCompressNonDecimalMaxMinByte();
}
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double maxValue = (double) maxValueObject;
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolderInfo = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i] / Math.pow(10, decimal);
-
- if (value[i] == 0) {
- vals[i] = maxValue;
- } else {
- BigDecimal diff = BigDecimal.valueOf(value[i] / Math.pow(10, decimal));
- BigDecimal max = BigDecimal.valueOf(maxValue);
- vals[i] = max.subtract(diff).doubleValue();
- }
+ @Override
+ public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType, byte[] compressData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ return null;
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException("Get long value is not supported");
+ }
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ if (intValue == 0) {
+ return maxValue.doubleValue();
+ } else {
+ BigDecimal diff = BigDecimal.valueOf(intValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
}
- dataHolderInfo.setReadableDoubleValues(vals);
- return dataHolderInfo;
}
- @Override public UnCompressValue uncompress(DataType dataType) {
- return null;
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
}
+ @Override public void setUncompressValues(int[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
}