You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/12/06 15:49:41 UTC
[2/6] incubator-carbondata git commit: change package name
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
deleted file mode 100644
index 75c933f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneFloat.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.type;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class UnCompressNoneFloat implements ValueCompressonHolder.UnCompressValue<float[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNoneFloat.class.getName());
- /**
- * compressor
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
- private float[] value;
-
- private DataType actualDataType;
-
- public UnCompressNoneFloat(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(float[] value) {
- this.value = value;
-
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException ex5) {
- LOGGER.error(ex5, ex5.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
- byte1.setValue(compressor.compressFloat(value));
- return byte1;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToFloatArray(buffer, value.length);
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte(this.actualDataType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- double[] vals = new double[value.length];
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
- dataHolder.setReadableDoubleValues(vals);
- return dataHolder;
- }
-
- @Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dataType) {
- return null;
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
deleted file mode 100644
index 6a3655d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.type;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNoneInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- public UnCompressNoneInt(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
-
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException exc) {
- LOGGER.error(exc, exc.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
- byte1.setValue(compressor.compressInt(value));
- return byte1;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
- return null;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
-
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte(this.actualDataType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_SHORT:
- return unCompressShort();
- case DATA_INT:
- return unCompressInt();
- case DATA_LONG:
- case DATA_BIGINT:
- return unCompressLong();
- default:
- return unCompressDouble();
- }
- }
-
- private CarbonReadDataHolder unCompressShort() {
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- short[] vals = new short[value.length];
- for (int i = 0; i < vals.length; i++) {
- vals[i] = (short)value[i];
- }
- dataHolder.setReadableShortValues(vals);
- return dataHolder;
- }
-
- private CarbonReadDataHolder unCompressInt() {
- CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
- dataHolder.setReadableIntValues(value);
- return dataHolder;
- }
-
- private CarbonReadDataHolder unCompressDouble() {
- CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
- double[] vals = new double[value.length];
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
- dataHolderInfoObj.setReadableDoubleValues(vals);
- return dataHolderInfoObj;
- }
-
- private CarbonReadDataHolder unCompressLong() {
- CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
- long[] vals = new long[value.length];
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
-
- dataHolderInfoObj.setReadableLongValues(vals);
- return dataHolderInfoObj;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
deleted file mode 100644
index eebfdeb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.type;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNoneLong.class.getName());
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance();
- /**
- * value.
- */
- protected long[] value;
-
- private DataType actualDataType;
-
- public UnCompressNoneLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException clnNotSupportedExc) {
- LOGGER.error(clnNotSupportedExc,
- clnNotSupportedExc.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
- byte1.setValue(compressor.compressLong(value));
- return byte1;
-
- }
-
- @Override
- public ValueCompressonHolder.UnCompressValue uncompress(ValueCompressionUtil.DataType dType) {
- return null;
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(value);
- }
-
- @Override public void setValueInBytes(byte[] byteValue) {
- ByteBuffer buffer = ByteBuffer.wrap(byteValue);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, byteValue.length);
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte(this.actualDataType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong();
- default:
- return unCompressDouble();
- }
- }
-
- private CarbonReadDataHolder unCompressDouble() {
- CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
-
- double[] vals = new double[value.length];
-
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
- dataHldr.setReadableDoubleValues(vals);
- return dataHldr;
- }
-
- private CarbonReadDataHolder unCompressLong() {
- CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
-
- long[] vals = new long[value.length];
-
- for (int i = 0; i < vals.length; i++) {
- vals[i] = value[i];
- }
- dataHldr.setReadableLongValues(vals);
- return dataHldr;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
deleted file mode 100644
index 3cfb530..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.type;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValue<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(UnCompressNoneShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance();
-
- /**
- * value.
- */
- private short[] shortValue;
-
- private DataType actualDataType;
-
- public UnCompressNoneShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] shortValue) {
- this.shortValue = shortValue;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue getNew() {
- try {
- return (ValueCompressonHolder.UnCompressValue) clone();
- } catch (CloneNotSupportedException cns1) {
- LOGGER.error(cns1, cns1.getMessage());
- }
- return null;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
- byte1.setValue(compressor.compressShort(shortValue));
- return byte1;
- }
-
- @Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
- return null;
- }
-
- @Override public byte[] getBackArrayData() {
- return ValueCompressionUtil.convertToBytes(shortValue);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- shortValue = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- /**
- * @see ValueCompressonHolder.UnCompressValue#getCompressorObject()
- */
- @Override public ValueCompressonHolder.UnCompressValue getCompressorObject() {
- return new UnCompressNoneByte(this.actualDataType);
- }
-
- @Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
- switch (actualDataType) {
- case DATA_BIGINT:
- return unCompressLong();
- default:
- return unCompressDouble();
- }
- }
-
- private CarbonReadDataHolder unCompressDouble() {
- CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
-
- double[] vals = new double[shortValue.length];
-
- for (int i = 0; i < vals.length; i++) {
- vals[i] = shortValue[i];
- }
- dataHldr.setReadableDoubleValues(vals);
- return dataHldr;
- }
-
- private CarbonReadDataHolder unCompressLong() {
- CarbonReadDataHolder dataHldr = new CarbonReadDataHolder();
-
- long[] vals = new long[shortValue.length];
-
- for (int i = 0; i < vals.length; i++) {
- vals[i] = shortValue[i];
- }
- dataHldr.setReadableLongValues(vals);
- return dataHldr;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
index 6051736..9b6ce9f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -21,8 +21,8 @@ package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -38,7 +38,7 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
/**
* compressionModel.
*/
- protected ValueCompressionModel compressionModel;
+ protected WriterCompressModel compressionModel;
/**
* type
@@ -50,7 +50,7 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
*
* @param compressionModel
*/
- public AbstractHeavyCompressedDoubleArrayDataStore(ValueCompressionModel compressionModel) {
+ public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
this.compressionModel = compressionModel;
if (null != compressionModel) {
this.type = compressionModel.getType();
@@ -69,7 +69,7 @@ public abstract class AbstractHeavyCompressedDoubleArrayDataStore
ValueCompressionUtil.getValueCompressor(compressionModel.getActualDataType()[i])
.getCompressedValues(compressionModel.getCompType()[i], dataHolder[i],
compressionModel.getChangedDataType()[i], compressionModel.getMaxValue()[i],
- compressionModel.getDecimal()[i]));
+ compressionModel.getMantissa()[i]));
} else {
values[i].setValue(dataHolder[i].getWritableByteArrayValues());
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
index f726ba7..c4733f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
@@ -21,25 +21,14 @@ package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
import org.apache.carbondata.core.datastorage.store.FileHolder;
import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
public class HeavyCompressedDoubleArrayDataInMemoryStore
extends AbstractHeavyCompressedDoubleArrayDataStore {
- public HeavyCompressedDoubleArrayDataInMemoryStore(ValueCompressionModel compressionModel,
- long[] measuresOffsetsArray, int[] measuresLengthArray, String fileName,
- FileHolder fileHolder) {
- super(compressionModel);
- for (int i = 0; i < measuresLengthArray.length; i++) {
- values[i] = compressionModel.getUnCompressValues()[i].getCompressorObject();
- values[i].setValue(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[i], measuresLengthArray[i]));
- }
- }
-
- public HeavyCompressedDoubleArrayDataInMemoryStore(ValueCompressionModel compressionModel) {
+ public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
super(compressionModel);
}
@@ -51,14 +40,14 @@ public class HeavyCompressedDoubleArrayDataInMemoryStore
if (cols != null) {
for (int i = 0; i < cols.length; i++) {
vals[cols[i]] = values[cols[i]].uncompress(compressionModel.getChangedDataType()[cols[i]])
- .getValues(compressionModel.getDecimal()[cols[i]],
+ .getValues(compressionModel.getMantissa()[cols[i]],
compressionModel.getMaxValue()[cols[i]]);
}
} else {
for (int i = 0; i < vals.length; i++) {
vals[i] = values[i].uncompress(compressionModel.getChangedDataType()[i])
- .getValues(compressionModel.getDecimal()[i], compressionModel.getMaxValue()[i]);
+ .getValues(compressionModel.getMantissa()[i], compressionModel.getMaxValue()[i]);
}
}
return new CompressedDataMeasureDataWrapper(vals);
@@ -70,7 +59,7 @@ public class HeavyCompressedDoubleArrayDataInMemoryStore
}
CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
vals[cols] = values[cols].uncompress(compressionModel.getChangedDataType()[cols])
- .getValues(compressionModel.getDecimal()[cols], compressionModel.getMaxValue()[cols]);
+ .getValues(compressionModel.getMantissa()[cols], compressionModel.getMaxValue()[cols]);
return new CompressedDataMeasureDataWrapper(vals);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/AbstractDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/AbstractDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/AbstractDoubleArrayDataStore.java
deleted file mode 100644
index 8271e43..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/AbstractDoubleArrayDataStore.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.uncompressed;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public abstract class AbstractDoubleArrayDataStore implements NodeMeasureDataStore {
-
- protected ValueCompressonHolder.UnCompressValue[] values;
-
- protected ValueCompressionModel compressionModel;
-
- private char[] type;
-
- public AbstractDoubleArrayDataStore(ValueCompressionModel compressionModel) {
- this.compressionModel = compressionModel;
- if (null != compressionModel) {
- values =
- new ValueCompressonHolder.UnCompressValue[compressionModel.getUnCompressValues().length];
- type = compressionModel.getType();
- }
- }
-
- @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
- values =
- new ValueCompressonHolder.UnCompressValue[compressionModel.getUnCompressValues().length];
- for (int i = 0; i < compressionModel.getUnCompressValues().length; i++) {
- values[i] = compressionModel.getUnCompressValues()[i].getNew();
- if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE
- && type[i] != CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- if (type[i] == CarbonCommonConstants.BIG_INT_MEASURE) {
- values[i].setValue(ValueCompressionUtil
- .getCompressedValues(compressionModel.getCompType()[i],
- dataHolder[i].getWritableLongValues(), compressionModel.getChangedDataType()[i],
- (long) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i]));
- } else {
- values[i].setValue(ValueCompressionUtil
- .getCompressedValues(compressionModel.getCompType()[i],
- dataHolder[i].getWritableDoubleValues(), compressionModel.getChangedDataType()[i],
- (double) compressionModel.getMaxValue()[i], compressionModel.getDecimal()[i]));
- }
- } else {
- values[i].setValue(dataHolder[i].getWritableByteArrayValues());
- }
- }
-
- byte[][] resturnValue = new byte[values.length][];
-
- for (int i = 0; i < values.length; i++) {
- resturnValue[i] = values[i].getBackArrayData();
- }
- return resturnValue;
- }
-
- @Override public short getLength() {
- return values != null ? (short) values.length : 0;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
deleted file mode 100644
index 3aaa09d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
-
-public class DoubleArrayDataInMemoryStore extends AbstractDoubleArrayDataStore {
-
- // /**
- // * DoubleArrayDataInMemoryStore.
- // * @param size
- // * @param elementSize
- // * @param compressionModel
- // */
- // public DoubleArrayDataInMemoryStore(int size, int elementSize,
- // ValueCompressionModel compressionModel)
- // {
- // super(size, elementSize, compressionModel);
- // }
- //
- // /**
- // * DoubleArrayDataInMemoryStore.
- // * @param size
- // * @param elementSize
- // */
- // public DoubleArrayDataInMemoryStore(int size, int elementSize)
- // {
- // super(size, elementSize);
- // }
-
- // /**
- // * DoubleArrayDataInMemoryStore.
- // * @param size
- // * @param elementSize
- // * @param compressionModel
- // * @param measuresOffsetsArray
- // * @param measuresLengthArray
- // * @param fileName
- // * @param fileHolder
- // */
- // public DoubleArrayDataInMemoryStore(int size, int elementSize,
- // ValueCompressionModel compressionModel,
- // long[] measuresOffsetsArray, int[] measuresLengthArray, String fileName,
- // FileHolder fileHolder)
- // {
- // super(size, elementSize, compressionModel);
- // UnCompressValue[] unCompValues = compressionModel.getUnCompressValues();
- // if(null != unCompValues)
- // {
- // for(int i = 0;i < measuresLengthArray.length;i++)
- // {
- //
- // values[i] = unCompValues[i].getNew();
- // values[i].setValueInBytes(fileHolder.readByteArray(fileName,
- // measuresOffsetsArray[i],
- // measuresLengthArray[i]));
- // }
- // }
- // }
-
- /**
- * DoubleArrayDataInMemoryStore.
- *
- * @param size
- * @param elementSize
- * @param compressionModel
- * @param measuresOffsetsArray
- * @param measuresLengthArray
- * @param fileName
- * @param fileHolder
- */
- public DoubleArrayDataInMemoryStore(ValueCompressionModel compressionModel,
- long[] measuresOffsetsArray, int[] measuresLengthArray, String fileName,
- FileHolder fileHolder) {
- super(compressionModel);
- if (null != compressionModel) {
- UnCompressValue[] unCompValues = compressionModel.getUnCompressValues();
- if (null != unCompValues) {
- for (int i = 0; i < measuresLengthArray.length; i++) {
-
- values[i] = unCompValues[i].getNew();
- values[i].setValueInBytes(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[i], measuresLengthArray[i]));
- }
- }
- }
- }
-
- /**
- * DoubleArrayDataInMemoryStore.
- *
- * @param size
- * @param elementSize
- * @param compressionModel
- * @param measuresOffsetsArray
- * @param measuresLengthArray
- * @param fileName
- * @param fileHolder
- */
- public DoubleArrayDataInMemoryStore(ValueCompressionModel compressionModel) {
- super(compressionModel);
- }
-
- @Override public MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
- if (null == cols) {
- for (int i = 0; i < vals.length; i++) {
- vals[i] = values[i]
- .getValues(compressionModel.getDecimal()[i], compressionModel.getMaxValue()[i]);
- }
- } else {
- for (int i = 0; i < cols.length; i++) {
- vals[cols[i]] = values[cols[i]].getValues(compressionModel.getDecimal()[cols[i]],
- compressionModel.getMaxValue()[cols[i]]);
- }
- }
- // return new CompressedDataMeasureDataWrapper(values,
- // compressionModel.getDecimal(), compressionModel.getMaxValue());
- return new CompressedDataMeasureDataWrapper(vals);
- }
-
- @Override public MeasureDataWrapper getBackData(int cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
-
- vals[cols] = values[cols]
- .getValues(compressionModel.getDecimal()[cols], compressionModel.getMaxValue()[cols]);
- return new CompressedDataMeasureDataWrapper(vals);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
index 408524f..7561ec7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
@@ -20,43 +20,16 @@
package org.apache.carbondata.core.datastorage.util;
import org.apache.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
-import org.apache.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore;
public final class StoreFactory {
- /**
- * value type.
- */
- private static StoreType valueType;
-
- static {
- valueType = StoreType.HEAVY_VALUE_COMPRESSION;
- }
private StoreFactory() {
-
- }
-
- public static NodeMeasureDataStore createDataStore(ValueCompressionModel compressionModel) {
- switch (valueType) {
- case COMPRESSED_DOUBLE_ARRAY:
- return new DoubleArrayDataInMemoryStore(compressionModel);
-
- case HEAVY_VALUE_COMPRESSION:
- return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel);
- default:
- return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel);
- }
}
- /**
- * enum defined.
- */
- public enum StoreType {
- COMPRESSED_SINGLE_ARRAY,
- COMPRESSED_DOUBLE_ARRAY,
- HEAVY_VALUE_COMPRESSION
+ public static NodeMeasureDataStore createDataStore(WriterCompressModel compressionModel) {
+ return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
index 54e5361..c12c556 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.metadata;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
public class BlockletInfoColumnar {
@@ -98,7 +98,7 @@ public class BlockletInfoColumnar {
private NumberCompressor[] keyBlockUnCompressor;
- private ValueCompressionModel compressionModel;
+ private WriterCompressModel compressionModel;
/**
* column min array
@@ -367,11 +367,11 @@ public class BlockletInfoColumnar {
this.columnMinData = columnMinData;
}
- public ValueCompressionModel getCompressionModel() {
+ public WriterCompressModel getCompressionModel() {
return compressionModel;
}
- public void setCompressionModel(ValueCompressionModel compressionModel) {
+ public void setCompressionModel(WriterCompressModel compressionModel) {
this.compressionModel = compressionModel;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
index 471158d..64b39f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
@@ -39,19 +39,11 @@ public class ValueEncoderMeta implements Serializable {
* uniqueValue
*/
private Object uniqueValue;
- /**
- * decimal.
- */
- private int decimal;
- /**
- * aggType
- */
+ private int mantissa;
+
private char type;
- /**
- * dataTypeSelected
- */
private byte dataTypeSelected;
public Object getMaxValue() {
@@ -78,12 +70,12 @@ public class ValueEncoderMeta implements Serializable {
this.uniqueValue = uniqueValue;
}
- public int getDecimal() {
- return decimal;
+ public int getMantissa() {
+ return mantissa;
}
- public void setDecimal(int decimal) {
- this.decimal = decimal;
+ public void setMantissa(int mantissa) {
+ this.mantissa = mantissa;
}
public char getType() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index c435dd5..747862f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -35,7 +35,7 @@ import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.BlockIndex;
@@ -314,13 +314,13 @@ public class CarbonMetadataUtil {
return aos.toByteArray();
}
- private static ValueEncoderMeta createValueEncoderMeta(ValueCompressionModel compressionModel,
+ private static ValueEncoderMeta createValueEncoderMeta(WriterCompressModel compressionModel,
int index) {
ValueEncoderMeta encoderMeta = new ValueEncoderMeta();
encoderMeta.setMaxValue(compressionModel.getMaxValue()[index]);
encoderMeta.setMinValue(compressionModel.getMinValue()[index]);
encoderMeta.setDataTypeSelected(compressionModel.getDataTypeSelected()[index]);
- encoderMeta.setDecimal(compressionModel.getDecimal()[index]);
+ encoderMeta.setMantissa(compressionModel.getMantissa()[index]);
encoderMeta.setType(compressionModel.getType()[index]);
encoderMeta.setUniqueValue(compressionModel.getUniqueValue()[index]);
return encoderMeta;
@@ -420,7 +420,7 @@ public class CarbonMetadataUtil {
}
- private static ValueCompressionModel getValueCompressionModel(ValueEncoderMeta[] encoderMetas) {
+ private static WriterCompressModel getValueCompressionModel(ValueEncoderMeta[] encoderMetas) {
Object[] maxValue = new Object[encoderMetas.length];
Object[] minValue = new Object[encoderMetas.length];
int[] decimalLength = new int[encoderMetas.length];
@@ -430,13 +430,13 @@ public class CarbonMetadataUtil {
for (int i = 0; i < encoderMetas.length; i++) {
maxValue[i] = encoderMetas[i].getMaxValue();
minValue[i] = encoderMetas[i].getMinValue();
- decimalLength[i] = encoderMetas[i].getDecimal();
+ decimalLength[i] = encoderMetas[i].getMantissa();
uniqueValue[i] = encoderMetas[i].getUniqueValue();
aggType[i] = encoderMetas[i].getType();
dataTypeSelected[i] = encoderMetas[i].getDataTypeSelected();
}
return ValueCompressionUtil
- .getValueCompressionModel(maxValue, minValue, decimalLength, uniqueValue, aggType,
+ .getWriterCompressModel(maxValue, minValue, decimalLength, uniqueValue, aggType,
dataTypeSelected);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 41594ad..0fb69ce 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -64,7 +64,7 @@ import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
@@ -956,7 +956,7 @@ public final class CarbonUtil {
* @param measureDataChunkList
* @return value compression model
*/
- public static ValueCompressionModel getValueCompressionModel(
+ public static WriterCompressModel getValueCompressionModel(
List<ValueEncoderMeta> encodeMetaList) {
Object[] maxValue = new Object[encodeMetaList.size()];
Object[] minValue = new Object[encodeMetaList.size()];
@@ -968,19 +968,19 @@ public final class CarbonUtil {
/**
* to fill the meta data required for value compression model
*/
- for (int i = 0; i < dataTypeSelected.length; i++) {
+ for (int i = 0; i < dataTypeSelected.length; i++) { // always 1
ValueEncoderMeta valueEncoderMeta = encodeMetaList.get(i);
maxValue[i] = valueEncoderMeta.getMaxValue();
minValue[i] = valueEncoderMeta.getMinValue();
uniqueValue[i] = valueEncoderMeta.getUniqueValue();
- decimal[i] = valueEncoderMeta.getDecimal();
+ decimal[i] = valueEncoderMeta.getMantissa();
type[i] = valueEncoderMeta.getType();
dataTypeSelected[i] = valueEncoderMeta.getDataTypeSelected();
}
MeasureMetaDataModel measureMetadataModel =
new MeasureMetaDataModel(minValue, maxValue, decimal, dataTypeSelected.length, uniqueValue,
type, dataTypeSelected);
- return ValueCompressionUtil.getValueCompressionModel(measureMetadataModel);
+ return ValueCompressionUtil.getWriterCompressModel(measureMetadataModel);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 5e68411..d01b30a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -23,63 +23,56 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.compression.BigIntCompressor;
import org.apache.carbondata.core.compression.DoubleCompressor;
import org.apache.carbondata.core.compression.ValueCompressor;
import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressByteArray;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinByte;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinDefault;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinFloat;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinInt;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinLong;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinShort;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalByte;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalDefault;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalFloat;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalInt;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalLong;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinByte;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinDefault;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinFloat;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinInt;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinLong;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinShort;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalShort;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneByte;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneDefault;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneFloat;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneInt;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneLong;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressNoneShort;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressByteArray;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinByte;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinDefault;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinFloat;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinInt;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinLong;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinShort;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalByte;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalDefault;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalFloat;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalInt;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalLong;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinFloat;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalShort;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneByte;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneDefault;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneFloat;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneInt;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneLong;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneShort;
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
public final class ValueCompressionUtil {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ValueCompressionUtil.class.getName());
-
private ValueCompressionUtil() {
-
}
/**
* decide actual type of value
*
* @param value :the measure value
- * @param decimal :
+ * @param mantissa :
* @return: actual type of value
* @see
*/
- private static DataType getDataType(double value, int decimal, byte dataTypeSelected) {
+ private static DataType getDataType(double value, int mantissa, byte dataTypeSelected) {
DataType dataType = DataType.DATA_DOUBLE;
- if (decimal == 0) {
+ if (mantissa == 0) {
if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
dataType = DataType.DATA_BYTE;
} else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
@@ -131,89 +124,94 @@ public final class ValueCompressionUtil {
/**
* get the best compression type. priority list,from high to low:
- * COMPRESSION_TYPE.NONE COMPRESSION_TYPE.MAX_MIN
- * COMPRESSION_TYPE.NON_DECIMAL_CONVERT COMPRESSION_TYPE.MAX_MIN_NDC
+ * COMPRESSION_TYPE.ADAPTIVE COMPRESSION_TYPE.DELTA_DOUBLE
+ * COMPRESSION_TYPE.BIGINT COMPRESSION_TYPE.DELTA_NON_DECIMAL
*
* @param maxValue : max value of one measure
* @param minValue : min value of one measure
- * @param decimal : decimal num of one measure
+ * @param mantissa : decimal num of one measure
* @return : the best compression type
* @see
*/
- private static CompressionFinder getCompressionType(Object maxValue, Object minValue, int decimal,
- char aggregatorType, byte dataTypeSelected) {
- // 'c' for aggregate table,'b' fo rBigdecimal, 'l' for long,'n' for double
- switch (aggregatorType) {
- case 'c':
- return new CompressionFinder(COMPRESSION_TYPE.CUSTOM, DataType.DATA_BYTE,
- DataType.DATA_BYTE);
+ public static CompressionFinder getCompressionFinder(Object maxValue, Object minValue,
+ int mantissa, char measureStoreType, byte dataTypeSelected) {
+ // ''b' for decimal, 'l' for long, 'n' for double
+ switch (measureStoreType) {
case 'b':
- return new CompressionFinder(COMPRESSION_TYPE.CUSTOM_BIGDECIMAL, DataType.DATA_BYTE,
+ return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.DATA_BYTE,
DataType.DATA_BYTE);
case 'l':
- return getLongCompressorFinder(maxValue, minValue, decimal, aggregatorType,
- dataTypeSelected);
+ return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected);
+ case 'n':
+ return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected);
default:
- break;
+ throw new IllegalArgumentException("unsupported measure type");
}
+ }
+
+ private static CompressionFinder getDoubleCompressorFinder(Object maxValue, Object minValue,
+ int mantissa, byte dataTypeSelected) {
//Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
//but we can't use -1 to getDatatype, we should use -10000000.
double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ?
(double) maxValue:(double) minValue;
- // None Decimal
- if (decimal == 0) {
- if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) > getSize(
- getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
- return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_DOUBLE,
- getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
- } else if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) < getSize(
- getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
- return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
- getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
+ DataType adaptiveDataType = getDataType(absMaxValue, mantissa, dataTypeSelected);
+ DataType deltaDataType = getDataType((double) maxValue - (double) minValue, mantissa,
+ dataTypeSelected);
+
+ if (mantissa == 0) {
+ // short, int, long
+ int adaptiveSize = getSize(adaptiveDataType);
+ int deltaSize = getSize(deltaDataType);
+ if (adaptiveSize > deltaSize) {
+ return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_DOUBLE,
+ deltaDataType);
+ } else if (adaptiveSize < deltaSize) {
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+ deltaDataType);
} else {
- return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
- getDataType(absMaxValue, decimal, dataTypeSelected));
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+ adaptiveDataType);
}
- }
- // decimal
- else {
- DataType actualDataType = getDataType(absMaxValue, decimal, dataTypeSelected);
- DataType diffDataType =
- getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected);
+ } else {
+ // double
DataType maxNonDecDataType =
- getDataType(Math.pow(10, decimal) * absMaxValue, 0, dataTypeSelected);
+ getDataType(Math.pow(10, mantissa) * absMaxValue, 0, dataTypeSelected);
DataType diffNonDecDataType =
- getDataType(Math.pow(10, decimal) * ((double) maxValue - (double) minValue), 0,
+ getDataType(Math.pow(10, mantissa) * ((double) maxValue - (double) minValue), 0,
dataTypeSelected);
CompressionFinder[] finders = new CompressionFinder[] {
- new CompressionFinder(actualDataType, actualDataType, CompressionFinder.PRIORITY.ACTUAL,
- COMPRESSION_TYPE.NONE),
- new CompressionFinder(actualDataType, diffDataType, CompressionFinder.PRIORITY.DIFFSIZE,
- COMPRESSION_TYPE.MAX_MIN), new CompressionFinder(actualDataType, maxNonDecDataType,
- CompressionFinder.PRIORITY.MAXNONDECIMAL, COMPRESSION_TYPE.NON_DECIMAL_CONVERT),
- new CompressionFinder(actualDataType, diffNonDecDataType,
- CompressionFinder.PRIORITY.DIFFNONDECIMAL, COMPRESSION_TYPE.MAX_MIN_NDC) };
+ new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, adaptiveDataType, adaptiveDataType,
+ CompressionFinder.PRIORITY.ACTUAL),
+ new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, adaptiveDataType, deltaDataType,
+ CompressionFinder.PRIORITY.DIFFSIZE),
+ new CompressionFinder(COMPRESSION_TYPE.BIGINT, adaptiveDataType, maxNonDecDataType,
+ CompressionFinder.PRIORITY.MAXNONDECIMAL),
+ new CompressionFinder(COMPRESSION_TYPE.DELTA_NON_DECIMAL, adaptiveDataType,
+ diffNonDecDataType, CompressionFinder.PRIORITY.DIFFNONDECIMAL) };
// sort the compressionFinder.The top have the highest priority
Arrays.sort(finders);
- CompressionFinder compression = finders[0];
- return compression;
+ return finders[0];
}
}
private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue,
- int decimal, char aggregatorType, byte dataTypeSelected) {
- if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) > getSize(
- getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) {
- return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_BIGINT,
- getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected));
- } else if (getSize(getDataType((long) maxValue, decimal, dataTypeSelected)) < getSize(
- getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected))) {
- return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT,
- getDataType((long) maxValue - (long) minValue, decimal, dataTypeSelected));
+ int mantissa, byte dataTypeSelected) {
+ DataType adaptiveDataType = getDataType((long) maxValue, mantissa, dataTypeSelected);
+ int adaptiveSize = getSize(adaptiveDataType);
+ DataType deltaDataType = getDataType((long) maxValue - (long) minValue, mantissa,
+ dataTypeSelected);
+ int deltaSize = getSize(deltaDataType);
+ if (adaptiveSize > deltaSize) {
+ return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_BIGINT,
+ deltaDataType);
+ } else if (adaptiveSize < deltaSize) {
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+ deltaDataType);
} else {
- return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_BIGINT,
- getDataType((long) maxValue, decimal, dataTypeSelected));
+ return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+ adaptiveDataType);
}
}
@@ -222,42 +220,21 @@ public final class ValueCompressionUtil {
* @param values : the data of one measure
* @param changedDataType : changed data type
* @param maxValue : the max value of one measure
- * @param decimal : the decimal length of one measure
+ * @param mantissa : the decimal length of one measure
* @return: the compress data array
* @see
*/
public static Object getCompressedValues(COMPRESSION_TYPE compType, double[] values,
- DataType changedDataType, double maxValue, int decimal) {
- Object o;
- switch (compType) {
- case NONE:
-
- o = compressNone(changedDataType, values);
- return o;
-
- case MAX_MIN:
-
- o = compressMaxMin(changedDataType, values, maxValue);
- return o;
-
- case NON_DECIMAL_CONVERT:
-
- o = compressNonDecimal(changedDataType, values, decimal);
- return o;
-
- default:
- o = compressNonDecimalMaxMin(changedDataType, values, decimal, maxValue);
- return o;
- }
- }
-
- public static Object getCompressedValues(COMPRESSION_TYPE compType, long[] values,
- DataType changedDataType, long maxValue, int decimal) {
- Object o;
+ DataType changedDataType, double maxValue, int mantissa) {
switch (compType) {
- case NONE:
+ case ADAPTIVE:
+ return compressNone(changedDataType, values);
+ case DELTA_DOUBLE:
+ return compressMaxMin(changedDataType, values, maxValue);
+ case BIGINT:
+ return compressNonDecimal(changedDataType, values, mantissa);
default:
- return values;
+ return compressNonDecimalMaxMin(changedDataType, values, mantissa, maxValue);
}
}
@@ -282,35 +259,45 @@ public final class ValueCompressionUtil {
new ValueCompressonHolder.UnCompressValue[changedDataType.length];
for (int i = 0; i < changedDataType.length; i++) {
switch (compType[i]) {
- case NONE:
-
- compressValue[i] = unCompressNone(changedDataType[i], actualDataType[i]);
+ case ADAPTIVE:
+ compressValue[i] = getUnCompressNone(changedDataType[i], actualDataType[i]);
break;
- case MAX_MIN:
-
- compressValue[i] = unCompressMaxMin(changedDataType[i], actualDataType[i]);
+ case DELTA_DOUBLE:
+ compressValue[i] = getUnCompressDecimalMaxMin(changedDataType[i], actualDataType[i]);
break;
- case NON_DECIMAL_CONVERT:
-
- compressValue[i] = unCompressNonDecimal(changedDataType[i], DataType.DATA_DOUBLE);
+ case BIGINT:
+ compressValue[i] = getUnCompressNonDecimal(changedDataType[i]);
break;
- case CUSTOM:
- compressValue[i] = new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BYTE_ARRAY);
- break;
-
- case CUSTOM_BIGDECIMAL:
+ case BIGDECIMAL:
compressValue[i] = new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BIG_DECIMAL);
break;
default:
- compressValue[i] = unCompressNonDecimalMaxMin(changedDataType[i], null);
+ compressValue[i] = getUnCompressNonDecimalMaxMin(changedDataType[i]);
}
}
return compressValue;
+ }
+ private static ValueCompressonHolder.UnCompressValue getUncompressedValues(
+ COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType) {
+ switch (compType) {
+ case ADAPTIVE:
+ return getUnCompressNone(changedDataType, actualDataType);
+ case DELTA_DOUBLE:
+ return getUnCompressDecimalMaxMin(changedDataType, actualDataType);
+ case DELTA_NON_DECIMAL:
+ return getUnCompressNonDecimalMaxMin(changedDataType);
+ case BIGINT:
+ return getUnCompressNonDecimal(changedDataType);
+ case BIGDECIMAL:
+ return new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BIG_DECIMAL);
+ default:
+ throw new IllegalArgumentException("unsupported compType: " + compType);
+ }
}
/**
@@ -319,9 +306,7 @@ public final class ValueCompressionUtil {
private static Object compressNone(DataType changedDataType, double[] value) {
int i = 0;
switch (changedDataType) {
-
case DATA_BYTE:
-
byte[] result = new byte[value.length];
for (double a : value) {
@@ -331,7 +316,6 @@ public final class ValueCompressionUtil {
return result;
case DATA_SHORT:
-
short[] shortResult = new short[value.length];
for (double a : value) {
@@ -341,7 +325,6 @@ public final class ValueCompressionUtil {
return shortResult;
case DATA_INT:
-
int[] intResult = new int[value.length];
for (double a : value) {
@@ -352,7 +335,6 @@ public final class ValueCompressionUtil {
case DATA_LONG:
case DATA_BIGINT:
-
long[] longResult = new long[value.length];
for (double a : value) {
@@ -362,7 +344,6 @@ public final class ValueCompressionUtil {
return longResult;
case DATA_FLOAT:
-
float[] floatResult = new float[value.length];
for (double a : value) {
@@ -372,9 +353,7 @@ public final class ValueCompressionUtil {
return floatResult;
default:
-
return value;
-
}
}
@@ -449,16 +428,16 @@ public final class ValueCompressionUtil {
/**
* compress data to other type through sub value for example: 1. subValue =
- * value * Math.pow(10, decimal) 2. subValue: double->int
+ * value * Math.pow(10, mantissa) 2. subValue: double->int
*/
- private static Object compressNonDecimal(DataType changedDataType, double[] value, int decimal) {
+ private static Object compressNonDecimal(DataType changedDataType, double[] value, int mantissa) {
int i = 0;
switch (changedDataType) {
case DATA_BYTE:
byte[] result = new byte[value.length];
for (double a : value) {
- result[i] = (byte) (Math.round(Math.pow(10, decimal) * a));
+ result[i] = (byte) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return result;
@@ -466,7 +445,7 @@ public final class ValueCompressionUtil {
short[] shortResult = new short[value.length];
for (double a : value) {
- shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * a));
+ shortResult[i] = (short) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return shortResult;
@@ -475,7 +454,7 @@ public final class ValueCompressionUtil {
int[] intResult = new int[value.length];
for (double a : value) {
- intResult[i] = (int) (Math.round(Math.pow(10, decimal) * a));
+ intResult[i] = (int) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return intResult;
@@ -485,7 +464,7 @@ public final class ValueCompressionUtil {
long[] longResult = new long[value.length];
for (double a : value) {
- longResult[i] = (long) (Math.round(Math.pow(10, decimal) * a));
+ longResult[i] = (long) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return longResult;
@@ -495,7 +474,7 @@ public final class ValueCompressionUtil {
float[] floatResult = new float[value.length];
for (double a : value) {
- floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * a));
+ floatResult[i] = (float) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return floatResult;
@@ -504,7 +483,7 @@ public final class ValueCompressionUtil {
double[] defaultResult = new double[value.length];
for (double a : value) {
- defaultResult[i] = (double) (Math.round(Math.pow(10, decimal) * a));
+ defaultResult[i] = (double) (Math.round(Math.pow(10, mantissa) * a));
i++;
}
return defaultResult;
@@ -513,11 +492,11 @@ public final class ValueCompressionUtil {
/**
* compress data to other type through sub value for example: 1. subValue =
- * maxValue - value 2. subValue = subValue * Math.pow(10, decimal) 3.
+ * maxValue - value 2. subValue = subValue * Math.pow(10, mantissa) 3.
* subValue: double->int
*/
private static Object compressNonDecimalMaxMin(DataType changedDataType, double[] value,
- int decimal, double maxValue) {
+ int mantissa, double maxValue) {
int i = 0;
BigDecimal max = BigDecimal.valueOf(maxValue);
switch (changedDataType) {
@@ -528,7 +507,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- result[i] = (byte) (Math.round(diff * Math.pow(10, decimal)));
+ result[i] = (byte) (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return result;
@@ -540,7 +519,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- shortResult[i] = (short) (Math.round(diff * Math.pow(10, decimal)));
+ shortResult[i] = (short) (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return shortResult;
@@ -552,7 +531,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- intResult[i] = (int) (Math.round(diff * Math.pow(10, decimal)));
+ intResult[i] = (int) (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return intResult;
@@ -564,7 +543,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- longResult[i] = (long) (Math.round(diff * Math.pow(10, decimal)));
+ longResult[i] = (long) (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return longResult;
@@ -576,7 +555,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- floatResult[i] = (float) (Math.round(diff * Math.pow(10, decimal)));
+ floatResult[i] = (float) (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return floatResult;
@@ -588,7 +567,7 @@ public final class ValueCompressionUtil {
for (double a : value) {
BigDecimal val = BigDecimal.valueOf(a);
double diff = max.subtract(val).doubleValue();
- defaultResult[i] = (Math.round(diff * Math.pow(10, decimal)));
+ defaultResult[i] = (Math.round(diff * Math.pow(10, mantissa)));
i++;
}
return defaultResult;
@@ -599,31 +578,20 @@ public final class ValueCompressionUtil {
/**
* uncompress data for example: int -> double
*/
- public static ValueCompressonHolder.UnCompressValue unCompressNone(DataType compDataType,
+ public static ValueCompressonHolder.UnCompressValue getUnCompressNone(DataType compDataType,
DataType actualDataType) {
switch (compDataType) {
case DATA_BYTE:
-
return new UnCompressNoneByte(actualDataType);
-
case DATA_SHORT:
-
return new UnCompressNoneShort(actualDataType);
-
case DATA_INT:
-
return new UnCompressNoneInt(actualDataType);
-
case DATA_LONG:
-
return new UnCompressNoneLong(actualDataType);
-
case DATA_FLOAT:
-
return new UnCompressNoneFloat(actualDataType);
-
default:
-
return new UnCompressNoneDefault(actualDataType);
}
}
@@ -631,150 +599,108 @@ public final class ValueCompressionUtil {
/**
* uncompress data 1. value = maxValue - subValue 2. value: int->double
*/
- public static ValueCompressonHolder.UnCompressValue unCompressMaxMin(DataType compDataType,
- DataType actualDataType) {
+ public static ValueCompressonHolder.UnCompressValue getUnCompressDecimalMaxMin(
+ DataType compDataType, DataType actualDataType) {
switch (compDataType) {
case DATA_BYTE:
-
return new UnCompressMaxMinByte(actualDataType);
-
case DATA_SHORT:
-
return new UnCompressMaxMinShort(actualDataType);
-
case DATA_INT:
-
return new UnCompressMaxMinInt(actualDataType);
-
case DATA_LONG:
-
return new UnCompressMaxMinLong(actualDataType);
-
case DATA_FLOAT:
-
return new UnCompressMaxMinFloat(actualDataType);
-
default:
-
return new UnCompressMaxMinDefault(actualDataType);
-
}
}
/**
- * uncompress data value = value/Math.pow(10, decimal)
+ * uncompress data value = value/Math.pow(10, mantissa)
*/
- public static ValueCompressonHolder.UnCompressValue unCompressNonDecimal(DataType compDataType,
- DataType actualDataType) {
+ public static ValueCompressonHolder.UnCompressValue getUnCompressNonDecimal(
+ DataType compDataType) {
switch (compDataType) {
case DATA_BYTE:
-
return new UnCompressNonDecimalByte();
-
case DATA_SHORT:
-
return new UnCompressNonDecimalShort();
-
case DATA_INT:
-
return new UnCompressNonDecimalInt();
-
case DATA_LONG:
-
return new UnCompressNonDecimalLong();
-
case DATA_FLOAT:
-
return new UnCompressNonDecimalFloat();
-
default:
-
return new UnCompressNonDecimalDefault();
-
}
}
/**
- * uncompress data value = (maxValue - subValue)/Math.pow(10, decimal)
+ * uncompress data value = (maxValue - subValue)/Math.pow(10, mantissa)
*/
- public static ValueCompressonHolder.UnCompressValue unCompressNonDecimalMaxMin(
- DataType compDataType, DataType actualDataType) {
+ public static ValueCompressonHolder.UnCompressValue getUnCompressNonDecimalMaxMin(
+ DataType compDataType) {
switch (compDataType) {
case DATA_BYTE:
-
return new UnCompressNonDecimalMaxMinByte();
-
case DATA_SHORT:
-
return new UnCompressNonDecimalMaxMinShort();
-
case DATA_INT:
-
return new UnCompressNonDecimalMaxMinInt();
-
case DATA_LONG:
-
return new UnCompressNonDecimalMaxMinLong();
-
case DATA_FLOAT:
-
return new UnCompressNonDecimalMaxMinFloat();
-
default:
-
return new UnCompressNonDecimalMaxMinDefault();
-
}
}
/**
- * Create Value compression model
- *
- * @param maxValue
- * @param minValue
- * @param decimalLength
- * @param uniqueValue
- * @param aggType
- * @param dataTypeSelected
- * @return
+ * Create Value compression model for write path
*/
- public static ValueCompressionModel getValueCompressionModel(Object[] maxValue, Object[] minValue,
- int[] decimalLength, Object[] uniqueValue, char[] aggType, byte[] dataTypeSelected) {
-
+ public static WriterCompressModel getWriterCompressModel(Object[] maxValue, Object[] minValue,
+ int[] mantissa, Object[] uniqueValue, char[] aggType, byte[] dataTypeSelected) {
MeasureMetaDataModel metaDataModel =
- new MeasureMetaDataModel(minValue, maxValue, decimalLength, maxValue.length, uniqueValue,
+ new MeasureMetaDataModel(minValue, maxValue, mantissa, maxValue.length, uniqueValue,
aggType, dataTypeSelected);
- return getValueCompressionModel(metaDataModel);
+ return getWriterCompressModel(metaDataModel);
}
- public static ValueCompressionModel getValueCompressionModel(MeasureMetaDataModel measureMDMdl) {
+ /**
+ * Create Value compression model for write path
+ */
+ public static WriterCompressModel getWriterCompressModel(MeasureMetaDataModel measureMDMdl) {
int measureCount = measureMDMdl.getMeasureCount();
Object[] minValue = measureMDMdl.getMinValue();
Object[] maxValue = measureMDMdl.getMaxValue();
Object[] uniqueValue = measureMDMdl.getUniqueValue();
- int[] decimal = measureMDMdl.getDecimal();
+ int[] mantissa = measureMDMdl.getMantissa();
char[] type = measureMDMdl.getType();
byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
- ValueCompressionModel compressionModel = new ValueCompressionModel();
+ WriterCompressModel compressionModel = new WriterCompressModel();
DataType[] actualType = new DataType[measureCount];
DataType[] changedType = new DataType[measureCount];
COMPRESSION_TYPE[] compType = new COMPRESSION_TYPE[measureCount];
for (int i = 0; i < measureCount; i++) {
- CompressionFinder compresssionFinder = ValueCompressionUtil
- .getCompressionType(maxValue[i], minValue[i], decimal[i], type[i], dataTypeSelected[i]);
+ CompressionFinder compresssionFinder =
+ ValueCompressionUtil.getCompressionFinder(maxValue[i],
+ minValue[i], mantissa[i], type[i], dataTypeSelected[i]);
actualType[i] = compresssionFinder.actualDataType;
changedType[i] = compresssionFinder.changedDataType;
compType[i] = compresssionFinder.compType;
}
compressionModel.setMaxValue(maxValue);
- compressionModel.setDecimal(decimal);
+ compressionModel.setMantissa(mantissa);
compressionModel.setChangedDataType(changedType);
compressionModel.setCompType(compType);
compressionModel.setActualDataType(actualType);
compressionModel.setMinValue(minValue);
compressionModel.setUniqueValue(uniqueValue);
compressionModel.setType(type);
- compressionModel.setMinValueFactForAgg(measureMDMdl.getMinValueFactForAgg());
compressionModel.setDataTypeSelected(dataTypeSelected);
ValueCompressonHolder.UnCompressValue[] values = ValueCompressionUtil
.getUncompressedValues(compressionModel.getCompType(), compressionModel.getActualDataType(),
@@ -783,6 +709,24 @@ public final class ValueCompressionUtil {
return compressionModel;
}
+ /**
+ * Create Value compression model for read path
+ */
+ public static ReaderCompressModel getReaderCompressModel(ValueEncoderMeta meta) {
+ ReaderCompressModel compressModel = new ReaderCompressModel();
+ CompressionFinder compressFinder =
+ getCompressionFinder(meta.getMaxValue(), meta.getMinValue(), meta.getMantissa(),
+ meta.getType(), meta.getDataTypeSelected());
+ compressModel.setUnCompressValues(
+ ValueCompressionUtil.getUncompressedValues(
+ compressFinder.compType,
+ compressFinder.actualDataType,
+ compressFinder.changedDataType));
+ compressModel.setChangedDataType(compressFinder.changedDataType);
+ compressModel.setValueEncoderMeta(meta);
+ return compressModel;
+ }
+
public static byte[] convertToBytes(short[] values) {
ByteBuffer buffer = ByteBuffer.allocate(values.length * 2);
for (short val : values) {
@@ -874,61 +818,45 @@ public final class ValueCompressionUtil {
/**
* use to identify compression type.
*/
- public static enum COMPRESSION_TYPE {
+ public enum COMPRESSION_TYPE {
/**
- *
- */
- NONE, /**
- *
+ * adaptive compression based on data type
*/
- MAX_MIN, /**
- *
+ ADAPTIVE,
+
+ /**
+ * min max delta compression for double
*/
- NON_DECIMAL_CONVERT, /**
- *
+ DELTA_DOUBLE,
+
+ /**
+ * min max delta compression for short, int, long
*/
- MAX_MIN_NDC,
+ DELTA_NON_DECIMAL,
/**
- * custome
+ * for bigint
*/
- CUSTOM,
+ BIGINT,
- CUSTOM_BIGDECIMAL
+ /**
+ * for big decimal (PR388)
+ */
+ BIGDECIMAL
}
/**
* use to identify the type of data.
*/
- public static enum DataType {
- /**
- *
- */
- DATA_BYTE(), /**
- *
- */
- DATA_SHORT(), /**
- *
- */
- DATA_INT(), /**
- *
- */
- DATA_FLOAT(), /**
- *
- */
- DATA_LONG(), /**
- *
- */
- DATA_BIGINT(), /**
- *
- */
+ public enum DataType {
+ DATA_BYTE(),
+ DATA_SHORT(),
+ DATA_INT(),
+ DATA_FLOAT(),
+ DATA_LONG(),
+ DATA_BIGINT(),
DATA_DOUBLE();
-
- /**
- * DataType.
- */
private DataType() {
- //this.size = size;
}
}
@@ -938,25 +866,17 @@ public final class ValueCompressionUtil {
* best compression type
*/
private static class CompressionFinder implements Comparable<CompressionFinder> {
- /**
- * compType.
- */
+
private COMPRESSION_TYPE compType;
- /**
- * actualDataType.
- */
+
private DataType actualDataType;
- /**
- * changedDataType.
- */
+
private DataType changedDataType;
/**
* the size of changed data
*/
private int size;
- /**
- * priority.
- */
+
private PRIORITY priority;
/**
@@ -977,14 +897,14 @@ public final class ValueCompressionUtil {
/**
* CompressionFinder overloaded constructor.
*
+ * @param compType
* @param actualDataType
* @param changedDataType
* @param priority
- * @param compType
*/
- CompressionFinder(DataType actualDataType, DataType changedDataType, PRIORITY priority,
- COMPRESSION_TYPE compType) {
+ CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType changedDataType,
+ PRIORITY priority) {
super();
this.actualDataType = actualDataType;
this.changedDataType = changedDataType;
@@ -1041,7 +961,7 @@ public final class ValueCompressionUtil {
* ACTUAL is the highest priority and DIFFNONDECIMAL is the lowest
* priority
*/
- static enum PRIORITY {
+ enum PRIORITY {
/**
*
*/
@@ -1068,6 +988,3 @@ public final class ValueCompressionUtil {
}
}
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
index e7342b1..36c8ec3 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
@@ -13,9 +13,9 @@ import org.apache.carbondata.core.carbon.datastore.chunk.reader.measure.v1.Compr
import org.apache.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.compression.type.UnCompressByteArray;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressByteArray;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -30,7 +30,7 @@ public class CompressedMeasureChunkFileBasedReaderTest {
List<DataChunk> dataChunkList = new ArrayList<>();
dataChunkList.add(new DataChunk());
- ValueCompressionModel valueCompressionModel = new ValueCompressionModel();
+ WriterCompressModel writerCompressModel = new WriterCompressModel();
ValueCompressonHolder.UnCompressValue unCompressValue[] =
{ new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BYTE_ARRAY) };
@@ -38,16 +38,16 @@ public class CompressedMeasureChunkFileBasedReaderTest {
unCompressValue[0].setValueInBytes(valueInByte);
ValueCompressionUtil.DataType dataType[] = { ValueCompressionUtil.DataType.DATA_BYTE };
- valueCompressionModel.setUnCompressValues(unCompressValue);
- valueCompressionModel.setChangedDataType(dataType);
+ writerCompressModel.setUnCompressValues(unCompressValue);
+ writerCompressModel.setChangedDataType(dataType);
int decimal[] = { 5, 8, 2 };
- valueCompressionModel.setDecimal(decimal);
+ writerCompressModel.setMantissa(decimal);
Object maxValue[] = { 8 };
- valueCompressionModel.setMaxValue(maxValue);
+ writerCompressModel.setMaxValue(maxValue);
ValueEncoderMeta meta = new ValueEncoderMeta();
meta.setMaxValue(8.0);
meta.setMinValue(1.0);
- meta.setDecimal(1);
+ meta.setMantissa(1);
meta.setType('b');
List<ValueEncoderMeta> valueEncoderMetaList = new ArrayList<>();
valueEncoderMetaList.add(meta);