You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/06/13 17:04:57 UTC
[8/9] carbondata git commit: add EncodingStrategy
add EncodingStrategy
fix style“
fix style
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/edda2483
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/edda2483
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/edda2483
Branch: refs/heads/master
Commit: edda2483d074da7a26a5092fb896bea85689d8c5
Parents: 3a63bd2
Author: jackylk <ja...@huawei.com>
Authored: Tue Jun 13 22:38:48 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 13 22:33:19 2017 +0530
----------------------------------------------------------------------
.../dictionary/DoubleArrayTrieDictionary.java | 4 +-
.../core/compression/BigIntCompressor.java | 123 ---
.../core/compression/DoubleCompressor.java | 240 ------
.../core/compression/ValueCompressor.java | 55 --
.../core/datastore/MeasureDataWrapper.java | 28 -
.../datastore/chunk/MeasureColumnDataChunk.java | 20 +-
.../measure/AbstractMeasureChunkReader.java | 4 +
.../AbstractMeasureChunkReaderV2V3Format.java | 15 +
...CompressedMeasureChunkFileBasedReaderV1.java | 27 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 54 +-
...CompressedMeasureChunkFileBasedReaderV3.java | 38 +-
.../compression/MeasureMetaDataModel.java | 128 ----
.../compression/ReaderCompressModel.java | 59 --
.../compression/ValueCompressionHolder.java | 118 ---
.../compression/decimal/CompressByteArray.java | 107 ---
.../decimal/CompressionMaxMinByte.java | 115 ---
.../decimal/CompressionMaxMinDefault.java | 118 ---
.../decimal/CompressionMaxMinInt.java | 113 ---
.../decimal/CompressionMaxMinLong.java | 113 ---
.../decimal/CompressionMaxMinShort.java | 116 ---
.../nondecimal/CompressionNonDecimalByte.java | 99 ---
.../CompressionNonDecimalDefault.java | 102 ---
.../nondecimal/CompressionNonDecimalInt.java | 100 ---
.../nondecimal/CompressionNonDecimalLong.java | 102 ---
.../CompressionNonDecimalMaxMinByte.java | 105 ---
.../CompressionNonDecimalMaxMinDefault.java | 107 ---
.../CompressionNonDecimalMaxMinInt.java | 106 ---
.../CompressionNonDecimalMaxMinLong.java | 106 ---
.../CompressionNonDecimalMaxMinShort.java | 105 ---
.../nondecimal/CompressionNonDecimalShort.java | 100 ---
.../compression/none/CompressionNoneByte.java | 104 ---
.../none/CompressionNoneDefault.java | 103 ---
.../compression/none/CompressionNoneInt.java | 102 ---
.../compression/none/CompressionNoneLong.java | 103 ---
.../compression/none/CompressionNoneShort.java | 104 ---
.../dataholder/CarbonReadDataHolder.java | 48 --
.../impl/CompressedDataMeasureDataWrapper.java | 35 -
.../core/datastore/page/ColumnPage.java | 368 ++++++++-
.../core/datastore/page/ComplexColumnPage.java | 1 +
.../core/datastore/page/KeyColumnPage.java | 42 --
.../core/datastore/page/LazyColumnPage.java | 82 ++
.../core/datastore/page/PrimitiveCodec.java | 38 +
.../page/encoding/AdaptiveCompressionCodec.java | 68 ++
.../page/encoding/AdaptiveIntegerCodec.java | 206 ++++++
.../datastore/page/encoding/ColumnCodec.java | 35 -
.../page/encoding/ColumnPageCodec.java | 48 ++
.../page/encoding/CompressionCodec.java | 56 ++
.../page/encoding/DefaultEncodingStrategy.java | 146 ++++
.../page/encoding/DeltaIntegerCodec.java | 229 ++++++
.../datastore/page/encoding/DummyCodec.java | 37 -
.../datastore/page/encoding/EncodedData.java | 32 +
.../page/encoding/EncodingStrategy.java | 71 ++
.../encoding/UpscaleDeltaFloatingCodec.java | 187 +++++
.../page/encoding/UpscaleFloatingCodec.java | 180 +++++
.../page/statistics/ColumnPageStatistics.java | 172 -----
.../page/statistics/ColumnPageStatsVO.java | 221 ++++++
.../page/statistics/MeasurePageStatsVO.java | 2 +-
.../core/datastore/row/WriteStepRowUtil.java | 2 +-
.../core/memory/IntPointerBuffer.java | 112 +++
.../carbondata/core/memory/MemoryException.java | 29 +
.../core/memory/UnsafeMemoryManager.java | 149 ++++
.../core/metadata/datatype/DataType.java | 46 +-
.../table/column/CarbonImplicitDimension.java | 2 +-
.../schema/table/column/ColumnSchema.java | 2 +-
.../impl/AbstractScannedResultCollector.java | 13 +-
.../conditional/NotEqualsExpression.java | 1 -
.../executer/RowLevelFilterExecuterImpl.java | 14 +-
.../core/scan/result/AbstractScannedResult.java | 12 +-
.../vector/MeasureDataVectorProcessor.java | 36 +-
.../apache/carbondata/core/util/CarbonUtil.java | 2 +-
.../carbondata/core/util/CompressionFinder.java | 161 ----
.../core/util/ValueCompressionUtil.java | 741 -------------------
.../impl/CompressedDataMeasureWrapperTest.java | 45 --
.../DictionaryBasedResultCollectorTest.java | 152 ----
.../impl/RawBasedResultCollectorTest.java | 2 +-
.../impl/FilterQueryScannedResultTest.java | 208 ------
.../impl/NonFilterQueryScannedResultTest.java | 207 ------
.../core/util/ValueCompressionUtilTest.java | 486 ------------
.../src/test/resources/datawithmaxbigint.csv | 12 +
.../src/test/resources/datawithmaxminbigint.csv | 13 +
.../src/test/resources/datawithminbigint.csv | 12 +
.../dataload/TestLoadDataWithMaxMinBigInt.scala | 95 +++
.../dataload/TestLoadDataWithHiveSyntax.scala | 702 ------------------
...estLoadDataWithHiveSyntaxDefaultFormat.scala | 702 ++++++++++++++++++
.../TestLoadDataWithHiveSyntaxV1Format.scala | 709 ++++++++++++++++++
.../TestLoadDataWithHiveSyntaxV2Format.scala | 708 ++++++++++++++++++
.../sql/optimizer/CarbonLateDecodeRule.scala | 2 +-
.../UnsafeBatchParallelReadMergeSorterImpl.java | 3 +-
.../impl/UnsafeParallelReadMergeSorterImpl.java | 3 +-
...arallelReadMergeSorterWithBucketingImpl.java | 4 +-
.../newflow/sort/unsafe/IntPointerBuffer.java | 116 ---
.../sort/unsafe/UnsafeCarbonRowPage.java | 1 +
.../sort/unsafe/UnsafeMemoryManager.java | 126 ----
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 33 +-
.../unsafe/sort/UnsafeIntSortDataFormat.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 10 +-
.../processing/store/DefaultEncoder.java | 265 -------
.../carbondata/processing/store/TablePage.java | 28 +-
.../processing/store/TablePageEncoder.java | 201 +++++
.../processing/store/TablePageStatistics.java | 10 +-
.../store/writer/CarbonFactDataWriter.java | 3 +-
.../processing/store/writer/Encoder.java | 38 -
.../writer/v1/CarbonFactDataWriterImplV1.java | 4 +-
.../writer/v3/CarbonFactDataWriterImplV3.java | 5 +-
104 files changed, 4824 insertions(+), 6852 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index aeec846..f738008 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -109,7 +109,7 @@ public class DoubleArrayTrieDictionary {
}
/**
- * Get encode value of key
+ * Get apply value of key
*
* @param key
* @return
@@ -121,7 +121,7 @@ public class DoubleArrayTrieDictionary {
}
/**
- * Get encode value of bKeys
+ * Get apply value of bKeys
*
* @param bKeys
* @return
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
deleted file mode 100644
index 2db707f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.compression;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * It compresses big int data
- */
-public class BigIntCompressor extends ValueCompressor {
-
- @Override
- protected Object compressNonDecimalMaxMin(DataType convertedDataType,
- ColumnPage columnPage, int decimal, Object max) {
- // in case if bigint, decimal will be 0
- return compressMaxMin(convertedDataType, columnPage, max);
- }
-
- @Override
- protected Object compressNonDecimal(DataType convertedDataType, ColumnPage columnPage,
- int decimal) {
- // in case if bigint, decimal will be 0
- return compressAdaptive(convertedDataType, columnPage);
- }
-
- /**
- * 1. It gets delta value i.e difference of maximum value and actual value
- * 2. Convert the delta value computed above to convertedDatatype if it can
- * be stored with less byte
- */
- @Override
- protected Object compressMaxMin(DataType convertedDataType, ColumnPage columnPage,
- Object max) {
- long maxValue = (long) max;
- long[] value = columnPage.getLongPage();
- return compressValue(convertedDataType, value, maxValue, true);
- }
-
- /**
- * It converts actual value to converted data type if it can be stored with less bytes.
- */
- @Override
- protected Object compressAdaptive(DataType convertedDataType, ColumnPage columnPage) {
- long[] value = columnPage.getLongPage();
- return compressValue(convertedDataType, value, 0, false);
- }
-
- /**
- * It convert the value or delta value (max - actual) to converted data type.
- * Converted data type is computed based list of values it has.
- * for instance if value is 2,10,12,45
- * these value can be easily fitted in byte value,
- * hence it will be converted into byte to store.
- * @param convertedDataType
- * @param value
- * @param maxValue
- * @param isMinMax
- * @return
- */
- private Object compressValue(DataType convertedDataType, long[] value, long maxValue,
- boolean isMinMax) {
- switch (convertedDataType) {
- case BYTE:
- byte[] result = new byte[value.length];
- if (isMinMax) {
- for (int j = 0; j < value.length; j++) {
- result[j] = (byte) (maxValue - value[j]);
- }
- } else {
- for (int j = 0; j < value.length; j++) {
- result[j] = (byte) value[j];
- }
- }
- return result;
- case SHORT:
- short[] shortResult = new short[value.length];
- if (isMinMax) {
- for (int j = 0; j < value.length; j++) {
- shortResult[j] = (short) (maxValue - value[j]);
- }
- } else {
- for (int j = 0; j < value.length; j++) {
- shortResult[j] = (short) value[j];
- }
- }
- return shortResult;
- case INT:
- int[] intResult = new int[value.length];
- if (isMinMax) {
- for (int j = 0; j < value.length; j++) {
- intResult[j] = (int) (maxValue - value[j]);
- }
- } else {
- for (int j = 0; j < value.length; j++) {
- intResult[j] = (int) value[j];
- }
- }
- return intResult;
- default:
- if (isMinMax) {
- for (int j = 0; j < value.length; j++) {
- value[j] = (maxValue - value[j]);
- }
- }
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
deleted file mode 100644
index 5a24479..0000000
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.compression;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * Double compressor
- */
-public class DoubleCompressor extends ValueCompressor {
-
- @Override
- protected Object compressNonDecimalMaxMin(DataType convertedDataType,
- ColumnPage columnPage, int decimal, Object maxValue) {
- int i = 0;
- BigDecimal max = BigDecimal.valueOf((double)maxValue);
- double[] value = columnPage.getDoublePage();
- switch (convertedDataType) {
- case BYTE:
- byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- result[i] = (byte) (Math.round(diff * Math.pow(10, decimal)));
- i++;
- }
- return result;
- case SHORT:
- short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- shortResult[i] = (short) (Math.round(diff * Math.pow(10, decimal)));
- i++;
- }
- return shortResult;
- case INT:
- int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- intResult[i] = (int) (Math.round(diff * Math.pow(10, decimal)));
- i++;
- }
- return intResult;
- case LONG:
- long[] longResult = new long[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- longResult[i] = Math.round(diff * Math.pow(10, decimal));
- i++;
- }
- return longResult;
- case FLOAT:
- float[] floatResult = new float[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- floatResult[i] = (float) (Math.round(diff * Math.pow(10, decimal)));
- i++;
- }
- return floatResult;
- default:
- double[] defaultResult = new double[value.length];
- for (int j = 0; j < value.length; j++) {
- BigDecimal val = BigDecimal.valueOf(value[j]);
- double diff = max.subtract(val).doubleValue();
- defaultResult[i] = (Math.round(diff * Math.pow(10, decimal)));
- i++;
- }
- return defaultResult;
- }
- }
-
- @Override
- protected Object compressNonDecimal(DataType convertedDataType, ColumnPage columnPage,
- int decimal) {
- int i = 0;
- double[] value = columnPage.getDoublePage();
- switch (convertedDataType) {
- case BYTE:
- byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- result[i] = (byte) (Math.round(Math.pow(10, decimal) * value[j]));
- i++;
- }
- return result;
- case SHORT:
- short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * value[j]));
- i++;
- }
- return shortResult;
- case INT:
- int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- intResult[i] = (int) (Math.round(Math.pow(10, decimal) * value[j]));
- i++;
- }
- return intResult;
- case LONG:
- long[] longResult = new long[value.length];
- for (int j = 0; j < value.length; j++) {
- longResult[i] = Math.round(Math.pow(10, decimal) * value[j]);
- i++;
- }
- return longResult;
- case FLOAT:
- float[] floatResult = new float[value.length];
- for (int j = 0; j < value.length; j++) {
- floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * value[j]));
- i++;
- }
- return floatResult;
- default:
- double[] defaultResult = new double[value.length];
- for (int j = 0; j < value.length; j++) {
- defaultResult[i] = (double) (Math.round(Math.pow(10, decimal) * value[j]));
- i++;
- }
- return defaultResult;
- }
- }
-
- @Override
- protected Object compressMaxMin(DataType convertedDataType, ColumnPage columnPage,
- Object max) {
- double maxValue = (double) max;
- double[] value = columnPage.getDoublePage();
- int i = 0;
- switch (convertedDataType) {
- case BYTE:
- byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- result[i] = (byte) (maxValue - value[j]);
- i++;
- }
- return result;
- case SHORT:
- short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- shortResult[i] = (short) (maxValue - value[j]);
- i++;
- }
- return shortResult;
- case INT:
- int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- intResult[i] = (int) (maxValue - value[j]);
- i++;
- }
- return intResult;
- case LONG:
- long[] longResult = new long[value.length];
- for (int j = 0; j < value.length; j++) {
- longResult[i] = (long) (maxValue - value[j]);
- i++;
- }
- return longResult;
- case FLOAT:
- float[] floatResult = new float[value.length];
- for (int j = 0; j < value.length; j++) {
- floatResult[i] = (float) (maxValue - value[j]);
- i++;
- }
- return floatResult;
- default:
- double[] defaultResult = new double[value.length];
- for (int j = 0; j < value.length; j++) {
- defaultResult[i] = maxValue - value[j];
- i++;
- }
- return defaultResult;
- }
- }
-
- @Override
- protected Object compressAdaptive(DataType changedDataType, ColumnPage columnPage) {
- double[] value = columnPage.getDoublePage();
- int i = 0;
- switch (changedDataType) {
- case BYTE:
- byte[] result = new byte[value.length];
- for (int j = 0; j < value.length; j++) {
- result[i] = (byte) value[j];
- i++;
- }
- return result;
- case SHORT:
- short[] shortResult = new short[value.length];
- for (int j = 0; j < value.length; j++) {
- shortResult[i] = (short) value[j];
- i++;
- }
- return shortResult;
- case INT:
- int[] intResult = new int[value.length];
- for (int j = 0; j < value.length; j++) {
- intResult[i] = (int) value[j];
- i++;
- }
- return intResult;
- case LONG:
- long[] longResult = new long[value.length];
- for (int j = 0; j < value.length; j++) {
- longResult[i] = (long) value[j];
- i++;
- }
- return longResult;
- case FLOAT:
- float[] floatResult = new float[value.length];
- for (int j = 0; j < value.length; j++) {
- floatResult[i] = (float) value[j];
- i++;
- }
- return floatResult;
- default:
- return value;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
deleted file mode 100644
index b1a5b70..0000000
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.compression;
-
-import org.apache.carbondata.core.datastore.page.ColumnPage;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.CompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
-/**
- * Measure compressor
- */
-public abstract class ValueCompressor {
-
- public Object getCompressedValues(CompressionFinder compressionFinder,
- ColumnPage columnPage, Object maxValue, int decimal) {
- COMPRESSION_TYPE compType = compressionFinder.getCompType();
- DataType convertedDataType = compressionFinder.getConvertedDataType();
- switch (compType) {
- case ADAPTIVE:
- return compressAdaptive(convertedDataType, columnPage);
- case DELTA_DOUBLE:
- return compressMaxMin(convertedDataType, columnPage, maxValue);
- case BIGINT:
- return compressNonDecimal(convertedDataType, columnPage, decimal);
- default:
- return compressNonDecimalMaxMin(convertedDataType, columnPage, decimal, maxValue);
- }
- }
-
- abstract Object compressNonDecimalMaxMin(DataType convertedDataType,
- ColumnPage columnPage, int decimal, Object maxValue);
-
- abstract Object compressNonDecimal(DataType convertedDataType,
- ColumnPage columnPage, int decimal);
-
- abstract Object compressMaxMin(DataType convertedDataType,
- ColumnPage columnPage, Object maxValue);
-
- abstract Object compressAdaptive(DataType convertedDataType,
- ColumnPage columnPage);
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
deleted file mode 100644
index 6c7ae79..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore;
-
-import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
-
-/**
- * MeasureDataWrapper, interface.
- */
-public interface MeasureDataWrapper {
- CarbonReadDataHolder[] getValues();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index b86f865..049527b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -16,7 +16,7 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
/**
@@ -27,9 +27,9 @@ import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
public class MeasureColumnDataChunk {
/**
- * measure chunk
+ * measure column page
*/
- private CarbonReadDataHolder measureDataHolder;
+ private ColumnPage measurePage;
/**
* below to hold null value holds this information
@@ -39,17 +39,17 @@ public class MeasureColumnDataChunk {
private PresenceMeta nullValueIndexHolder;
/**
- * @return the measureDataHolder
+ * @return the measurePage
*/
- public CarbonReadDataHolder getMeasureDataHolder() {
- return measureDataHolder;
+ public ColumnPage getColumnPage() {
+ return measurePage;
}
/**
- * @param measureDataHolder the measureDataHolder to set
+ * @param measurePage the column page to set
*/
- public void setMeasureDataHolder(CarbonReadDataHolder measureDataHolder) {
- this.measureDataHolder = measureDataHolder;
+ public void setColumnPage(ColumnPage measurePage) {
+ this.measurePage = measurePage;
}
/**
@@ -67,6 +67,6 @@ public class MeasureColumnDataChunk {
}
public void freeMemory() {
- this.measureDataHolder.freeMemory();
+ this.measurePage.freeMemory();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
index 611d01a..5605554 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
@@ -17,12 +17,16 @@
package org.apache.carbondata.core.datastore.chunk.reader.measure;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.datastore.page.encoding.DefaultEncodingStrategy;
+import org.apache.carbondata.core.datastore.page.encoding.EncodingStrategy;
/**
* Measure block reader abstract class
*/
public abstract class AbstractMeasureChunkReader implements MeasureColumnChunkReader {
+ protected EncodingStrategy strategy = new DefaultEncodingStrategy();
+
/**
* file path from which blocks will be read
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index bcfe416..eba1777 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -23,8 +23,13 @@ import java.util.List;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.format.DataChunk2;
/**
* Abstract class for V2, V3 format measure column reader
@@ -123,4 +128,14 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
int startColumnBlockletIndex, int endColumnBlockletIndex) throws IOException;
+ protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
+ DataChunk2 measureColumnChunk, int copyPoint) {
+ // for measure, it should have only one ValueEncoderMeta
+ assert (measureColumnChunk.getEncoder_meta().size() == 1);
+ byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+ ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
+ ColumnPageCodec codec = strategy.createCodec(meta);
+ byte[] rawData = measureRawColumnChunk.getRawData().array();
+ return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 64e9b45..cdaaf81 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -24,13 +24,11 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
-import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
/**
* Compressed measure chunk reader
@@ -98,24 +96,17 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
int blockIndex = measureRawColumnChunk.getBlockletId();
DataChunk dataChunk = measureColumnChunks.get(blockIndex);
ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
- ReaderCompressModel compressModel = ValueCompressionUtil.getReaderCompressModel(meta);
- ValueCompressionHolder values = compressModel.getValueCompressionHolder();
- ByteBuffer rawData = measureRawColumnChunk.getRawData();
-
- // unCompress data
- values.uncompress(compressModel.getConvertedDataType(), rawData.array(),
- measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength(),
- compressModel.getMantissa(), compressModel.getMaxValue(), numberOfRows);
-
- CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
+ ColumnPageCodec codec = strategy.createCodec(meta);
+ ColumnPage page = codec.decode(measureRawColumnChunk.getRawData().array(),
+ measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
// create and set the data chunk
- MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
- datChunk.setMeasureDataHolder(measureDataHolder);
+ MeasureColumnDataChunk decodedChunk = new MeasureColumnDataChunk();
+ decodedChunk.setColumnPage(page);
// set the enun value indexes
- datChunk
+ decodedChunk
.setNullValueIndexHolder(dataChunk.getNullValueIndexForColumn());
- return datChunk;
+ return decodedChunk;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 9397767..a94193a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -18,22 +18,14 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastore.page.statistics.MeasurePageStatsVO;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.format.DataChunk2;
/**
@@ -115,53 +107,19 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk,
int pageNumber) throws IOException {
MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
- DataChunk2 measureColumnChunk = null;
int copyPoint = measureRawColumnChunk.getOffSet();
int blockIndex = measureRawColumnChunk.getBlockletId();
ByteBuffer rawData = measureRawColumnChunk.getRawData();
- if (measureColumnChunkOffsets.size() - 1 == blockIndex) {
- measureColumnChunk =
- CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
- synchronized (measureRawColumnChunk.getFileReader()) {
- rawData = measureRawColumnChunk.getFileReader().readByteBuffer(filePath,
- measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex),
- measureColumnChunk.data_page_length);
- }
- } else {
- measureColumnChunk =
- CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
+ DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint,
+ measureColumnChunkLength.get(blockIndex));
+ if (measureColumnChunkOffsets.size() - 1 != blockIndex) {
copyPoint += measureColumnChunkLength.get(blockIndex);
}
- List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
- for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
- valueEncodeMeta.add(
- CarbonUtil.deserializeEncoderMetaV2(measureColumnChunk.getEncoder_meta().get(i).array()));
- }
-
- MeasurePageStatsVO stats = CarbonUtil.getMeasurePageStats(valueEncodeMeta);
- int measureCount = valueEncodeMeta.size();
- CompressionFinder[] finders = new CompressionFinder[measureCount];
- DataType[] convertedType = new DataType[measureCount];
- for (int i = 0; i < measureCount; i++) {
- CompressionFinder compresssionFinder =
- ValueCompressionUtil.getCompressionFinder(stats.getMax(i), stats.getMin(i),
- stats.getDecimal(i), stats.getDataType(i), stats.getDataTypeSelected(i));
- finders[i] = compresssionFinder;
- convertedType[i] = compresssionFinder.getConvertedDataType();
- }
-
- ValueCompressionHolder values = ValueCompressionUtil.getValueCompressionHolder(finders)[0];
- // uncompress
- values.uncompress(convertedType[0], rawData.array(), copyPoint,
- measureColumnChunk.data_page_length, stats.getDecimal(0),
- stats.getMax(0), numberOfRows);
-
- CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
+ ColumnPage page = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint);
// set the data chunk
- datChunk.setMeasureDataHolder(measureDataHolder);
-
+ datChunk.setColumnPage(page);
// set the enun value indexes
datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
return datChunk;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index c97a3a9..325387c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -18,22 +18,14 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v3;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReaderV2V3Format;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastore.page.statistics.MeasurePageStatsVO;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
@@ -218,35 +210,13 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
// data chunk length + page offset
int copyPoint = measureRawColumnChunk.getOffSet() + measureColumnChunkLength
.get(measureRawColumnChunk.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
- List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
- for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
- valueEncodeMeta.add(
- CarbonUtil.deserializeEncoderMetaV3(measureColumnChunk.getEncoder_meta().get(i).array()));
- }
-
- MeasurePageStatsVO stats = CarbonUtil.getMeasurePageStats(valueEncodeMeta);
- int measureCount = valueEncodeMeta.size();
- CompressionFinder[] finders = new CompressionFinder[measureCount];
- DataType[] convertedType = new DataType[measureCount];
- for (int i = 0; i < measureCount; i++) {
- CompressionFinder compresssionFinder =
- ValueCompressionUtil.getCompressionFinder(stats.getMax(i), stats.getMin(i),
- stats.getDecimal(i), stats.getDataType(i), stats.getDataTypeSelected(i));
- finders[i] = compresssionFinder;
- convertedType[i] = compresssionFinder.getConvertedDataType();
- }
+ ColumnPage decodedPage = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint);
- ValueCompressionHolder values = ValueCompressionUtil.getValueCompressionHolder(finders)[0];
- // uncompress
- ByteBuffer rawData = measureRawColumnChunk.getRawData();
- values.uncompress(convertedType[0], rawData.array(), copyPoint,
- measureColumnChunk.data_page_length, stats.getDecimal(0),
- stats.getMax(0), measureRawColumnChunk.getRowCount()[pageNumber]);
- CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
// set the data chunk
- datChunk.setMeasureDataHolder(measureDataHolder);
+ datChunk.setColumnPage(decodedPage);
// set the null value indexes
datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
return datChunk;
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
deleted file mode 100644
index c2d0200..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-public class MeasureMetaDataModel {
- /**
- * maxValue
- */
- private Object[] maxValue;
-
- /**
- * minValue
- */
- private Object[] minValue;
-
- /**
- * mantissa
- */
- private int[] mantissa;
-
- /**
- * measureCount
- */
- private int measureCount;
-
- /**
- * uniqueValue
- */
- private Object[] uniqueValue;
-
- /**
- * type
- */
- private DataType[] type;
-
- /**
- * dataTypeSelected
- */
- private byte[] dataTypeSelected;
-
- public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
- int measureCount, Object[] uniqueValue, DataType[] type, byte[] dataTypeSelected) {
- this.minValue = minValue;
- this.maxValue = maxValue;
- this.mantissa = mantissa;
- this.measureCount = measureCount;
- this.uniqueValue = uniqueValue;
- this.type = type;
- this.dataTypeSelected = dataTypeSelected;
- }
-
- /**
- * get Max value
- *
- * @return
- */
- public Object[] getMaxValue() {
- return maxValue;
- }
-
- /**
- * getMinValue
- *
- * @return
- */
- public Object[] getMinValue() {
- return minValue;
- }
-
- /**
- * getMantissa
- *
- * @return
- */
- public int[] getMantissa() {
- return mantissa;
- }
-
- /**
- * getMeasureCount
- *
- * @return
- */
- public int getMeasureCount() {
- return measureCount;
- }
-
- /**
- * nonExistValue
- *
- * @return
- */
- public Object[] getUniqueValue() {
- return uniqueValue;
- }
-
- /**
- * @return the type
- */
- public DataType[] getType() {
- return type;
- }
-
- /**
- * @return the dataTypeSelected
- */
- public byte[] getDataTypeSelected() {
- return dataTypeSelected;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
deleted file mode 100644
index 60687d1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression;
-
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-// Used in read path for decompression preparation
-public class ReaderCompressModel {
- private ValueEncoderMeta valueEncoderMeta;
-
- private DataType convertedDataType;
-
- private ValueCompressionHolder valueHolder;
-
- public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
- this.valueEncoderMeta = valueEncoderMeta;
- }
-
- public DataType getConvertedDataType() {
- return convertedDataType;
- }
-
- public void setConvertedDataType(DataType convertedDataType) {
- this.convertedDataType = convertedDataType;
- }
-
- public Object getMaxValue() {
- return valueEncoderMeta.getMaxValue();
- }
-
- public int getMantissa() {
- return valueEncoderMeta.getDecimal();
- }
-
- public ValueCompressionHolder getValueCompressionHolder() {
- return valueHolder;
- }
-
- public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
- this.valueHolder = valueHolder;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
deleted file mode 100644
index 614eb32..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * ValueCompressionHolder is the base class for handling
- * compression / decompression of the measure data chunk
- */
-public abstract class ValueCompressionHolder<T> {
-
- /**
- * compressedValue
- */
- protected byte[] compressedValue;
-
- /**
- * @param compressor the compressor used to decompress the data
- * @param dataType data type of the data
- * @param data compressed data
- */
- protected void unCompress(Compressor compressor, DataType dataType, byte[] data, int offset,
- int length, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- switch (dataType) {
- case BYTE:
- setValue((T) compressor.unCompressByte(data, offset, length), numberOfRows, maxValueObject,
- decimalPlaces);
- break;
- case SHORT:
- setValue((T) compressor.unCompressShort(data, offset, length), numberOfRows, maxValueObject,
- decimalPlaces);
- break;
- case INT:
- setValue((T) compressor.unCompressInt(data, offset, length), numberOfRows, maxValueObject,
- decimalPlaces);
- break;
- case LONG:
- setValue((T) compressor.unCompressLong(data, offset, length), numberOfRows, maxValueObject,
- decimalPlaces);
- break;
- case FLOAT:
- setValue((T) compressor.unCompressFloat(data, offset, length), numberOfRows, maxValueObject,
- decimalPlaces);
- break;
- default:
- setValue((T) compressor.unCompressDouble(data, offset, length), numberOfRows,
- maxValueObject, decimalPlaces);
- break;
- }
- }
-
- /**
- * @param compressor the compressor used to compress the data
- * @param dataType data type of the data
- * @param data original data
- */
- public byte[] compress(Compressor compressor, DataType dataType, Object data) {
- switch (dataType) {
- case BYTE:
- return compressor.compressByte((byte[]) data);
- case SHORT:
- return compressor.compressShort((short[]) data);
- case INT:
- return compressor.compressInt((int[]) data);
- case LONG:
- return compressor.compressLong((long[]) data);
- case FLOAT:
- return compressor.compressFloat((float[]) data);
- case DOUBLE:
- default:
- return compressor.compressDouble((double[]) data);
- }
- }
-
- public abstract void setValue(T value);
-
- public abstract void setValue(T data, int numberOfRows, Object maxValueObject, int decimalPlaces);
-
- public abstract T getValue();
-
- public abstract void setValueInBytes(byte[] value);
-
- public abstract void compress();
-
- public abstract void uncompress(DataType dataType, byte[] compressData, int offset, int length,
- int decimal, Object maxValueObject, int numberOfRows);
-
- public byte[] getCompressedData() {
- return compressedValue;
- }
-
- public abstract long getLongValue(int index);
-
- public abstract double getDoubleValue(int index);
-
- public abstract BigDecimal getBigDecimalValue(int index);
-
- public abstract void freeMemory();
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
deleted file mode 100644
index 7a36e66..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressByteArray.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-public class CompressByteArray extends ValueCompressionHolder<byte[]> {
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- /**
- * value.
- */
- private byte[] value;
-
- @Override public void setValue(byte[] value) {
- this.value = value;
-
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
-
- }
-
- @Override public void compress() {
- compressedValue = compressor.compressByte(value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimal, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimal);
- }
-
- @Override public byte[] getValue() {
- List<byte[]> valsList = new ArrayList<byte[]>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- ByteBuffer buffer = ByteBuffer.wrap(value);
- buffer.rewind();
- int length = 0;
- byte[] actualValue = null;
- //CHECKSTYLE:OFF Approval No:Approval-367
- while (buffer.hasRemaining()) { //CHECKSTYLE:ON
- length = buffer.getInt();
- actualValue = new byte[length];
- buffer.get(actualValue);
- valsList.add(actualValue);
- }
- return valsList.get(0);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException("Get long value is not defined for CompressByteArray");
- }
-
- @Override public double getDoubleValue(int index) {
- throw new UnsupportedOperationException(
- "Get double value is not defined for CompressByteArray");
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- return this.measureChunkStore.getBigDecimal(index);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DECIMAL, numberOfRows);
- this.measureChunkStore.putData(data);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
deleted file mode 100644
index 45e3b47..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- protected byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- /**
- * actual data type
- */
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinByte(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public byte[] getValue() {
- return this.value;
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.BYTE, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (long) (maxValue - byteValue);
- }
-
- @Override public double getDoubleValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (maxValue - byteValue);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinByte");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
deleted file mode 100644
index 6bd1947..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- /**
- * actual data type
- */
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinDefault(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(double[] value) {
- this.value = value;
- }
-
- @Override public double[] getValue() {
- return this.value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DOUBLE, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionMaxMinDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- double doubleValue = measureChunkStore.getDouble(index);
- return maxValue - doubleValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinDefault");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(double[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DOUBLE, numberOfRows);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
deleted file mode 100644
index 60ddfea..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinInt(DataType actualType) {
- this.actualDataType = actualType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() {
- return this.value;
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.INT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return (long) (maxValue - intValue);
- }
-
- @Override public double getDoubleValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return maxValue - intValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinInt");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(int[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.INT, numberOfRows);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
deleted file mode 100644
index 159e741..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- /**
- * value.
- */
- protected long[] value;
-
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.LONG, value);
- }
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public long[] getValue() {
- return this.value;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return (long) maxValue - longValue;
- }
-
- @Override public double getDoubleValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return maxValue - longValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinLong");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(long[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.LONG, numberOfRows);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
deleted file mode 100644
index 1d36375..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- /**
- * value.
- */
- private short[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] value) {
- this.value = value;
-
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public short[] getValue() {
- return this.value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.SHORT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return (long) maxValue - shortValue;
- }
-
- @Override public double getDoubleValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return maxValue - shortValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinShort");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(short[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.SHORT, numberOfRows);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/edda2483/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
deleted file mode 100644
index 1c46dea..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.datastore.compression.nondecimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastore.compression.Compressor;
-import org.apache.carbondata.core.datastore.compression.CompressorFactory;
-import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public byte[] getValue() {
- return this.value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.BYTE, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject, int numberOfRows) {
- super.unCompress(compressor, dataType, compressedData, offset, length, numberOfRows,
- maxValueObject, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalByte");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getByte(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalByte");
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override
- public void setValue(byte[] data, int numberOfRows, Object maxValueObject, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.BYTE, numberOfRows);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-}