You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/01/14 07:57:59 UTC
[50/57] [abbrv] incubator-carbondata git commit: fix style
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index e47cf42..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public abstract class AbstractHeavyCompressedDoubleArrayDataStore
- implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
-{
-
- private LogService LOGGER =
- LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
-
- /**
- * values.
- */
- protected ValueCompressionHolder[] values;
-
- /**
- * compressionModel.
- */
- protected WriterCompressModel compressionModel;
-
- /**
- * type
- */
- private char[] type;
-
- /**
- * AbstractHeavyCompressedDoubleArrayDataStore constructor.
- *
- * @param compressionModel
- */
- public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
- this.compressionModel = compressionModel;
- if (null != compressionModel) {
- this.type = compressionModel.getType();
- values =
- new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
- }
- }
-
- // this method first invokes encoding routine to encode the data chunk,
- // followed by invoking compression routine for preparing the data chunk for writing.
- @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
- byte[][] returnValue = new byte[values.length][];
- for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
- values[i] = compressionModel.getValueCompressionHolder()[i];
- if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
- // first perform encoding of the data chunk
- values[i].setValue(
- ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
- .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
- compressionModel.getMaxValue()[i],
- compressionModel.getMantissa()[i]));
- } else {
- values[i].setValue(dataHolder[i].getWritableByteArrayValues());
- }
- values[i].compress();
- returnValue[i] = values[i].getCompressedData();
- }
-
- return returnValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
deleted file mode 100644
index ea42beb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-
-public class HeavyCompressedDoubleArrayDataInMemoryStore
- extends AbstractHeavyCompressedDoubleArrayDataStore {
-
- public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
- super(compressionModel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 9d4287a..7acfb39 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
new file mode 100644
index 0000000..95ec433
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+
+public interface FileHolder {
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath) throws IOException;
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
new file mode 100644
index 0000000..ff6ed34
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+ CarbonReadDataHolder[] getValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
new file mode 100644
index 0000000..c4e5d15
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+ /**
+ * This method will be used to get the writable key array.
+ * writable measure data array will hold below information:
+ * <size of measure data array><measure data array>
+ * total length will be 4 bytes for size + measure data array length
+ *
+ * @return writable array (compressed or normal)
+ */
+ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index dde8e76..f8c4374 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -29,11 +29,10 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl
- .MultiDimKeyVarLengthVariableSplitGenerator;
+import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c7ce3db..8774579 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -23,7 +23,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index ce09305..3c08f45 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 6edc977..0730607 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -20,8 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 549d6d8..342c392 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -22,8 +22,7 @@ import java.util.Arrays;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index c83f2d7..3d4e3bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,14 +18,10 @@
*/
package org.apache.carbondata.core.datastore.chunk.reader;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
- .v1.CompressedDimensionChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
- .v2.CompressedDimensionChunkFileBasedReaderV2;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
- .v1.CompressedMeasureChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
- .v2.CompressedMeasureChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 92dcb60..4981241 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 94004d9..a706d71 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
index 4cc24d9..2de673a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -19,9 +19,9 @@
package org.apache.carbondata.core.datastore.chunk.reader.dimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index dd487d9..a8d15fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 3bddf4e..ae5c9b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 48cb244..1f3fec0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -21,12 +21,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v1;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 3325299..2731163 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -23,13 +23,13 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
index df054c5..e1ae9b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
@@ -17,14 +17,10 @@
package org.apache.carbondata.core.datastore.chunk.store;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
- .SafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
- .SafeVariableLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
- .UnsafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
- .UnsafeVariableLengthDimesionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimesionDataChunkStore;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..b6155fb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+ private boolean alreadySorted;
+
+ private int[] dataAfterComp;
+
+ private int[] indexMap;
+
+ private byte[][] keyBlock;
+
+ private int[] dataIndexMap;
+
+ private int totalSize;
+
+ public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+ boolean isSortRequired) {
+ ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ if (isSortRequired) {
+ Arrays.sort(columnWithIndexs);
+ }
+ compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+ if (compressData) {
+ compressDataMyOwnWay(columnWithIndexs);
+ }
+ }
+
+ /**
+ * Create an object with each column array and respective index
+ *
+ * @return
+ */
+ private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ boolean isNoDictionary) {
+ ColumnWithIntIndex[] columnWithIndexs;
+ if (isNoDictionary) {
+ columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+ }
+
+ } else {
+ columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+ }
+ }
+
+ return columnWithIndexs;
+ }
+
+ private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+ byte[][] keyBlock) {
+ int[] indexes = new int[columnWithIndexs.length];
+ for (int i = 0; i < indexes.length; i++) {
+ indexes[i] = columnWithIndexs[i].getIndex();
+ keyBlock[i] = columnWithIndexs[i].getColumn();
+ }
+ this.keyBlock = keyBlock;
+ return indexes;
+ }
+
+ /**
+ * It compresses depends up on the sequence numbers.
+ * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+ * first array the start and end of sequential numbers and second array
+ * keeps the indexes of where sequential numbers starts. If there is no
+ * sequential numbers then the same array it returns with empty second
+ * array.
+ *
+ * @param indexes
+ */
+ public void compressMyOwnWay(int[] indexes) {
+ List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int k = 0;
+ int i = 1;
+ for (; i < indexes.length; i++) {
+ if (indexes[i] - indexes[i - 1] == 1) {
+ k++;
+ } else {
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ k = 0;
+ }
+ }
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ dataAfterComp = convertToArray(list);
+ if (indexes.length == dataAfterComp.length) {
+ indexMap = new int[0];
+ } else {
+ indexMap = convertToArray(map);
+ }
+ if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ alreadySorted = true;
+ }
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ /**
+ * @return the alreadySorted
+ */
+ public boolean isAlreadySorted() {
+ return alreadySorted;
+ }
+
+ /**
+ * @return the dataAfterComp
+ */
+ public int[] getDataAfterComp() {
+ return dataAfterComp;
+ }
+
+ /**
+ * @return the indexMap
+ */
+ public int[] getIndexMap() {
+ return indexMap;
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+ byte[] prvKey = indexes[0].getColumn();
+ List<ColumnWithIntIndex> list =
+ new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(indexes[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ for (int i = 1; i < indexes.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+ prvKey = indexes[i].getColumn();
+ list.add(indexes[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (indexes.length == keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ }
+
+ private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i).getColumn();
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public byte[] getMin() {
+ return keyBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return keyBlock[keyBlock.length - 1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..d7b980e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+ private byte[][] keyBlock;
+ private byte[][] sortedBlock;
+ private int totalSize;
+ private int[] dataIndexMap;
+
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+ boolean isNoDictionary) {
+ // without invertedindex but can be RLE
+ if (compressData) {
+ // with RLE
+ byte[] prvKey = keyBlockInput[0];
+ List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(keyBlockInput[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int length = keyBlockInput.length;
+ for(int i = 1; i < length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+ prvKey = keyBlockInput[i];
+ list.add(keyBlockInput[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (keyBlockInput.length == this.keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ } else {
+ this.keyBlock = keyBlockInput;
+ dataIndexMap = new int[0];
+ }
+
+ this.sortedBlock = new byte[keyBlock.length][];
+ System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+ if (isNoDictionary) {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+ }
+ });
+ } else {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+ }
+ });
+ }
+
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for(int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ private byte[][] convertToKeyArray(List<byte[]> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override
+ public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override
+ public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override
+ public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getDataAfterComp() {
+ return new int[0];
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getIndexMap() {
+ return new int[0];
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ @Override public byte[] getMin() {
+ return sortedBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return sortedBlock[sortedBlock.length - 1];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..4d4c19d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+public class ColumnGroupModel {
+
+ /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..4b35a7f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+ protected byte[] column;
+
+ private int index;
+
+ public ColumnWithIntIndex(byte[] column, int index) {
+ this.column = column;
+ this.index = index;
+ }
+
+ public ColumnWithIntIndex() {
+ }
+
+ /**
+ * @return the column
+ */
+ public byte[] getColumn() {
+ return column;
+ }
+
+ /**
+ * @param column the column to set
+ */
+ public void setColumn(byte[] column) {
+ this.column = column;
+ }
+
+ /**
+ * @return the index
+ */
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * @param index the index to set
+ */
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+ return Arrays.equals(column, o.column) && index == o.index;
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..a6eb6c4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+ implements Comparable<ColumnWithIntIndex> {
+
+ public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+ super(column, index);
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return UnsafeComparer.INSTANCE
+ .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+ return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + getIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..6c3acf9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.nio.ByteBuffer;
+
+public class ColumnarKeyStoreDataHolder {
+ private byte[] keyblockData;
+ private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+ final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.keyblockData = keyblockData;
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public int getSurrogateKey(int columnIndex) {
+ byte[] actual = new byte[4];
+ int startIndex;
+ if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+ startIndex =
+ columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+ .getEachRowSize();
+ } else {
+ startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+ }
+ int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+ System.arraycopy(keyblockData, startIndex, actual, destPos,
+ columnarKeyStoreMetadata.getEachRowSize());
+ return ByteBuffer.wrap(actual).getInt();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
new file mode 100644
index 0000000..706708f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+class ColumnarKeyStoreMetadata {
+
+ private int[] columnReverseIndex;
+
+ private int eachRowSize;
+
+ ColumnarKeyStoreMetadata(int eachRowSize) {
+ this.eachRowSize = eachRowSize;
+ }
+
+ /**
+ * @return the eachRowSize
+ */
+ int getEachRowSize() {
+ return eachRowSize;
+ }
+
+ /**
+ * @return the columnReverseIndex
+ */
+ int[] getColumnReverseIndex() {
+ return columnReverseIndex;
+ }
+
+ /**
+ * @param columnReverseIndex the columnReverseIndex to set
+ */
+ void setColumnReverseIndex(int[] columnReverseIndex) {
+ this.columnReverseIndex = columnReverseIndex;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
new file mode 100644
index 0000000..ac8f6b7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+public interface IndexStorage<T> {
+ boolean isAlreadySorted();
+
+ T getDataAfterComp();
+
+ T getIndexMap();
+
+ byte[][] getKeyBlock();
+
+ T getDataIndexMap();
+
+ int getTotalSize();
+
+ /**
+ * @return min value of block
+ */
+ byte[] getMin();
+
+ /**
+ * @return max value of block
+ */
+ byte[] getMax();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
new file mode 100644
index 0000000..4d4eac9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+public final class UnBlockIndexer {
+
+ private UnBlockIndexer() {
+
+ }
+
+ public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
+ int actualSize = indexData.length;
+ for (int i = 0; i < indexMap.length; i++) {
+ actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
+ }
+ int[] indexes = new int[actualSize];
+ int k = 0;
+ for (int i = 0; i < indexData.length; i++) {
+ int index = Arrays.binarySearch(indexMap, i);
+ if (index > -1) {
+ for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
+ indexes[k] = j;
+ k++;
+ }
+ i++;
+ } else {
+ indexes[k] = indexData[i];
+ k++;
+ }
+ }
+ return indexes;
+ }
+
+ public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
+ if (index.length < 1) {
+ return data;
+ }
+ int numberOfCopy = 0;
+ int actualSize = 0;
+ int srcPos = 0;
+ int destPos = 0;
+ for (int i = 1; i < index.length; i += 2) {
+ actualSize += index[i];
+ }
+ byte[] uncompressedData = new byte[actualSize * keyLen];
+ int picIndex = 0;
+ for (int i = 0; i < data.length; i += keyLen) {
+ numberOfCopy = index[picIndex * 2 + 1];
+ picIndex++;
+ for (int j = 0; j < numberOfCopy; j++) {
+ System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
+ destPos += keyLen;
+ }
+ srcPos += keyLen;
+ }
+ return uncompressedData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
new file mode 100644
index 0000000..d9803a0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public interface Compressor {
+
+ byte[] compressByte(byte[] unCompInput);
+
+ byte[] unCompressByte(byte[] compInput);
+
+ byte[] unCompressByte(byte[] compInput, int offset, int length);
+
+ byte[] compressShort(short[] unCompInput);
+
+ short[] unCompressShort(byte[] compInput);
+
+ short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
+ byte[] compressInt(int[] unCompInput);
+
+ int[] unCompressInt(byte[] compInput);
+
+ int[] unCompressInt(byte[] compInput, int offset, int length);
+
+ byte[] compressLong(long[] unCompInput);
+
+ long[] unCompressLong(byte[] compInput);
+
+ long[] unCompressLong(byte[] compInput, int offset, int length);
+
+ byte[] compressFloat(float[] unCompInput);
+
+ float[] unCompressFloat(byte[] compInput);
+
+ float[] unCompressFloat(byte[] compInput, int offset, int length);
+
+ byte[] compressDouble(double[] unCompInput);
+
+ double[] unCompressDouble(byte[] compInput);
+
+ double[] unCompressDouble(byte[] compInput, int offset, int length);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
new file mode 100644
index 0000000..022ac97
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+public class CompressorFactory {
+
+ private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
+
+ private final Compressor compressor;
+
+ private CompressorFactory() {
+ String compressorType = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+ switch (compressorType) {
+ case "snappy":
+ compressor = new SnappyCompressor();
+ break;
+ default:
+ throw new RuntimeException(
+ "Invalid compressor type provided! Please provide valid compressor type");
+ }
+ }
+
+ public static CompressorFactory getInstance() {
+ return COMPRESSOR_FACTORY;
+ }
+
+ public Compressor getCompressor() {
+ return compressor;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
new file mode 100644
index 0000000..1ded77d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public class MeasureMetaDataModel {
+ /**
+ * maxValue
+ */
+ private Object[] maxValue;
+
+ /**
+ * minValue
+ */
+ private Object[] minValue;
+
+ /**
+ * mantissa
+ */
+ private int[] mantissa;
+
+ /**
+ * measureCount
+ */
+ private int measureCount;
+
+ /**
+ * uniqueValue
+ */
+ private Object[] uniqueValue;
+
+ /**
+ * type
+ */
+ private char[] type;
+
+ /**
+ * dataTypeSelected
+ */
+ private byte[] dataTypeSelected;
+
+ public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
+ int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+ this.minValue = minValue;
+ this.maxValue = maxValue;
+ this.mantissa = mantissa;
+ this.measureCount = measureCount;
+ this.uniqueValue = uniqueValue;
+ this.type = type;
+ this.dataTypeSelected = dataTypeSelected;
+ }
+
+ /**
+ * get Max value
+ *
+ * @return
+ */
+ public Object[] getMaxValue() {
+ return maxValue;
+ }
+
+ /**
+ * getMinValue
+ *
+ * @return
+ */
+ public Object[] getMinValue() {
+ return minValue;
+ }
+
+ /**
+ * getMantissa
+ *
+ * @return
+ */
+ public int[] getMantissa() {
+ return mantissa;
+ }
+
+ /**
+ * getMeasureCount
+ *
+ * @return
+ */
+ public int getMeasureCount() {
+ return measureCount;
+ }
+
+ /**
+ * getUniqueValue
+ *
+ * @return
+ */
+ public Object[] getUniqueValue() {
+ return uniqueValue;
+ }
+
+ /**
+ * @return the type
+ */
+ public char[] getType() {
+ return type;
+ }
+
+ /**
+ * @return the dataTypeSelected
+ */
+ public byte[] getDataTypeSelected() {
+ return dataTypeSelected;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
new file mode 100644
index 0000000..7a5c670
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+// Used in read path for decompression preparation
+public class ReaderCompressModel {
+ private ValueEncoderMeta valueEncoderMeta;
+
+ private ValueCompressionUtil.DataType convertedDataType;
+
+ private ValueCompressionHolder valueHolder;
+
+ public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
+ this.valueEncoderMeta = valueEncoderMeta;
+ }
+
+ public ValueCompressionUtil.DataType getConvertedDataType() {
+ return convertedDataType;
+ }
+
+ public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+ this.convertedDataType = convertedDataType;
+ }
+
+ public Object getMaxValue() {
+ return valueEncoderMeta.getMaxValue();
+ }
+
+ public int getMantissa() {
+ return valueEncoderMeta.getMantissa();
+ }
+
+ public ValueCompressionHolder getValueCompressionHolder() {
+ return valueHolder;
+ }
+
+ public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
+ this.valueHolder = valueHolder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
new file mode 100644
index 0000000..dd78bb1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
+
+public class SnappyCompressor implements Compressor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SnappyCompressor.class.getName());
+
+ private final SnappyNative snappyNative;
+
+ public SnappyCompressor() {
+ Snappy snappy = new Snappy();
+ Field privateField = null;
+ try {
+ privateField = snappy.getClass().getDeclaredField("impl");
+ } catch (NoSuchFieldException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
+ privateField.setAccessible(true);
+ try {
+ snappyNative = (SnappyNative) privateField.get(snappy);
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput) {
+ try {
+ return Snappy.rawCompress(unCompInput, unCompInput.length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput) {
+ try {
+ return Snappy.uncompress(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return compInput;
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+ int uncompressedLength = 0;
+ byte[] data = null;
+ try {
+ uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ data = new byte[uncompressedLength];
+ Snappy.uncompress(compInput, offset, length, data, 0);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return data;
+ }
+
+ @Override public byte[] compressShort(short[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput) {
+ try {
+ return Snappy.uncompressShortArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+ try {
+ return Snappy.uncompressShortArray(compInput, offset, lenght);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressInt(int[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput) {
+ try {
+ return Snappy.uncompressIntArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressIntArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressLong(long[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput) {
+ try {
+ return Snappy.uncompressLongArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressLongArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressFloat(float[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput) {
+ try {
+ return Snappy.uncompressFloatArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressFloatArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressDouble(double[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput) {
+ try {
+ return Snappy.uncompressDoubleArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+ try {
+ int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ double[] result = new double[uncompressedLength / 8];
+ snappyNative.rawUncompress(compInput, offset, length, result, 0);
+ return result;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
new file mode 100644
index 0000000..8fed2ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+
+/**
+ * ValueCompressionHolder is the base class for handling
+ * compression / decompression of the measure data chunk
+ */
+public abstract class ValueCompressionHolder<T> {
+
+ /**
+ * compressedValue
+ */
+ protected byte[] compressedValue;
+
+ /**
+ * @param compressor the compressor used to decompress the data
+ * @param dataType data type of the data
+ * @param data compressed data
+ */
+ public void unCompress(Compressor compressor, DataType dataType, byte[] data,
+ int offset, int length) {
+ switch (dataType) {
+ case DATA_BYTE:
+ setValue((T)compressor.unCompressByte(data, offset, length));
+ break;
+ case DATA_SHORT:
+ setValue((T)compressor.unCompressShort(data, offset, length));
+ break;
+ case DATA_INT:
+ setValue((T)compressor.unCompressInt(data, offset, length));
+ break;
+ case DATA_LONG:
+ case DATA_BIGINT:
+ setValue((T)compressor.unCompressLong(data, offset, length));
+ break;
+ case DATA_FLOAT:
+ setValue((T)compressor.unCompressFloat(data, offset, length));
+ break;
+ default:
+ setValue((T)compressor.unCompressDouble(data, offset, length));
+ break;
+ }
+ }
+
+ /**
+ * @param compressor the compressor used to compress the data
+ * @param dataType data type of the data
+ * @param data original data
+ */
+ public byte[] compress(Compressor compressor, DataType dataType, Object data) {
+ switch (dataType) {
+ case DATA_BYTE:
+ return compressor.compressByte((byte[])data);
+ case DATA_SHORT:
+ return compressor.compressShort((short[])data);
+ case DATA_INT:
+ return compressor.compressInt((int[])data);
+ case DATA_LONG:
+ case DATA_BIGINT:
+ return compressor.compressLong((long[])data);
+ case DATA_FLOAT:
+ return compressor.compressFloat((float[])data);
+ case DATA_DOUBLE:
+ default:
+ return compressor.compressDouble((double[])data);
+ }
+ }
+
+ public abstract void setValue(T value);
+
+ public abstract T getValue();
+
+ public abstract void setValueInBytes(byte[] value);
+
+ public abstract void compress();
+
+ public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
+ int length, int decimal, Object maxValueObject);
+
+ public byte[] getCompressedData() { return compressedValue; }
+
+ public abstract long getLongValue(int index);
+
+ public abstract double getDoubleValue(int index);
+
+ public abstract BigDecimal getBigDecimalValue(int index);
+
+ public abstract void freeMemory();
+
+
+}