You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/12/06 04:04:13 UTC
[2/4] incubator-carbondata git commit: modify compress interface
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
index d12ebe1..6a3655d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneInt.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -39,7 +39,7 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
/**
* intCompressor.
*/
- private static Compressor<int[]> intCompressor = SnappyCompression.SnappyIntCompression.INSTANCE;
+ private static Compressor compressor = CompressorFactory.getInstance();
/**
* value.
*/
@@ -70,9 +70,8 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
- byte1.setValue(intCompressor.compress(value));
-
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
+ byte1.setValue(compressor.compressInt(value));
return byte1;
}
@@ -95,6 +94,11 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
@Override public CarbonReadDataHolder getValues(int decimal, Object maxValueObject) {
switch (actualDataType) {
+ case DATA_SHORT:
+ return unCompressShort();
+ case DATA_INT:
+ return unCompressInt();
+ case DATA_LONG:
case DATA_BIGINT:
return unCompressLong();
default:
@@ -102,13 +106,28 @@ public class UnCompressNoneInt implements ValueCompressonHolder.UnCompressValue<
}
}
+ private CarbonReadDataHolder unCompressShort() {
+ CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ short[] vals = new short[value.length];
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = (short)value[i];
+ }
+ dataHolder.setReadableShortValues(vals);
+ return dataHolder;
+ }
+
+ private CarbonReadDataHolder unCompressInt() {
+ CarbonReadDataHolder dataHolder = new CarbonReadDataHolder();
+ dataHolder.setReadableIntValues(value);
+ return dataHolder;
+ }
+
private CarbonReadDataHolder unCompressDouble() {
CarbonReadDataHolder dataHolderInfoObj = new CarbonReadDataHolder();
double[] vals = new double[value.length];
for (int i = 0; i < vals.length; i++) {
vals[i] = value[i];
}
-
dataHolderInfoObj.setReadableDoubleValues(vals);
return dataHolderInfoObj;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
index 4661b7a..eebfdeb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneLong.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -39,8 +39,7 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
/**
* longCompressor.
*/
- private static Compressor<long[]> longCompressor =
- SnappyCompression.SnappyLongCompression.INSTANCE;
+ private static Compressor compressor = CompressorFactory.getInstance();
/**
* value.
*/
@@ -68,8 +67,8 @@ public class UnCompressNoneLong implements ValueCompressonHolder.UnCompressValue
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
- UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
- byte1.setValue(longCompressor.compress(value));
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
+ byte1.setValue(compressor.compressLong(value));
return byte1;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
index 2fdf514..3cfb530 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/UnCompressNoneShort.java
@@ -24,7 +24,7 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
@@ -40,8 +40,7 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
/**
* shortCompressor.
*/
- private static Compressor<short[]> shortCompressor =
- SnappyCompression.SnappyShortCompression.INSTANCE;
+ private static Compressor compressor = CompressorFactory.getInstance();
/**
* value.
@@ -56,7 +55,6 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
@Override public void setValue(short[] shortValue) {
this.shortValue = shortValue;
-
}
@Override public ValueCompressonHolder.UnCompressValue getNew() {
@@ -69,12 +67,9 @@ public class UnCompressNoneShort implements ValueCompressonHolder.UnCompressValu
}
@Override public ValueCompressonHolder.UnCompressValue compress() {
-
- UnCompressNoneByte byte1 = new UnCompressNoneByte(this.actualDataType);
- byte1.setValue(shortCompressor.compress(shortValue));
-
+ UnCompressNoneByte byte1 = new UnCompressNoneByte(actualDataType);
+ byte1.setValue(compressor.compressShort(shortValue));
return byte1;
-
}
@Override public ValueCompressonHolder.UnCompressValue uncompress(DataType dataType) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
index 79507ce..1cfc7a2 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
@@ -21,78 +21,55 @@ package org.apache.carbondata.core.datastorage.store.dataholder;
import java.math.BigDecimal;
+// This class is used with Uncompressor to hold the decompressed column chunk in memory
public class CarbonReadDataHolder {
- /**
- * doubleValues
- */
- private double[] doubleValues;
+ private byte[][] byteValues;
+
+ private short[] shortValues;
+
+ private int[] intValues;
- /**
- * longValues
- */
private long[] longValues;
- /**
- * bigDecimalValues
- */
private BigDecimal[] bigDecimalValues;
- /**
- * byteValues
- */
- private byte[][] byteValues;
+ private double[] doubleValues;
- /**
- * @return the doubleValues
- */
- public double[] getReadableDoubleValues() {
- return doubleValues;
+ public void setReadableByteValues(byte[][] byteValues) {
+ this.byteValues = byteValues;
}
- /**
- * @param doubleValues the doubleValues to set
- */
- public void setReadableDoubleValues(double[] doubleValues) {
- this.doubleValues = doubleValues;
+ public void setReadableShortValues(short[] shortValues) {
+ this.shortValues = shortValues;
}
- /**
- * @return the byteValues
- */
- public byte[][] getReadableByteArrayValues() {
- return byteValues;
+ public void setReadableIntValues(int[] intValues) {
+ this.intValues = intValues;
}
- /**
- * @param longValues the longValues to set
- */
public void setReadableLongValues(long[] longValues) {
this.longValues = longValues;
}
- /**
- * @param longValues the bigDecimalValues to set
- */
public void setReadableBigDecimalValues(BigDecimal[] bigDecimalValues) {
this.bigDecimalValues = bigDecimalValues;
}
- /**
- * @param byteValues the byteValues to set
- */
- public void setReadableByteValues(byte[][] byteValues) {
- this.byteValues = byteValues;
+ public void setReadableDoubleValues(double[] doubleValues) {
+ this.doubleValues = doubleValues;
}
- /**
- * below method will be used to get the double value by index
- *
- * @param index
- * @return double values
- */
- public double getReadableDoubleValueByIndex(int index) {
- return this.doubleValues[index];
+ public byte[] getReadableByteArrayValueByIndex(int index) {
+ return this.byteValues[index];
+ }
+
+ public long getReadableShortValueByIndex(int index) {
+ return this.shortValues[index];
+ }
+
+ public long getReadableIntValueByIndex(int index) {
+ return this.intValues[index];
}
public long getReadableLongValueByIndex(int index) {
@@ -103,13 +80,8 @@ public class CarbonReadDataHolder {
return this.bigDecimalValues[index];
}
- /**
- * below method will be used to get the readable byte array value by index
- *
- * @param index
- * @return byte array value
- */
- public byte[] getReadableByteArrayValueByIndex(int index) {
- return this.byteValues[index];
+ public double getReadableDoubleValueByIndex(int index) {
+ return this.doubleValues[index];
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataFileStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataFileStore.java
deleted file mode 100644
index 2fd873b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataFileStore.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
-
-public class HeavyCompressedDoubleArrayDataFileStore
- extends AbstractHeavyCompressedDoubleArrayDataStore {
- /**
- * measuresOffsetsArray.
- */
- private long[] measuresOffsetsArray;
-
- /**
- * measuresLengthArray.
- */
- private int[] measuresLengthArray;
-
- /**
- * fileName.
- */
- private String fileName;
-
- /**
- * HeavyCompressedDoubleArrayDataFileStore.
- *
- * @param compressionModel
- * @param measuresOffsetsArray
- * @param measuresLengthArray
- * @param fileName
- */
- public HeavyCompressedDoubleArrayDataFileStore(ValueCompressionModel compressionModel,
- long[] measuresOffsetsArray, int[] measuresLengthArray, String fileName) {
- super(compressionModel);
- if (null != compressionModel) {
- this.fileName = fileName;
- this.measuresLengthArray = measuresLengthArray;
- this.measuresOffsetsArray = measuresOffsetsArray;
- for (int i = 0; i < values.length; i++) {
- values[i] = compressionModel.getUnCompressValues()[i].getNew().getCompressorObject();
- }
- }
- }
-
- @Override public MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
-
- if (cols != null) {
- for (int i = 0; i < cols.length; i++) {
- ValueCompressonHolder.UnCompressValue copy = values[cols[i]].getNew();
- copy.setValue(fileHolder
- .readByteArray(fileName, measuresOffsetsArray[cols[i]], measuresLengthArray[cols[i]]));
- vals[cols[i]] = copy.uncompress(compressionModel.getChangedDataType()[cols[i]])
- .getValues(compressionModel.getDecimal()[cols[i]],
- compressionModel.getMaxValue()[cols[i]]);
- copy = null;
- }
- } else {
- for (int j = 0; j < vals.length; j++) {
- ValueCompressonHolder.UnCompressValue copy = values[j].getNew();
- copy.setValue(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[j], measuresLengthArray[j]));
- vals[j] = copy.uncompress(compressionModel.getChangedDataType()[j])
- .getValues(compressionModel.getDecimal()[j], compressionModel.getMaxValue()[j]);
- copy = null;
- }
- }
- return new CompressedDataMeasureDataWrapper(vals);
-
- }
-
- @Override public MeasureDataWrapper getBackData(int cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[values.length];
- ValueCompressonHolder.UnCompressValue copy = values[cols].getNew();
- copy.setValue(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[cols], measuresLengthArray[cols]));
- vals[cols] = copy.uncompress(compressionModel.getChangedDataType()[cols])
- .getValues(compressionModel.getDecimal()[cols], compressionModel.getMaxValue()[cols]);
- return new CompressedDataMeasureDataWrapper(vals);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataFileStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataFileStore.java
deleted file mode 100644
index 182b868..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataFileStore.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
-
-public class DoubleArrayDataFileStore extends AbstractDoubleArrayDataStore {
-
- private long[] measuresOffsetsArray;
-
- private int[] measuresLengthArray;
-
- private String fileName;
-
- public DoubleArrayDataFileStore(ValueCompressionModel compressionModel,
- long[] measuresOffsetsArray, String fileName, int[] measuresLengthArray) {
- super(compressionModel);
- this.fileName = fileName;
- this.measuresLengthArray = measuresLengthArray;
- this.measuresOffsetsArray = measuresOffsetsArray;
- }
-
- @Override public MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- UnCompressValue[] unComp = new UnCompressValue[measuresLengthArray.length];
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[measuresLengthArray.length];
- if (cols != null) {
- for (int i = 0; i < cols.length; i++) {
- unComp[cols[i]] = compressionModel.getUnCompressValues()[cols[i]].getNew();
- unComp[cols[i]].setValueInBytes(fileHolder
- .readByteArray(fileName, measuresOffsetsArray[cols[i]], measuresLengthArray[cols[i]]));
- vals[cols[i]] = unComp[cols[i]].getValues(compressionModel.getDecimal()[cols[i]],
- compressionModel.getMaxValue()[cols[i]]);
- }
- } else {
- for (int i = 0; i < unComp.length; i++) {
-
- unComp[i] = compressionModel.getUnCompressValues()[i].getNew();
- unComp[i].setValueInBytes(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[i], measuresLengthArray[i]));
- vals[i] = unComp[i]
- .getValues(compressionModel.getDecimal()[i], compressionModel.getMaxValue()[i]);
- }
- }
- return new CompressedDataMeasureDataWrapper(vals);
- }
-
- @Override public MeasureDataWrapper getBackData(int cols, FileHolder fileHolder) {
- if (null == compressionModel) {
- return null;
- }
- UnCompressValue[] unComp = new UnCompressValue[measuresLengthArray.length];
- CarbonReadDataHolder[] vals = new CarbonReadDataHolder[measuresLengthArray.length];
-
- unComp[cols] = compressionModel.getUnCompressValues()[cols].getNew();
- unComp[cols].setValueInBytes(
- fileHolder.readByteArray(fileName, measuresOffsetsArray[cols], measuresLengthArray[cols]));
- vals[cols] = unComp[cols]
- .getValues(compressionModel.getDecimal()[cols], compressionModel.getMaxValue()[cols]);
- return new CompressedDataMeasureDataWrapper(vals);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
index 1effb25..3aaa09d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/uncompressed/DoubleArrayDataInMemoryStore.java
@@ -26,11 +26,6 @@ import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonH
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
-/**
- * DoubleArrayDataInMemoryStore.
- *
- * @author S71955
- */
public class DoubleArrayDataInMemoryStore extends AbstractDoubleArrayDataStore {
// /**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/AbstractColumnarKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/AbstractColumnarKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/AbstractColumnarKeyStore.java
deleted file mode 100644
index 51c5fb8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/AbstractColumnarKeyStore.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStore;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
-
-public abstract class AbstractColumnarKeyStore implements ColumnarKeyStore {
-
- /**
- * compressor will be used to compress the data
- */
- protected static final Compressor<byte[]> COMPRESSOR =
- SnappyCompression.SnappyByteCompression.INSTANCE;
-
- protected ColumnarKeyStoreInfo columnarStoreInfo;
-
- protected byte[][] columnarKeyBlockDataIndex;
-
- protected byte[][] columnarKeyBlockData;
-
- protected Map<Integer, Integer> mapOfColumnIndexAndColumnBlockIndex;
-
- protected Map<Integer, Integer> mapOfAggDataIndex;
-
- protected byte[][] columnarUniqueblockKeyBlockIndex;
-
- public AbstractColumnarKeyStore(ColumnarKeyStoreInfo columnarStoreInfo, boolean isInMemory,
- FileHolder fileHolder) {
- this.columnarStoreInfo = columnarStoreInfo;
- this.mapOfColumnIndexAndColumnBlockIndex =
- new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- this.mapOfAggDataIndex =
- new HashMap<Integer, Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- int index = 0;
- for (int i = 0; i < this.columnarStoreInfo.getIsSorted().length; i++) {
- if (!this.columnarStoreInfo.getIsSorted()[i]) {
- this.mapOfColumnIndexAndColumnBlockIndex.put(i, index++);
- }
- }
- index = 0;
- for (int i = 0; i < this.columnarStoreInfo.getAggKeyBlock().length; i++) {
- if (this.columnarStoreInfo.getAggKeyBlock()[i]) {
- mapOfAggDataIndex.put(i, index++);
- }
- }
- if (isInMemory) {
- this.columnarKeyBlockData = new byte[this.columnarStoreInfo.getIsSorted().length][];
- this.columnarKeyBlockDataIndex = new byte[this.mapOfColumnIndexAndColumnBlockIndex.size()][];
- this.columnarUniqueblockKeyBlockIndex = new byte[this.mapOfAggDataIndex.size()][];
- for (int i = 0; i < columnarStoreInfo.getSizeOfEachBlock().length; i++) {
- columnarKeyBlockData[i] = fileHolder.readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockOffsets()[i], columnarStoreInfo.getKeyBlockLengths()[i]);
-
- if (!this.columnarStoreInfo.getIsSorted()[i]) {
- this.columnarKeyBlockDataIndex[mapOfColumnIndexAndColumnBlockIndex.get(i)] = fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockIndexOffsets()[mapOfColumnIndexAndColumnBlockIndex
- .get(i)],
- columnarStoreInfo.getKeyBlockIndexLength()[mapOfColumnIndexAndColumnBlockIndex
- .get(i)]);
- }
-
- if (this.columnarStoreInfo.getAggKeyBlock()[i]) {
- this.columnarUniqueblockKeyBlockIndex[mapOfAggDataIndex.get(i)] = fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getDataIndexMapOffsets()[mapOfAggDataIndex.get(i)],
- columnarStoreInfo.getDataIndexMapLength()[mapOfAggDataIndex.get(i)]);
- }
- }
- }
- }
-
- protected int[] getColumnIndexForNonFilter(int[] columnIndex) {
- int[] columnIndexTemp = new int[columnIndex.length];
-
- for (int i = 0; i < columnIndex.length; i++) {
- columnIndexTemp[columnIndex[i]] = i;
- }
- return columnIndexTemp;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarFileKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarFileKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarFileKeyStore.java
deleted file mode 100644
index 94d4b8f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarFileKeyStore.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar.compressed;
-
-import java.util.List;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreMetadata;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.impl.key.columnar.AbstractColumnarKeyStore;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-public class CompressedColumnarFileKeyStore extends AbstractColumnarKeyStore {
-
- public CompressedColumnarFileKeyStore(ColumnarKeyStoreInfo columnarStoreInfo) {
- super(columnarStoreInfo, false, null);
- }
-
- @Override public ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder,
- int[] blockIndex, boolean[] needCompressedData, int[] noDictionaryColIndexes) {
- ColumnarKeyStoreDataHolder[] columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder[blockIndex.length];
-
- for (int i = 0; i < columnarKeyStoreDataHolders.length; i++) {
- byte[] columnarKeyBlockData = null;
- int[] columnKeyBlockIndex = null;
- int[] columnKeyBlockReverseIndexes = null;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = null;
- int[] dataIndex = null;
- boolean isUnCompressed = true;
- columnarKeyBlockData = COMPRESSOR.unCompress(fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockOffsets()[blockIndex[i]],
- columnarStoreInfo.getKeyBlockLengths()[blockIndex[i]]));
- boolean isNoDictionaryBlock =
- CompressedColumnarKeyStoreUtil.isNoDictionaryBlock(noDictionaryColIndexes, blockIndex[i]);
- if (!isNoDictionaryBlock && this.columnarStoreInfo.getAggKeyBlock()[blockIndex[i]]) {
- dataIndex = columnarStoreInfo.getNumberCompressor().unCompress(fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getDataIndexMapOffsets()[mapOfAggDataIndex.get(blockIndex[i])],
- columnarStoreInfo.getDataIndexMapLength()[mapOfAggDataIndex.get(blockIndex[i])]));
- if (!needCompressedData[i]) {
- columnarKeyBlockData = UnBlockIndexer.uncompressData(columnarKeyBlockData, dataIndex,
- columnarStoreInfo.getSizeOfEachBlock()[blockIndex[i]]);
- dataIndex = null;
- } else {
- isUnCompressed = false;
- }
- }
- if (!columnarStoreInfo.getIsSorted()[blockIndex[i]]) {
- columnKeyBlockIndex = CarbonUtil
- .getUnCompressColumnIndex(columnarStoreInfo.getKeyBlockIndexLength()[blockIndex[i]],
- fileHolder.readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockIndexOffsets()[blockIndex[i]],
- columnarStoreInfo.getKeyBlockIndexLength()[blockIndex[i]]),
- columnarStoreInfo.getNumberCompressor());
- columnKeyBlockReverseIndexes = getColumnIndexForNonFilter(columnKeyBlockIndex);
- }
- //Since its an high cardinality dimension adding the direct surrogates as part of
- //columnarKeyStoreMetadata so that later it will be used with bytearraywrapper instance.
- if (isNoDictionaryBlock) {
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(0);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndexes);
- columnarKeyStoreMetadata.setNoDictionaryValColumn(true);
- columnarKeyStoreMetadata.setUnCompressed(true);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[i]]);
- //System is reading the direct surrogates data from byte array which contains both
- // length and the direct surrogates data
- List<byte[]> noDictionaryValBasedKeyBlockData = CompressedColumnarKeyStoreUtil
- .readColumnarKeyBlockDataForNoDictionaryCols(columnarKeyBlockData);
- columnarKeyStoreDataHolders[i] =
- new ColumnarKeyStoreDataHolder(noDictionaryValBasedKeyBlockData,
- columnarKeyStoreMetadata);
- } else {
- columnarKeyStoreMetadata =
- new ColumnarKeyStoreMetadata(columnarStoreInfo.getSizeOfEachBlock()[blockIndex[i]]);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[i]]);
- columnarKeyStoreMetadata.setDataIndex(dataIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndexes);
- columnarKeyStoreMetadata.setUnCompressed(isUnCompressed);
- columnarKeyStoreDataHolders[i] =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockData, columnarKeyStoreMetadata);
- }
- }
- return columnarKeyStoreDataHolders;
- }
-
- @Override
- public ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
- boolean needCompressedData, int[] noDictionaryColIndexes) {
- byte[] columnarKeyBlockData = null;
- int[] columnKeyBlockIndex = null;
- int[] columnKeyBlockReverseIndex = null;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = null;
- int[] dataIndex = null;
- boolean isUnCompressed = true;
- columnarKeyBlockData = COMPRESSOR.unCompress(fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockOffsets()[blockIndex],
- columnarStoreInfo.getKeyBlockLengths()[blockIndex]));
- boolean isNoDictionaryBlock =
- CompressedColumnarKeyStoreUtil.isNoDictionaryBlock(noDictionaryColIndexes, blockIndex);
- if (!isNoDictionaryBlock && this.columnarStoreInfo.getAggKeyBlock()[blockIndex]) {
- dataIndex = columnarStoreInfo.getNumberCompressor().unCompress(fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getDataIndexMapOffsets()[mapOfAggDataIndex.get(blockIndex)],
- columnarStoreInfo.getDataIndexMapLength()[mapOfAggDataIndex.get(blockIndex)]));
- if (!needCompressedData) {
- columnarKeyBlockData = UnBlockIndexer.uncompressData(columnarKeyBlockData, dataIndex,
- columnarStoreInfo.getSizeOfEachBlock()[blockIndex]);
- dataIndex = null;
- } else {
- isUnCompressed = false;
- }
- }
- if (!columnarStoreInfo.getIsSorted()[blockIndex]) {
- columnKeyBlockIndex = CarbonUtil
- .getUnCompressColumnIndex(columnarStoreInfo.getKeyBlockIndexLength()[blockIndex],
- fileHolder.readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockIndexOffsets()[blockIndex],
- columnarStoreInfo.getKeyBlockIndexLength()[blockIndex]),
- columnarStoreInfo.getNumberCompressor());
- columnKeyBlockReverseIndex = getColumnIndexForNonFilter(columnKeyBlockIndex);
- }
- //Since its an high cardinality dimension, For filter queries.
- if (isNoDictionaryBlock) {
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(0);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolders = CompressedColumnarKeyStoreUtil
- .createColumnarKeyStoreMetadataForHCDims(blockIndex, columnarKeyBlockData,
- columnKeyBlockIndex, columnKeyBlockReverseIndex, columnarStoreInfo);
- new ColumnarKeyStoreDataHolder(columnarKeyBlockData, columnarKeyStoreMetadata);
- return columnarKeyStoreDataHolders;
- }
- columnarKeyStoreMetadata =
- new ColumnarKeyStoreMetadata(columnarStoreInfo.getSizeOfEachBlock()[blockIndex]);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex]);
- columnarKeyStoreMetadata.setDataIndex(dataIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndex);
- columnarKeyStoreMetadata.setUnCompressed(isUnCompressed);
-
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockData, columnarKeyStoreMetadata);
- return columnarKeyStoreDataHolders;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarInMemoryStore.java
deleted file mode 100644
index 5d3d4b5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarInMemoryStore.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar.compressed;
-
-import java.util.List;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreMetadata;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.impl.key.columnar.AbstractColumnarKeyStore;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-public class CompressedColumnarInMemoryStore extends AbstractColumnarKeyStore {
-
- public CompressedColumnarInMemoryStore(ColumnarKeyStoreInfo columnarStoreInfo,
- FileHolder fileHolder) {
- super(columnarStoreInfo, true, fileHolder);
- }
-
- @Override public ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder,
- int[] blockIndex, boolean[] needCompressedData, int[] noDictionaryColIndexes) {
- ColumnarKeyStoreDataHolder[] columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder[blockIndex.length];
- for (int i = 0; i < columnarKeyStoreDataHolders.length; i++) {
- byte[] columnarKeyBlockDataTemp = null;
- int[] columnKeyBlockIndex = null;
- int[] columnKeyBlockReverseIndexes = null;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = null;
- int columnarKeyBlockIndex = 0;
- int[] dataIndex = null;
- boolean isUnCompressed = true;
- columnarKeyBlockDataTemp = COMPRESSOR.unCompress(columnarKeyBlockData[blockIndex[i]]);
- boolean isNoDictionaryBlock =
- CompressedColumnarKeyStoreUtil.isNoDictionaryBlock(noDictionaryColIndexes, blockIndex[i]);
- if (!isNoDictionaryBlock && this.columnarStoreInfo.getAggKeyBlock()[blockIndex[i]]) {
- dataIndex = columnarStoreInfo.getNumberCompressor()
- .unCompress(columnarUniqueblockKeyBlockIndex[mapOfAggDataIndex.get(blockIndex[i])]);
- if (!needCompressedData[i]) {
- columnarKeyBlockDataTemp = UnBlockIndexer
- .uncompressData(columnarKeyBlockDataTemp, dataIndex,
- columnarStoreInfo.getSizeOfEachBlock()[blockIndex[i]]);
- dataIndex = null;
- } else {
- isUnCompressed = false;
- }
- }
- if (!columnarStoreInfo.getIsSorted()[blockIndex[i]]) {
- columnarKeyBlockIndex = mapOfColumnIndexAndColumnBlockIndex.get(blockIndex[i]);
- columnKeyBlockIndex = CarbonUtil.getUnCompressColumnIndex(
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex],
- columnarKeyBlockDataIndex[columnarKeyBlockIndex],
- columnarStoreInfo.getNumberCompressor());
- columnKeyBlockReverseIndexes = getColumnIndexForNonFilter(columnKeyBlockIndex);
- }
- if (isNoDictionaryBlock) {
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(0);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndexes);
- columnarKeyStoreMetadata.setNoDictionaryValColumn(true);
- columnarKeyStoreMetadata.setUnCompressed(true);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[i]]);
- //System is reading the direct surrogates data from byte array which contains both
- // length and the direct surrogates data
- List<byte[]> noDictionaryValBasedKeyBlockData = CompressedColumnarKeyStoreUtil
- .readColumnarKeyBlockDataForNoDictionaryCols(columnarKeyBlockDataTemp);
- columnarKeyStoreDataHolders[i] =
- new ColumnarKeyStoreDataHolder(noDictionaryValBasedKeyBlockData,
- columnarKeyStoreMetadata);
- }
- columnarKeyStoreMetadata =
- new ColumnarKeyStoreMetadata(columnarStoreInfo.getSizeOfEachBlock()[blockIndex[i]]);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[i]]);
- columnarKeyStoreMetadata.setDataIndex(dataIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndexes);
- columnarKeyStoreMetadata.setUnCompressed(isUnCompressed);
- columnarKeyStoreDataHolders[i] =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockDataTemp, columnarKeyStoreMetadata);
- }
- return columnarKeyStoreDataHolders;
- }
-
- @Override
- public ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
- boolean needCompressedData, int[] noDictionaryVals) {
-
- byte[] columnarKeyBlockDataTemp = null;
- int[] columnKeyBlockIndex = null;
- int[] columnKeyBlockReverseIndex = null;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = null;
- int columnarKeyBlockIndex = 0;
- int[] dataIndex = null;
- boolean isUnCompressed = true;
- columnarKeyBlockDataTemp = COMPRESSOR.unCompress(columnarKeyBlockData[blockIndex]);
- boolean isNoDictionaryBlock =
- CompressedColumnarKeyStoreUtil.isNoDictionaryBlock(noDictionaryVals, blockIndex);
- if (!isNoDictionaryBlock && this.columnarStoreInfo.getAggKeyBlock()[blockIndex]) {
- dataIndex = columnarStoreInfo.getNumberCompressor()
- .unCompress(columnarUniqueblockKeyBlockIndex[mapOfAggDataIndex.get(blockIndex)]);
- if (!needCompressedData) {
- columnarKeyBlockDataTemp = UnBlockIndexer
- .uncompressData(columnarKeyBlockDataTemp, dataIndex,
- columnarStoreInfo.getSizeOfEachBlock()[blockIndex]);
- dataIndex = null;
- } else {
- isUnCompressed = false;
- }
- }
- if (!columnarStoreInfo.getIsSorted()[blockIndex]) {
- columnarKeyBlockIndex = mapOfColumnIndexAndColumnBlockIndex.get(blockIndex);
- columnKeyBlockIndex = CarbonUtil.getUnCompressColumnIndex(
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex],
- columnarKeyBlockDataIndex[columnarKeyBlockIndex],
- columnarStoreInfo.getNumberCompressor());
- columnKeyBlockReverseIndex = getColumnIndexForNonFilter(columnKeyBlockIndex);
- }
- if (isNoDictionaryBlock) {
- ColumnarKeyStoreDataHolder colKeystoreDataHolders = CompressedColumnarKeyStoreUtil
- .createColumnarKeyStoreMetadataForHCDims(blockIndex, columnarKeyBlockDataTemp,
- columnKeyBlockIndex, columnKeyBlockReverseIndex, columnarStoreInfo);
- return colKeystoreDataHolders;
- }
- columnarKeyStoreMetadata =
- new ColumnarKeyStoreMetadata(columnarStoreInfo.getSizeOfEachBlock()[blockIndex]);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex]);
- columnarKeyStoreMetadata.setDataIndex(dataIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndex);
- columnarKeyStoreMetadata.setUnCompressed(isUnCompressed);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockDataTemp, columnarKeyStoreMetadata);
- return columnarKeyStoreDataHolders;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarKeyStoreUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarKeyStoreUtil.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarKeyStoreUtil.java
deleted file mode 100644
index b0d7ff8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/compressed/CompressedColumnarKeyStoreUtil.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar.compressed;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreMetadata;
-
-/**
- * Utility helper class for managing the processing of columnar key store block.
- */
-public final class CompressedColumnarKeyStoreUtil {
-
- private CompressedColumnarKeyStoreUtil() {
-
- }
-
- /**
- * @param columnarKeyBlockData
- * @param columnarKeyStoreMetadata
- * @return
- * @author s71955 The high cardinality dimensions rows will be send in byte
- * array with its data length appended in the
- * ColumnarKeyStoreDataHolder byte array since high cardinality dim
- * data will not be part of MDKey/Surrogate keys. In this method the
- * byte array will be scanned and the length which is stored in
- * short will be removed.
- */
- public static List<byte[]> readColumnarKeyBlockDataForNoDictionaryCols(
- byte[] columnarKeyBlockData) {
- List<byte[]> columnarKeyBlockDataList = new ArrayList<byte[]>(50);
- ByteBuffer noDictionaryValKeyStoreDataHolder = ByteBuffer.allocate(columnarKeyBlockData.length);
- noDictionaryValKeyStoreDataHolder.put(columnarKeyBlockData);
- noDictionaryValKeyStoreDataHolder.flip();
- while (noDictionaryValKeyStoreDataHolder.hasRemaining()) {
- short dataLength = noDictionaryValKeyStoreDataHolder.getShort();
- byte[] noDictionaryValKeyData = new byte[dataLength];
- noDictionaryValKeyStoreDataHolder.get(noDictionaryValKeyData);
- columnarKeyBlockDataList.add(noDictionaryValKeyData);
- }
- return columnarKeyBlockDataList;
-
- }
-
- /**
- * @param blockIndex
- * @param columnarKeyBlockData
- * @param columnKeyBlockIndex
- * @param columnKeyBlockReverseIndex
- * @param columnarStoreInfo
- * @return
- */
- public static ColumnarKeyStoreDataHolder createColumnarKeyStoreMetadataForHCDims(int blockIndex,
- byte[] columnarKeyBlockData, int[] columnKeyBlockIndex, int[] columnKeyBlockReverseIndex,
- ColumnarKeyStoreInfo columnarStoreInfo) {
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(0);
- columnarKeyStoreMetadata.setNoDictionaryValColumn(true);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndex);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex]);
- columnarKeyStoreMetadata.setUnCompressed(true);
- List<byte[]> noDictionaryValBasedKeyBlockData = CompressedColumnarKeyStoreUtil
- .readColumnarKeyBlockDataForNoDictionaryCols(columnarKeyBlockData);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder(noDictionaryValBasedKeyBlockData, columnarKeyStoreMetadata);
- return columnarKeyStoreDataHolders;
- }
-
- /**
- * This API will determine whether the requested block index is a No dictionary
- * column index.
- *
- * @param noDictionaryColIndexes
- * @param blockIndex
- * @return
- */
- public static boolean isNoDictionaryBlock(int[] noDictionaryColIndexes, int blockIndex) {
- if (null != noDictionaryColIndexes) {
- for (int noDictionaryValIndex : noDictionaryColIndexes) {
- if (noDictionaryValIndex == blockIndex) {
- return true;
- }
- }
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarFileKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarFileKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarFileKeyStore.java
deleted file mode 100644
index d0b17dc..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarFileKeyStore.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreMetadata;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.impl.key.columnar.AbstractColumnarKeyStore;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-public class UnCompressedColumnarFileKeyStore extends AbstractColumnarKeyStore {
-
- public UnCompressedColumnarFileKeyStore(ColumnarKeyStoreInfo columnarStoreInfo) {
- super(columnarStoreInfo, false, null);
- }
-
- @Override public ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder,
- int[] blockIndex, boolean[] needCompressedData, int[] noDictionaryColIndexes) {
- ColumnarKeyStoreDataHolder[] columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder[blockIndex.length];
- byte[] columnarKeyBlockData = null;
- int[] columnKeyBlockIndex = null;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = null;
- int columnarKeyBlockIndex = 0;
- int[] dataIndex = null;
- int[] columnKeyBlockReverseIndex = null;
- for (int j = 0; j < columnarKeyStoreDataHolders.length; j++) {
- columnarKeyBlockData = fileHolder.readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockOffsets()[blockIndex[j]],
- columnarStoreInfo.getKeyBlockLengths()[blockIndex[j]]);
- if (this.columnarStoreInfo.getAggKeyBlock()[blockIndex[j]]) {
- dataIndex = columnarStoreInfo.getNumberCompressor().unCompress(fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getDataIndexMapOffsets()[mapOfAggDataIndex.get(blockIndex[j])],
- columnarStoreInfo.getDataIndexMapLength()[mapOfAggDataIndex.get(blockIndex[j])]));
- if (!needCompressedData[j]) {
- columnarKeyBlockData = UnBlockIndexer.uncompressData(columnarKeyBlockData, dataIndex,
- columnarStoreInfo.getSizeOfEachBlock()[blockIndex[j]]);
- dataIndex = null;
- }
- }
- if (!columnarStoreInfo.getIsSorted()[blockIndex[j]]) {
- columnarKeyBlockIndex = mapOfColumnIndexAndColumnBlockIndex.get(blockIndex[j]);
- columnKeyBlockIndex = CarbonUtil.getUnCompressColumnIndex(
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex], fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockIndexOffsets()[columnarKeyBlockIndex],
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex]),
- columnarStoreInfo.getNumberCompressor());
- columnKeyBlockReverseIndex = getColumnIndexForNonFilter(columnKeyBlockIndex);
- }
- columnarKeyStoreMetadata =
- new ColumnarKeyStoreMetadata(columnarStoreInfo.getSizeOfEachBlock()[blockIndex[j]]);
- columnarKeyStoreMetadata.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[j]]);
- columnarKeyStoreMetadata.setColumnIndex(columnKeyBlockIndex);
- columnarKeyStoreMetadata.setDataIndex(dataIndex);
- columnarKeyStoreMetadata.setColumnReverseIndex(columnKeyBlockReverseIndex);
- columnarKeyStoreDataHolders[j] =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockData, columnarKeyStoreMetadata);
- }
- return columnarKeyStoreDataHolders;
- }
-
- @Override
- public ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
- boolean needCompressedData, int[] noDictionaryColIndexes) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarInMemoryStore.java
deleted file mode 100644
index da69e01..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/columnar/uncompressed/UnCompressedColumnarInMemoryStore.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.columnar.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreDataHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreMetadata;
-import org.apache.carbondata.core.datastorage.store.impl.key.columnar.AbstractColumnarKeyStore;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-public class UnCompressedColumnarInMemoryStore extends AbstractColumnarKeyStore {
-
- public UnCompressedColumnarInMemoryStore(ColumnarKeyStoreInfo columnarStoreInfo,
- FileHolder fileHolder) {
- super(columnarStoreInfo, true, fileHolder);
- }
-
- @Override public ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder,
- int[] blockIndex, boolean[] needCompressedData, int[] noDictionaryColIndexes) {
- int columnarKeyBlockIndex = 0;
- int[] columnIndex = null;
- ColumnarKeyStoreDataHolder[] columnarKeyStoreDataHolders =
- new ColumnarKeyStoreDataHolder[blockIndex.length];
- ColumnarKeyStoreMetadata columnarKeyStoreMetadataTemp = null;
- for (int i = 0; i < columnarKeyStoreDataHolders.length; i++) {
- columnarKeyStoreMetadataTemp = new ColumnarKeyStoreMetadata(0);
- if (!columnarStoreInfo.getIsSorted()[blockIndex[i]]) {
- columnarKeyBlockIndex = mapOfColumnIndexAndColumnBlockIndex.get(blockIndex[i]);
- columnIndex = CarbonUtil.getUnCompressColumnIndex(
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex], fileHolder
- .readByteArray(columnarStoreInfo.getFilePath(),
- columnarStoreInfo.getKeyBlockIndexOffsets()[columnarKeyBlockIndex],
- columnarStoreInfo.getKeyBlockIndexLength()[columnarKeyBlockIndex]),
- columnarStoreInfo.getNumberCompressor());
- columnIndex = getColumnIndexForNonFilter(columnIndex);
- columnarKeyStoreMetadataTemp.setColumnIndex(columnIndex);
- }
- columnarKeyStoreMetadataTemp.setSorted(columnarStoreInfo.getIsSorted()[blockIndex[i]]);
- columnarKeyStoreDataHolders[i] =
- new ColumnarKeyStoreDataHolder(columnarKeyBlockData[blockIndex[i]],
- columnarKeyStoreMetadataTemp);
- }
- return columnarKeyStoreDataHolders;
- }
-
- @Override
- public ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
- boolean needCompressedData, int[] noDictionaryVals) {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/AbstractCompressedSingleArrayStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/AbstractCompressedSingleArrayStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/AbstractCompressedSingleArrayStore.java
deleted file mode 100644
index 493b61f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/AbstractCompressedSingleArrayStore.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.compressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.NodeKeyStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
-
-public abstract class AbstractCompressedSingleArrayStore implements NodeKeyStore {
-
- /**
- * compressor will be used to compress the data
- */
- protected static final Compressor<byte[]> COMPRESSOR =
- SnappyCompression.SnappyByteCompression.INSTANCE;
- /**
- * size of each element
- */
- protected final int sizeOfEachElement;
- /**
- * data store which will hold the data
- */
- protected byte[] datastore;
- /**
- * total number of elements;
- */
- protected int totalNumberOfElements;
-
- public AbstractCompressedSingleArrayStore(int size, int elementSize) {
- this(size, elementSize, true);
- }
-
- public AbstractCompressedSingleArrayStore(int size, int elementSize, boolean createDataStore) {
- this.sizeOfEachElement = elementSize;
- this.totalNumberOfElements = size;
- if (createDataStore) {
- datastore = new byte[this.totalNumberOfElements * this.sizeOfEachElement];
- }
- }
-
- /**
- * This method will be used to insert key to store
- */
- @Override public void put(int index, byte[] value) {
- System.arraycopy(value, 0, datastore, ((index) * sizeOfEachElement), sizeOfEachElement);
- }
-
- /**
- * This method will be used to get the writable key array.
- * writable key array will hold below information:
- * <size of key array><key array>
- * total length will be stored in 4 bytes+ key array length for key array
- *
- * @return writable array (compressed or normal)
- */
- @Override public byte[] getWritableKeyArray() {
- // compress the data store
- byte[] compressedKeys = COMPRESSOR.compress(datastore);
- return compressedKeys;
- }
-
- /**
- * This method will be used to get the actual key array present in the
- * store .
- * Here back array will be uncompress array
- *
- * @param fileHolder file holder will be used to read the file
- * @return uncompressed keys
- * will return uncompressed key
- */
- @Override public byte[] getBackArray(FileHolder fileHolder) {
- return COMPRESSOR.unCompress(datastore);
- }
-
- /**
- * This method will be used to get the key array based on index
- *
- * @param index index in store
- * @param fileHolder file holder will be used to read the file
- * @return key
- */
- @Override public byte[] get(int index, FileHolder fileHolder) {
- // uncompress the store data
- byte[] unCompress = COMPRESSOR.unCompress(datastore);
- // create new array of size of each element
- byte[] copy = new byte[sizeOfEachElement];
- // copy array for given index
- // copy will done based on below calculation
- // eg: index is 4 and size of each key is 6 then copy from 6*4= 24th
- // index till 29th index
- System.arraycopy(unCompress, ((index) * sizeOfEachElement), copy, 0, sizeOfEachElement);
- return copy;
- }
-
- /**
- * This method will clear the store and create the new empty store
- */
- @Override public void clear() {
- datastore = new byte[this.totalNumberOfElements * this.sizeOfEachElement];
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyFileStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyFileStore.java
deleted file mode 100644
index 0d113d6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyFileStore.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.compressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class CompressedSingleArrayKeyFileStore extends AbstractCompressedSingleArrayStore {
-
- /**
- * offset, this will be used for seek position
- */
- private long offset;
-
- /**
- * fully qualified file path
- */
- private String filePath;
-
- /**
- * length to be read
- */
- private int length;
-
- public CompressedSingleArrayKeyFileStore(int size, int elementSize, long offset, String filePath,
- int length) {
- super(size, elementSize, false);
- this.offset = offset;
- this.filePath = filePath;
- this.length = length;
- }
-
- /**
- * This method will be used to get the actual keys array present in the
- * store . Here back array will be uncompress array. This method will first read
- * the data from file based on offset and length then uncompress the array
- * to get the actual array
- *
- * @param fileHolder file holder will be used to read the file
- * @return uncompressed
- * keys will return uncompressed key
- */
- @Override public byte[] getBackArray(FileHolder fileHolder) {
- if (null != fileHolder) {
- // read from file based on offset and index, fileholder will read that
- // much byte from that offset, then uncompress and return
- return COMPRESSOR.unCompress(fileHolder.readByteArray(filePath, offset, length));
- } else {
- return new byte[0];
- }
- }
-
- /**
- * This method will be used to get the key array based on index
- * This method will first read
- * the data from file based on offset and length then uncompress the array
- * to get the actual array, then get the array for index and return
- *
- * @param index index in store
- * @param fileHolder file holder will be used to read the file
- * @return key
- */
- @Override public byte[] get(int index, FileHolder fileHolder) {
- // read from file based on offset and index, fileholder will read that
- // much byte from that offset, then uncompress to get the actual array
- byte[] unCompress = COMPRESSOR.unCompress(fileHolder.readByteArray(filePath, offset, length));
- // create new array of size of each element
- byte[] copy = new byte[sizeOfEachElement];
- // copy array for given index
- // copy will done based on below calculation
- // eg: index is 4 and size of each key is 6 then copy from 6*4= 24th
- // index till 29th index
- System.arraycopy(unCompress, ((index) * sizeOfEachElement), copy, 0, sizeOfEachElement);
- return copy;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyInMemoryStore.java
deleted file mode 100644
index 612d434..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/compressed/CompressedSingleArrayKeyInMemoryStore.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.compressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class CompressedSingleArrayKeyInMemoryStore extends AbstractCompressedSingleArrayStore {
- /**
- * @param size
- * @param elementSize
- */
- public CompressedSingleArrayKeyInMemoryStore(int size, int elementSize) {
- super(size, elementSize);
- }
-
- /**
- * @param size
- * @param elementSize
- * @param offset
- * @param filePath
- * @param fileHolder
- * @param length
- */
- public CompressedSingleArrayKeyInMemoryStore(int size, int elementSize, long offset,
- String filePath, FileHolder fileHolder, int length) {
- this(size, elementSize);
- datastore = fileHolder.readByteArray(filePath, offset, length);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/AbstractSingleArrayKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/AbstractSingleArrayKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/AbstractSingleArrayKeyStore.java
deleted file mode 100644
index e4141c3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/AbstractSingleArrayKeyStore.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.NodeKeyStore;
-
-public abstract class AbstractSingleArrayKeyStore implements NodeKeyStore {
-
- /**
- * size of each element
- */
- protected final int sizeOfEachElement;
- /**
- * total number of elements
- */
- protected final int totalNumberOfElements;
- /**
- * data store which will hold the data
- */
- protected byte[] datastore;
-
- public AbstractSingleArrayKeyStore(int size, int elementSize) {
- this.sizeOfEachElement = elementSize;
- this.totalNumberOfElements = size;
- datastore = new byte[size * elementSize];
- }
-
- /**
- * This method will be used to insert mdkey to store
- *
- * @param index index of mdkey
- * @param value mdkey
- */
- @Override public void put(int index, byte[] value) {
- System.arraycopy(value, 0, datastore, ((index) * sizeOfEachElement), sizeOfEachElement);
- }
-
- /**
- * This method will be used to get the writable key array.
- * writable key array will hold below information:
- * <size of key array><key array>
- * total length will be stored in 4 bytes+ key array length for key array
- *
- * @return writable array
- */
- @Override public byte[] getWritableKeyArray() {
- // create and allocate size for byte buffer
- // 4 bytes for size of array(for array length) + size of array(for array)
- return datastore;
- }
-
- /**
- * This method will be used to get the actual key array present in the
- * store.
- *
- * @param fileHolder file holder will be used to read the file
- * @return uncompressed keys
- * will return uncompressed key
- */
- @Override public byte[] getBackArray(FileHolder fileHolder) {
- return datastore;
- }
-
- /**
- * This method will be used to get the key array based on index
- *
- * @param index index in store
- * @param fileHolder file holder will be used to read the file
- * @return key
- */
- @Override public byte[] get(int index, FileHolder fileHolder) {
- // create new array of size of each element
- byte[] copy = new byte[sizeOfEachElement];
-
- // copy array for given index
- // copy will done based on below calculation
- // eg: index is 4 and size of each key is 6 then copy from 6*4= 24th
- // index till 29th index
- System.arraycopy(datastore, ((index) * sizeOfEachElement), copy, 0, sizeOfEachElement);
- return copy;
- }
-
- /**
- * This method will clear the store and create the new empty store
- */
- @Override public void clear() {
- datastore = new byte[this.totalNumberOfElements * this.sizeOfEachElement];
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyFileStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyFileStore.java
deleted file mode 100644
index 2d44245..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyFileStore.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class SingleArrayKeyFileStore extends AbstractSingleArrayKeyStore {
- /**
- * offset, this will be used for seek position
- */
- private long offset;
-
- /**
- * fully qualified file path
- */
- private String filePath;
-
- /**
- * length to be read
- */
- private int length;
-
- /**
- * @param size
- * @param elementSize
- */
- public SingleArrayKeyFileStore(int size, int elementSize) {
- super(size, elementSize);
- }
-
- /**
- * @param size
- * @param elementSize
- * @param offset
- * @param filePath
- * @param length
- */
- public SingleArrayKeyFileStore(int size, int elementSize, long offset, String filePath,
- int length) {
- this(size, elementSize);
- this.offset = offset;
- this.filePath = filePath;
- this.length = length;
- datastore = null;
- }
-
- /**
- * This method will be used to get the actual keys array present in the
- * store. This method will read
- * the data from file based on offset and length then return the data read from file
- *
- * @param fileHolder file holder will be used to read the file
- * @return uncompressed
- * keys will return uncompressed key
- */
- @Override public byte[] getBackArray(FileHolder fileHolder) {
- if (null != fileHolder) {
- return fileHolder.readByteArray(filePath, offset, length);
- } else {
- return new byte[0];
- }
- }
-
- /**
- * This method will be used to get the key array based on index This method
- * will first read the data from file based on offset and length then get
- * the array for index and return
- *
- * @param index index in store
- * @param fileHolder file holder will be used to read the file
- * @return key
- */
- @Override public byte[] get(int index, FileHolder fileHolder) {
- // read from file based on offset and index, fileholder will read that
- // much byte from that offset,
- byte[] unCompress = fileHolder.readByteArray(filePath, offset, length);
- // create new array of size of each element
- byte[] copy = new byte[sizeOfEachElement];
- // copy array for given index
- // copy will done based on below calculation
- // eg: index is 4 and size of each key is 6 then copy from 6*4= 24th
- // index till 29th index
- System.arraycopy(unCompress, ((index) * sizeOfEachElement), copy, 0, sizeOfEachElement);
- return copy;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyInMemoryStore.java
deleted file mode 100644
index 6e0dde3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/key/uncompressed/SingleArrayKeyInMemoryStore.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.key.uncompressed;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class SingleArrayKeyInMemoryStore extends AbstractSingleArrayKeyStore {
-
- public SingleArrayKeyInMemoryStore(int size, int elementSize) {
- super(size, elementSize);
- }
-
- public SingleArrayKeyInMemoryStore(int size, int elementSize, long offset, String filePath,
- FileHolder fileHolder, int length) {
- this(size, elementSize);
- datastore = fileHolder.readByteArray(filePath, offset, length);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index bdd6fae..c435dd5 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression.SnappyByteCompression;
+import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
@@ -572,8 +572,8 @@ public class CarbonMetadataUtil {
//meta
PresenceMeta presenceMeta = new PresenceMeta();
presenceMeta.setPresent_bit_streamIsSet(true);
- presenceMeta.setPresent_bit_stream(SnappyByteCompression.INSTANCE
- .compress(blockletInfoColumnar.getMeasureNullValueIndex()[i].toByteArray()));
+ presenceMeta.setPresent_bit_stream(CompressorFactory.getInstance()
+ .compressByte(blockletInfoColumnar.getMeasureNullValueIndex()[i].toByteArray()));
dataChunk.setPresence(presenceMeta);
//TODO : PresenceMeta needs to be implemented and set here
// dataChunk.setPresence(new PresenceMeta());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/25b4ba2c/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
index e1e4088..57379c1 100644
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReaderTest.java
@@ -19,27 +19,27 @@
package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
-import static junit.framework.TestCase.assertEquals;
-
import java.util.ArrayList;
import java.util.List;
import mockit.Mock;
import mockit.MockUp;
-
import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
+import org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension
+ .v1.CompressedDimensionChunkFileBasedReaderV1;
import org.apache.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.datastorage.store.FileHolder;
import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.datastorage.store.compression.SnappyCompressor;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.util.CarbonUtil;
import org.junit.BeforeClass;
import org.junit.Test;
+import static junit.framework.TestCase.assertEquals;
+
public class CompressedDimensionChunkFileBasedReaderTest {
static CompressedDimensionChunkFileBasedReaderV1 compressedDimensionChunkFileBasedReader;
@@ -77,8 +77,8 @@ public class CompressedDimensionChunkFileBasedReaderTest {
}
};
- new MockUp<SnappyCompression.SnappyByteCompression>() {
- @Mock public byte[] unCompress(byte[] compInput) {
+ new MockUp<SnappyCompressor>() {
+ @Mock public byte[] unCompressByte(byte[] compInput) {
byte mockedValue[] = { 1 };
return mockedValue;
}
@@ -121,8 +121,8 @@ public class CompressedDimensionChunkFileBasedReaderTest {
}
};
- new MockUp<SnappyCompression.SnappyByteCompression>() {
- @Mock public byte[] unCompress(byte[] compInput) {
+ new MockUp<SnappyCompressor>() {
+ @Mock public byte[] unCompressByte(byte[] compInput) {
byte mockedValue[] = { 1 };
return mockedValue;
}